From a81d35a1b178d3c05d33d21888dbb632dbe48579 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Thu, 1 Feb 2024 17:39:26 +0100
Subject: [PATCH 01/13] Update user script for DQM job submission

---
 .../dqm_job_submitter/dqm_processor.sh        |  18 ++-
 .../dqm_job_submitter/submit_dqm_processor.py | 132 ++++++++++--------
 2 files changed, 88 insertions(+), 62 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index 728c19fa..cfcfa6aa 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -55,6 +55,7 @@ function exit_script() {
     # Some cleanup before leaving:
     # [ -d $CONTAINER ] && rm -rf $CONTAINER
     # [ -f $CONTAINER ] && rm -f $CONTAINER
+    [ -d $NECTARCAMDATA ] && rm -rf $NECTARCAMDATA
     [ -d $OUTDIR ] && rm -rf $OUTDIR
     [ -f ${OUTDIR}.tar.gz ] && rm -f ${OUTDIR}.tar.gz
     [ -d ${OUTDIR} ] && rm -rf ${OUTDIR}
@@ -63,15 +64,17 @@ function exit_script() {
     exit $return_code
 }
 
+export NECTARCAMDATA=$PWD/runs
+[ ! -d $NECTARCAMDATA ] && mkdir -p $NECTARCAMDATA || exit_script $?
+mv nectarcam*.sqlite NectarCAM.Run*.fits.fz $NECTARCAMDATA/.
+
 # Halim's DQM code needs to use a specific output directory:
 export NECTARDIR=$PWD/$OUTDIR
-[ ! -d $NECTARDIR ] && mkdir -p $NECTARDIR || exit_script $?
-# mv nectarcam*.sqlite NectarCAM.Run*.fits.fz $NECTARDIR/.
 
-LISTRUNS=""
-for run in $PWD/NectarCAM.Run${runnb}.*.fits.fz; do
-    LISTRUNS="$LISTRUNS $(basename $run)"
-done
+#LISTRUNS=""
+#for run in $NECTARCAMDATA/NectarCAM.Run${runnb}.*.fits.fz; do
+#    LISTRUNS="$LISTRUNS $(basename $run)"
+#done
 
 # Create a wrapper BASH script with cleaned environment, see https://redmine.cta-observatory.org/issues/51483
 cat > $WRAPPER <<EOF
@@ -102,7 +105,8 @@ fi
 echo
 echo "Running" 
 # Instantiate the nectarchain Singularity image, run our DQM example run within it:
-cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_calib.py $PWD $NECTARDIR -i $LISTRUNS"
+# cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 NECTARCAMDATA $NECTARDIR -i $LISTRUNS"
+cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 -r $runnb NECTARCAMDATA $NECTARDIR"
 echo \$cmd
 eval \$cmd
 EOF
diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
index ad8779c9..03e5cb40 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
@@ -4,64 +4,72 @@
 # Time-stamp: "2023-05-30 13:09:04 jlenain"
 
 import argparse
-import sys
 import logging
+import sys
 from time import sleep
 
+# The magic DIRAC 2 lines
+import DIRAC
+
 # astropy imports
 from astropy import time
 from astropy import units as u
 
+DIRAC.initialize()
+
 # DIRAC imports
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Interfaces.API.Job import Job
 from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
 
-logging.basicConfig(format='[%(levelname)s] %(message)s')
+logging.basicConfig(format="[%(levelname)s] %(message)s")
 logger = logging.getLogger(__name__)
 
 dirac = Dirac()
 
 # Option and argument parser
 parser = argparse.ArgumentParser()
-parser.add_argument('-d', '--date',
-                    default=None,
-                    help='date for which NectarCAM runs should be processed',
-                    type=str)
-parser.add_argument('-r', '--run',
-                    default=None,
-                    help='only process a specific run (optional)',
-                    type=str)
-parser.add_argument('--dry-run',
-                    action='store_true',
-                    default=False,
-                    help='dry run (does not actually submit jobs)')
-parser.add_argument('--log',
-                    default='info',
-                    help='debug output',
-                    type=str)
+parser.add_argument(
+    "-d",
+    "--date",
+    default=None,
+    help="date for which NectarCAM runs should be processed",
+    type=str,
+)
+parser.add_argument(
+    "-r", "--run", default=None, help="only process a specific run (optional)", type=str
+)
+parser.add_argument(
+    "--dry-run",
+    action="store_true",
+    default=False,
+    help="dry run (does not actually submit jobs)",
+)
+parser.add_argument("--log", default="info", help="debug output", type=str)
 args = parser.parse_args()
 
 logger.setLevel(args.log.upper())
 
 if args.date is None:
-    logger.critical('A date should be provided, in a format astropy.time.Time compliant. E.g. "2022-04-01".')
+    logger.critical(
+        'A date should be provided, in a format astropy.time.Time compliant. E.g. "2022-04-01".'
+    )
     sys.exit(1)
 
-executable_wrapper="dqm_processor.sh"
+executable_wrapper = "dqm_processor.sh"
 
 ## Possible massive job processing via loop on run numbers:
 # for run in ['2720', '3277', '...']:
 
 ## or from DIRAC FileCatalog directory listing:
 processDate = time.Time(args.date)
-dfcDir = f'/vo.cta.in2p3.fr/nectarcam/{processDate.ymdhms[0]}/{processDate.ymdhms[0]}{str(processDate.ymdhms[1]).zfill(2)}{str(processDate.ymdhms[2]).zfill(2)}'
+dfcDir = f"/vo.cta.in2p3.fr/nectarcam/{processDate.ymdhms[0]}/{processDate.ymdhms[0]}{str(processDate.ymdhms[1]).zfill(2)}{str(processDate.ymdhms[2]).zfill(2)}"
 
 # The relevant DB file may be stored in the directory corresponding to the day after:
-processDateTomorrow = processDate + 1. * u.day
-dfcDirTomorrow = f'/vo.cta.in2p3.fr/nectarcam/{processDateTomorrow.ymdhms[0]}/{processDateTomorrow.ymdhms[0]}{str(processDateTomorrow.ymdhms[1]).zfill(2)}{str(processDateTomorrow.ymdhms[2]).zfill(2)}'
+processDateTomorrow = processDate + 1.0 * u.day
+dfcDirTomorrow = f"/vo.cta.in2p3.fr/nectarcam/{processDateTomorrow.ymdhms[0]}/{processDateTomorrow.ymdhms[0]}{str(processDateTomorrow.ymdhms[1]).zfill(2)}{str(processDateTomorrow.ymdhms[2]).zfill(2)}"
 
-# Sometimes, for unkown reason, the connection to the DFC can fail, try a few times:
+# Sometimes, for unknown reason, the connection to the DFC can fail, try a few times:
 sleep_time = 2
 num_retries = 3
 for x in range(0, num_retries):
@@ -76,77 +84,91 @@
     else:
         break
 if not dfc:
-    logger.fatal(f'Connection to FileCatalogClient failed, aborting...')
+    logger.fatal(f"Connection to FileCatalogClient failed, aborting...")
     sys.exit(1)
 
 infos = dfc.listDirectory(dfcDir)
 infosTomorrow = dfc.listDirectory(dfcDirTomorrow)
-if not infos['OK'] or not infos['Value']['Successful']:
-    logger.critical(f"Could not properly retrieve the file metadata for {dfcDir} ... Exiting !")
+if not infos["OK"] or not infos["Value"]["Successful"]:
+    logger.critical(
+        f"Could not properly retrieve the file metadata for {dfcDir} ... Exiting !"
+    )
     sys.exit(1)
-if not infosTomorrow['OK'] or not infosTomorrow['Value']['Successful']:
-    logger.warning(f"Could not properly retrieve the file metadata for {dfcDirTomorrow} ... Continuing !")
-meta = infos['Value']['Successful'][dfcDir]
+if not infosTomorrow["OK"] or not infosTomorrow["Value"]["Successful"]:
+    logger.warning(
+        f"Could not properly retrieve the file metadata for {dfcDirTomorrow} ... Continuing !"
+    )
+meta = infos["Value"]["Successful"][dfcDir]
 try:
-    metaTomorrow = infosTomorrow['Value']['Successful'][dfcDirTomorrow]
+    metaTomorrow = infosTomorrow["Value"]["Successful"][dfcDirTomorrow]
 except KeyError:
     metaTomorrow = None
 
 runlist = []
 
 sqlfilelist = []
-for f in meta['Files']:
-    if f.endswith('.fits.fz'):
-        run = f.split('NectarCAM.Run')[1].split('.')[0]
+for f in meta["Files"]:
+    if f.endswith(".fits.fz"):
+        run = f.split("NectarCAM.Run")[1].split(".")[0]
         if run not in runlist and run is not None:
             runlist.append(run)
-    if f.endswith('.sqlite'):
+    if f.endswith(".sqlite"):
         sqlfilelist.append(f)
 if metaTomorrow:
-    for f in metaTomorrow['Files']:
-        if f.endswith('.sqlite'):
+    for f in metaTomorrow["Files"]:
+        if f.endswith(".sqlite"):
             sqlfilelist.append(f)
 if args.run is not None:
     if args.run not in runlist:
-        logger.critical(f'Your specified run {args.run} was not found in {dfcDir}, aborting...')
+        logger.critical(
+            f"Your specified run {args.run} was not found in {dfcDir}, aborting..."
+        )
         sys.exit(1)
     runlist = [args.run]
-    
-logger.info(f'Found runs {runlist} in {dfcDir}')
+
+logger.info(f"Found runs {runlist} in {dfcDir}")
 
 if len(sqlfilelist) == 0:
-    logger.critical('Could not find any SQLite file in {dfcDir} nor in {dfcDirTomorrow}, aborting...')
+    logger.critical(
+        "Could not find any SQLite file in {dfcDir} nor in {dfcDirTomorrow}, aborting..."
+    )
     sys.exit(1)
-logger.info(f'Found SQLite files {sqlfilelist} in {dfcDir} and {dfcDirTomorrow}')
+logger.info(f"Found SQLite files {sqlfilelist} in {dfcDir} and {dfcDirTomorrow}")
 
 # Now, submit the DIRAC jobs:
 # for run in ['2721']:
 for run in runlist:
     j = Job()
     # j.setExecutable(f'{executable_wrapper}', '<SOME POSSIBLE ARGUMENTS such as run number>')
-    j.setExecutable(f'{executable_wrapper}', f'-r {run}')
+    j.setExecutable(f"{executable_wrapper}", f"-r {run}")
     # Force job to be run from a given Computing Element:
     # j.setDestination('LCG.GRIF.fr')
-    j.setName(f'NectarCAM DQM run {run}')
-    j.setJobGroup('NectarCAM DQM')
-    sandboxlist = [f'{executable_wrapper}']
-    for f in meta['Files']:
-        if f.endswith('.fits.fz') and f'NectarCAM.Run{run}' in f:
-            sandboxlist.append(f'LFN:{f}')
+    j.setName(f"NectarCAM DQM run {run}")
+    j.setJobGroup("NectarCAM DQM")
+    sandboxlist = [f"{executable_wrapper}"]
+    for f in meta["Files"]:
+        if f.endswith(".fits.fz") and f"NectarCAM.Run{run}" in f:
+            sandboxlist.append(f"LFN:{f}")
     for s in sqlfilelist:
-        sandboxlist.append(f'LFN:{s}')
+        sandboxlist.append(f"LFN:{s}")
     if len(sandboxlist) < 2:
-        logger.critical(f'''Misformed sandboxlist, actual data .fits.fz files missing:
+        logger.critical(
+            f"""Misformed sandboxlist, actual data .fits.fz files missing:
 {sandboxlist}
 
 Aborting...
-''')
+"""
+        )
         sys.exit(1)
-    logger.info(f'''Submitting job for run {run}, with the following InputSandbox:
+    logger.info(
+        f"""Submitting job for run {run}, with the following InputSandbox:
 {sandboxlist}
-''')
+"""
+    )
     j.setInputSandbox(sandboxlist)
 
     if not args.dry_run:
-        res = dirac.submitJob(j)  # , mode='local')  # for local execution, simulating a DIRAC job on the local machine, instead of submitting it to a DIRAC Computing Element
+        res = dirac.submitJob(
+            j
+        )  # , mode='local')  # for local execution, simulating a DIRAC job on the local machine, instead of submitting it to a DIRAC Computing Element
         logger.info(f"Submission Result: {res['Value']}")

From 2b9132d5679376958f66448249c5acc624ff3169 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Thu, 1 Feb 2024 17:53:26 +0100
Subject: [PATCH 02/13] Work on user script for DQM job submission

---
 .../jlenain/dqm_job_submitter/dqm_processor.sh        | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index cfcfa6aa..ba0d915e 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -5,7 +5,7 @@
 
 function usage ()
 {
-    echo "Usage: `basename $0` -r <run number>"
+    echo "Usage: $(basename $0) -r <run number>"
 }
 
 function help ()
@@ -82,6 +82,11 @@ cat > $WRAPPER <<EOF
 echo "Cleaning environment \$CLEANED_ENV" 
 [ -z "\$CLEANED_ENV" ] && exec /bin/env -i CLEANED_ENV="Done" HOME=\${HOME} SHELL=/bin/bash /bin/bash -l "\$0" "\$@" 
 
+# From https://github.com/DIRACGrid/COMDIRAC/wiki/Injob
+# initialize job for COMDIRAC commands
+export DCOMMANDS_CONFIG_DIR=$PWD
+dconfig --guess
+dinit --fromProxy
 
 # Some environment variables related to python, to be passed to container, be it for old Singularity version or recent Apptainer ones:
 export SINGULARITYENV_MPLCONFIGDIR=/tmp
@@ -105,8 +110,8 @@ fi
 echo
 echo "Running" 
 # Instantiate the nectarchain Singularity image, run our DQM example run within it:
-# cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 NECTARCAMDATA $NECTARDIR -i $LISTRUNS"
-cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 -r $runnb NECTARCAMDATA $NECTARDIR"
+# cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 $NECTARCAMDATA $NECTARDIR -i $LISTRUNS"
+cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 -r $runnb $NECTARCAMDATA $NECTARDIR"
 echo \$cmd
 eval \$cmd
 EOF

From 3ea16bebe1409bc4170dcc47c0d1a62d25b5d6c6 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 11:06:03 +0100
Subject: [PATCH 03/13] Give up on COMDIRAC initialization: commands not found
 on DIRAC workers

---
 .../user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index ba0d915e..4cfd47c1 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -82,12 +82,6 @@ cat > $WRAPPER <<EOF
 echo "Cleaning environment \$CLEANED_ENV" 
 [ -z "\$CLEANED_ENV" ] && exec /bin/env -i CLEANED_ENV="Done" HOME=\${HOME} SHELL=/bin/bash /bin/bash -l "\$0" "\$@" 
 
-# From https://github.com/DIRACGrid/COMDIRAC/wiki/Injob
-# initialize job for COMDIRAC commands
-export DCOMMANDS_CONFIG_DIR=$PWD
-dconfig --guess
-dinit --fromProxy
-
 # Some environment variables related to python, to be passed to container, be it for old Singularity version or recent Apptainer ones:
 export SINGULARITYENV_MPLCONFIGDIR=/tmp
 export SINGULARITYENV_NUMBA_CACHE_DIR=/tmp

From c13508d74b7d88936737da501f0170a53cf80e66 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 12:01:36 +0100
Subject: [PATCH 04/13] The environment variable NECTARCAMDATA also needs to be
 passed into the container at runtime

---
 .../user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh | 2 ++
 .../jlenain/dqm_job_submitter/submit_dqm_processor.py       | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index 4cfd47c1..cde62c98 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -85,10 +85,12 @@ echo "Cleaning environment \$CLEANED_ENV"
 # Some environment variables related to python, to be passed to container, be it for old Singularity version or recent Apptainer ones:
 export SINGULARITYENV_MPLCONFIGDIR=/tmp
 export SINGULARITYENV_NUMBA_CACHE_DIR=/tmp
+export SINGULARITYENV_NECTARCAMDATA=$NECTARCAMDATA
 export SINGULARITYENV_NECTARDIR=$NECTARDIR
 
 export APPTAINERENV_MPLCONFIGDIR=/tmp
 export APPTAINERENV_NUMBA_CACHE_DIR=/tmp
+export APPTAINERENV_NECTARCAMDATA=$NECTARCAMDATA
 export APPTAINERENV_NECTARDIR=$NECTARDIR
 
 # Handle Singularity or Apptainer case:
diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
index 03e5cb40..e3615c76 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
@@ -8,15 +8,15 @@
 import sys
 from time import sleep
 
-# The magic DIRAC 2 lines
+# DIRAC client initialization
 import DIRAC
 
+DIRAC.initialize()
+
 # astropy imports
 from astropy import time
 from astropy import units as u
 
-DIRAC.initialize()
-
 # DIRAC imports
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Interfaces.API.Job import Job

From 711afa94e4ddd854151ac9d1b8480bad8a17290e Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 12:22:38 +0100
Subject: [PATCH 05/13] Bug in handling $NECTARCAMDATA/runs

---
 .../user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index cde62c98..3a6b24cd 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -64,9 +64,9 @@ function exit_script() {
     exit $return_code
 }
 
-export NECTARCAMDATA=$PWD/runs
-[ ! -d $NECTARCAMDATA ] && mkdir -p $NECTARCAMDATA || exit_script $?
-mv nectarcam*.sqlite NectarCAM.Run*.fits.fz $NECTARCAMDATA/.
+export NECTARCAMDATA=$PWD
+[ ! -d $NECTARCAMDATA/runs ] && mkdir -p $NECTARCAMDATA/runs || exit_script $?
+mv nectarcam*.sqlite NectarCAM.Run*.fits.fz $NECTARCAMDATA/runs/.
 
 # Halim's DQM code needs to use a specific output directory:
 export NECTARDIR=$PWD/$OUTDIR

From 74692f40aab677ba20de9830d9ceba8f5716adef Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 14:26:46 +0100
Subject: [PATCH 06/13] I was deleting $PWD on exit, which prevented any output
 sandbox to be retrievable !

---
 .../dqm_job_submitter/dqm_processor.sh        | 22 +++++++++----------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index 3a6b24cd..7f519973 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -55,7 +55,7 @@ function exit_script() {
     # Some cleanup before leaving:
     # [ -d $CONTAINER ] && rm -rf $CONTAINER
     # [ -f $CONTAINER ] && rm -f $CONTAINER
-    [ -d $NECTARCAMDATA ] && rm -rf $NECTARCAMDATA
+    [ -d "$NECTARCAMDATA/runs" ] && rm -rf "$NECTARCAMDATA/runs"
     [ -d $OUTDIR ] && rm -rf $OUTDIR
     [ -f ${OUTDIR}.tar.gz ] && rm -f ${OUTDIR}.tar.gz
     [ -d ${OUTDIR} ] && rm -rf ${OUTDIR}
@@ -82,22 +82,20 @@ cat > $WRAPPER <<EOF
 echo "Cleaning environment \$CLEANED_ENV" 
 [ -z "\$CLEANED_ENV" ] && exec /bin/env -i CLEANED_ENV="Done" HOME=\${HOME} SHELL=/bin/bash /bin/bash -l "\$0" "\$@" 
 
-# Some environment variables related to python, to be passed to container, be it for old Singularity version or recent Apptainer ones:
-export SINGULARITYENV_MPLCONFIGDIR=/tmp
-export SINGULARITYENV_NUMBA_CACHE_DIR=/tmp
-export SINGULARITYENV_NECTARCAMDATA=$NECTARCAMDATA
-export SINGULARITYENV_NECTARDIR=$NECTARDIR
-
-export APPTAINERENV_MPLCONFIGDIR=/tmp
-export APPTAINERENV_NUMBA_CACHE_DIR=/tmp
-export APPTAINERENV_NECTARCAMDATA=$NECTARCAMDATA
-export APPTAINERENV_NECTARDIR=$NECTARDIR
-
 # Handle Singularity or Apptainer case:
+# Some environment variables, related to python or nectarchain, have to be passed to container, be it for old Singularity versions or recent Apptainer ones:
 if command -v singularity &> /dev/null; then
     CALLER=singularity
+    export SINGULARITYENV_MPLCONFIGDIR=/tmp
+    export SINGULARITYENV_NUMBA_CACHE_DIR=/tmp
+    export SINGULARITYENV_NECTARCAMDATA=$NECTARCAMDATA
+    export SINGULARITYENV_NECTARDIR=$NECTARDIR
 elif command -v apptainer &> /dev/null; then
     CALLER=apptainer
+    export APPTAINERENV_MPLCONFIGDIR=/tmp
+    export APPTAINERENV_NUMBA_CACHE_DIR=/tmp
+    export APPTAINERENV_NECTARCAMDATA=$NECTARCAMDATA
+    export APPTAINERENV_NECTARDIR=$NECTARDIR
 else
     echo "It seems neither Singularity nor Apptainer are available from here"
     exit 1

From c28dd1ecb1afc65dadb1373461a9648d7521f03d Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 15:24:02 +0100
Subject: [PATCH 07/13] I forgot to specify the destination SE in
 dirac-dms-add-file (:facepalm:)

---
 .../user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index 7f519973..bbafa934 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -116,6 +116,6 @@ chmod u+x $WRAPPER || exit_script $?
 
 # Archive the output directory and push it on DIRAC before leaving the job:
 tar zcf ${OUTDIR}.tar.gz ${OUTDIR}/ || exit_script $?
-dirac-dms-add-file ${DIRAC_OUTDIR}/${OUTDIR}.tar.gz ${OUTDIR}.tar.gz || exit_script $?
+dirac-dms-add-file ${DIRAC_OUTDIR}/${OUTDIR}.tar.gz ${OUTDIR}.tar.gz GRIF-USER || exit_script $?
 
 exit_script 0

From ac697606b8193d34a2eae64f6f33f24e83fd9cc8 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 15:36:26 +0100
Subject: [PATCH 08/13] Update destination SE on DIRAC to GRIF instead of LPNHE

---
 .../user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh b/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh
index 5f0ac95e..c6293083 100755
--- a/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh
+++ b/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/GainFitter.sh
@@ -94,7 +94,7 @@ cd $ORIGPWD
 
 # Archive the output directory and push it on DIRAC before leaving the job:
 tar zcf ${OUTDIR}.tar.gz ${OUTDIR}/
-dirac-dms-add-file ${DIRAC_OUTDIR}/${OUTDIR}.tar.gz ${OUTDIR}.tar.gz LPNHE-USER
+dirac-dms-add-file ${DIRAC_OUTDIR}/${OUTDIR}.tar.gz ${OUTDIR}.tar.gz GRIF-USER
 
 # Some cleanup before leaving:
 [ -d $OUTDIR ] && rm -rf $OUTDIR

From 750f320432fdf51d7ab28ae526b4aeffc9b97a26 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 15:39:50 +0100
Subject: [PATCH 09/13] Use canonical quoting (SC2086)

---
 .../dqm_job_submitter/dqm_processor.sh        | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index bbafa934..5ac2dd1c 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -5,7 +5,7 @@
 
 function usage ()
 {
-    echo "Usage: $(basename $0) -r <run number>"
+    echo "Usage: $(basename "$0") -r <run number>"
 }
 
 function help ()
@@ -36,7 +36,7 @@ while getopts ":hr:" option; do
 done
 shift $((OPTIND-1))
 
-if [ -z $runnb ]; then
+if [ -z "$runnb" ]; then
     usage
     exit 1
 fi
@@ -56,17 +56,17 @@ function exit_script() {
     # [ -d $CONTAINER ] && rm -rf $CONTAINER
     # [ -f $CONTAINER ] && rm -f $CONTAINER
     [ -d "$NECTARCAMDATA/runs" ] && rm -rf "$NECTARCAMDATA/runs"
-    [ -d $OUTDIR ] && rm -rf $OUTDIR
-    [ -f ${OUTDIR}.tar.gz ] && rm -f ${OUTDIR}.tar.gz
-    [ -d ${OUTDIR} ] && rm -rf ${OUTDIR}
+    [ -d "$OUTDIR" ] && rm -rf "$OUTDIR"
+    [ -f "${OUTDIR}.tar.gz" ] && rm -f "${OUTDIR}.tar.gz"
+    [ -d "${OUTDIR}" ] && rm -rf "${OUTDIR}"
     [ -f $WRAPPER ] && rm -f $WRAPPER
 
-    exit $return_code
+    exit "$return_code"
 }
 
 export NECTARCAMDATA=$PWD
-[ ! -d $NECTARCAMDATA/runs ] && mkdir -p $NECTARCAMDATA/runs || exit_script $?
-mv nectarcam*.sqlite NectarCAM.Run*.fits.fz $NECTARCAMDATA/runs/.
+[ ! -d "$NECTARCAMDATA/runs" ] && mkdir -p "$NECTARCAMDATA/runs" || exit_script $?
+mv nectarcam*.sqlite NectarCAM.Run*.fits.fz "$NECTARCAMDATA/runs/."
 
 # Halim's DQM code needs to use a specific output directory:
 export NECTARDIR=$PWD/$OUTDIR
@@ -115,7 +115,7 @@ chmod u+x $WRAPPER || exit_script $?
 
 
 # Archive the output directory and push it on DIRAC before leaving the job:
-tar zcf ${OUTDIR}.tar.gz ${OUTDIR}/ || exit_script $?
-dirac-dms-add-file ${DIRAC_OUTDIR}/${OUTDIR}.tar.gz ${OUTDIR}.tar.gz GRIF-USER || exit_script $?
+tar zcf "${OUTDIR}.tar.gz" "${OUTDIR}/" || exit_script $?
+dirac-dms-add-file "${DIRAC_OUTDIR}/${OUTDIR}.tar.gz" "${OUTDIR}.tar.gz" GRIF-USER || exit_script $?
 
 exit_script 0

From eb64162d3c6127feb58f8fa02f8493fbdf5fa5de Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 16:50:57 +0100
Subject: [PATCH 10/13] Changed destination SE

---
 .../user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index 5ac2dd1c..046e5370 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -116,6 +116,6 @@ chmod u+x $WRAPPER || exit_script $?
 
 # Archive the output directory and push it on DIRAC before leaving the job:
 tar zcf "${OUTDIR}.tar.gz" "${OUTDIR}/" || exit_script $?
-dirac-dms-add-file "${DIRAC_OUTDIR}/${OUTDIR}.tar.gz" "${OUTDIR}.tar.gz" GRIF-USER || exit_script $?
+dirac-dms-add-file "${DIRAC_OUTDIR}/${OUTDIR}.tar.gz" "${OUTDIR}.tar.gz" CC-IN2P3-USER || exit_script $?
 
 exit_script 0

From 33c983796037c382bd668825e9f3b48807927008 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 16:55:33 +0100
Subject: [PATCH 11/13] Even if singularity binary available, some jobs tell
 that: INFO:    Environment variable SINGULARITYENV_MPLCONFIGDIR is set, but
 APPTAINERENV_MPLCONFIGDIR is preferred

---
 .../dqm_job_submitter/dqm_processor.sh        | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index 046e5370..8a5cfacf 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -79,23 +79,26 @@ export NECTARDIR=$PWD/$OUTDIR
 # Create a wrapper BASH script with cleaned environment, see https://redmine.cta-observatory.org/issues/51483
 cat > $WRAPPER <<EOF
 #!/bin/env bash
+# See https://redmine.cta-observatory.org/issues/51483: start container from a clean environment
 echo "Cleaning environment \$CLEANED_ENV" 
 [ -z "\$CLEANED_ENV" ] && exec /bin/env -i CLEANED_ENV="Done" HOME=\${HOME} SHELL=/bin/bash /bin/bash -l "\$0" "\$@" 
 
-# Handle Singularity or Apptainer case:
 # Some environment variables, related to python or nectarchain, have to be passed to container, be it for old Singularity versions or recent Apptainer ones:
+export SINGULARITYENV_MPLCONFIGDIR=/tmp
+export SINGULARITYENV_NUMBA_CACHE_DIR=/tmp
+export SINGULARITYENV_NECTARCAMDATA=$NECTARCAMDATA
+export SINGULARITYENV_NECTARDIR=$NECTARDIR
+
+export APPTAINERENV_MPLCONFIGDIR=/tmp
+export APPTAINERENV_NUMBA_CACHE_DIR=/tmp
+export APPTAINERENV_NECTARCAMDATA=$NECTARCAMDATA
+export APPTAINERENV_NECTARDIR=$NECTARDIR
+
+# Handle Singularity or Apptainer case:
 if command -v singularity &> /dev/null; then
     CALLER=singularity
-    export SINGULARITYENV_MPLCONFIGDIR=/tmp
-    export SINGULARITYENV_NUMBA_CACHE_DIR=/tmp
-    export SINGULARITYENV_NECTARCAMDATA=$NECTARCAMDATA
-    export SINGULARITYENV_NECTARDIR=$NECTARDIR
 elif command -v apptainer &> /dev/null; then
     CALLER=apptainer
-    export APPTAINERENV_MPLCONFIGDIR=/tmp
-    export APPTAINERENV_NUMBA_CACHE_DIR=/tmp
-    export APPTAINERENV_NECTARCAMDATA=$NECTARCAMDATA
-    export APPTAINERENV_NECTARDIR=$NECTARDIR
 else
     echo "It seems neither Singularity nor Apptainer are available from here"
     exit 1

From 25a1e217fa424400fe1debb8bc9cdc3d92d7ea7e Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 17:11:16 +0100
Subject: [PATCH 12/13] Generate plots on DIRAC

---
 .../user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
index 8a5cfacf..ae2cf3a6 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/dqm_processor.sh
@@ -108,7 +108,7 @@ echo
 echo "Running" 
 # Instantiate the nectarchain Singularity image, run our DQM example run within it:
 # cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 $NECTARCAMDATA $NECTARDIR -i $LISTRUNS"
-cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 -r $runnb $NECTARCAMDATA $NECTARDIR"
+cmd="\$CALLER exec --home $PWD $CONTAINER /opt/conda/envs/nectarchain/bin/python /opt/cta/nectarchain/src/nectarchain/dqm/start_dqm.py --r0 --plot --runnb $runnb $NECTARCAMDATA $NECTARDIR"
 echo \$cmd
 eval \$cmd
 EOF

From a7d0ca465d4baf67b543b5ff83a50a86110cc240 Mon Sep 17 00:00:00 2001
From: Jean-Philippe Lenain <jlenain@in2p3.fr>
Date: Fri, 2 Feb 2024 17:31:53 +0100
Subject: [PATCH 13/13] Add some doc.

---
 .../jlenain/dqm_job_submitter/submit_dqm_processor.py     | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
index e3615c76..1ec0ed2a 100755
--- a/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
+++ b/src/nectarchain/user_scripts/jlenain/dqm_job_submitter/submit_dqm_processor.py
@@ -28,7 +28,7 @@
 dirac = Dirac()
 
 # Option and argument parser
-parser = argparse.ArgumentParser()
+parser = argparse.ArgumentParser(description="Submit jobs on DIRAC to run the DQM")
 parser.add_argument(
     "-d",
     "--date",
@@ -37,7 +37,11 @@
     type=str,
 )
 parser.add_argument(
-    "-r", "--run", default=None, help="only process a specific run (optional)", type=str
+    "-r",
+    "--run",
+    default=None,
+    help="only process a specific run (optional). When omitted, all the runs acquired on DATE are processed (1 job per run).",
+    type=str,
 )
 parser.add_argument(
     "--dry-run",