From 6934d27234bc0d1e4a4341615410badd12f13e93 Mon Sep 17 00:00:00 2001
From: Samuel Farrens <samuel.farrens@gmail.com>
Date: Wed, 16 Nov 2022 09:36:13 +0100
Subject: [PATCH 001/133] added Dockerfile and GH action

---
 .github/workflows/deploy-image.yml | 41 ++++++++++++++++++++++++++++++
 Dockerfile                         | 26 +++++++++++++++++++
 2 files changed, 67 insertions(+)
 create mode 100644 .github/workflows/deploy-image.yml
 create mode 100644 Dockerfile

diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
new file mode 100644
index 000000000..81c31c7ad
--- /dev/null
+++ b/.github/workflows/deploy-image.yml
@@ -0,0 +1,41 @@
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches: ['master']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 000000000..e9529e52c
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,26 @@
+FROM continuumio/miniconda3
+
+LABEL Description="ShapePipe Docker Image"
+WORKDIR /home
+ENV SHELL /bin/bash
+
+ARG CC=gcc-9
+ARG CXX=g++-9
+
+RUN apt-get update --allow-releaseinfo-change && \
+    apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install gcc-9 g++-9 -y && \
+    apt-get install locales -y && \
+    apt install libgl1-mesa-glx -y && \
+    apt-get clean
+
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
+    locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+COPY . /home
+
+RUN ./install_shapepipe --develop --vos

From 82bc72ac0a992d1f73b43ddd7b41b8807634877b Mon Sep 17 00:00:00 2001
From: sfarrens <samuel.farrens@gmail.com>
Date: Wed, 16 Nov 2022 16:44:17 +0100
Subject: [PATCH 002/133] added make dependency

---
 Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile b/Dockerfile
index e9529e52c..49cfa01ab 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,6 +10,7 @@ ARG CXX=g++-9
 RUN apt-get update --allow-releaseinfo-change && \
     apt-get update && \
     apt-get upgrade -y && \
+    apt-get install make -y && \
     apt-get install gcc-9 g++-9 -y && \
     apt-get install locales -y && \
     apt install libgl1-mesa-glx -y && \

From 5b447439ecaba42da88b9d93437bd5c6fa097674 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 15 May 2023 14:50:35 +0200
Subject: [PATCH 003/133] notebook

---
 scripts/jupyter/plot_spectro_areas.ipynb | 60 +++++++++++++++++-------
 1 file changed, 42 insertions(+), 18 deletions(-)

diff --git a/scripts/jupyter/plot_spectro_areas.ipynb b/scripts/jupyter/plot_spectro_areas.ipynb
index b41326a17..f608c7aee 100644
--- a/scripts/jupyter/plot_spectro_areas.ipynb
+++ b/scripts/jupyter/plot_spectro_areas.ipynb
@@ -9,14 +9,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Warning: surveys missing because pymangle is not installed\n"
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'skymapper'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn [1], line 12\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mastropy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m units \u001b[38;5;28;01mas\u001b[39;00m u\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[0;32m---> 12\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mskymapper\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mskm\u001b[39;00m\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'skymapper'"
      ]
     }
    ],
@@ -27,7 +31,7 @@
     "import numpy as np\n",
     "import pylab as plt\n",
     "from astropy.io import fits\n",
-    "from cs_util import cfis\n",
+    "from shapepipe.utilities import cfis\n",
     "from astropy.coordinates import SkyCoord, match_coordinates_sky\n",
     "from astropy import units as u\n",
     "import pandas as pd\n",
@@ -53,7 +57,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -101,7 +105,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -194,9 +198,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DESI: 17450894 objects in total, 4835230/17450894 = 27.7% potentially in UNIONS footprint\n"
+     ]
+    }
+   ],
    "source": [
     "surveys = []\n",
     "\n",
@@ -299,8 +311,8 @@
     "    \n",
     "if 'DESI-Iron'in use:\n",
     "    surveys.append(Survey.from_fits('DESI', 'cyan',\n",
-    "                                    '{}/.fits'.format(cat_home),\n",
-    "                                    key_ra='RA[deg]', key_dec='DEC[deg]'))\n",
+    "                                    '{}/galaxies_desi_iron_v0.1.fits'.format(cat_home),\n",
+    "                                    key_ra='TARGET_RA', key_dec='TARGET_DEC'))\n",
     "    "
    ]
   },
@@ -313,7 +325,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -378,9 +390,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "AttributeError",
+     "evalue": "module 'cs_util.cfis' has no attribute 'get_image_list'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[6], line 24\u001b[0m\n\u001b[1;32m     22\u001b[0m images \u001b[38;5;241m=\u001b[39m {}\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m band \u001b[38;5;129;01min\u001b[39;00m bands:\n\u001b[0;32m---> 24\u001b[0m     images[band] \u001b[38;5;241m=\u001b[39m \u001b[43mcfis\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_image_list\u001b[49m(tiles_ID[band], band, image_type, input_format\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mID_only\u001b[39m\u001b[38;5;124m'\u001b[39m, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
+      "\u001b[0;31mAttributeError\u001b[0m: module 'cs_util.cfis' has no attribute 'get_image_list'"
+     ]
+    }
+   ],
    "source": [
     "# Variables\n",
     "unit = 'deg'\n",
@@ -644,9 +668,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "sp_validation",
+   "display_name": "shapepipe",
    "language": "python",
-   "name": "sp_validation"
+   "name": "shapepipe"
   },
   "language_info": {
    "codemirror_mode": {
@@ -658,7 +682,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.0"
+   "version": "3.9.13"
   }
  },
  "nbformat": 4,

From 29c609e52985abe62c281d703a68771cc315e54f Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 1 Nov 2023 15:20:31 +0100
Subject: [PATCH 004/133] docker testing

---
 Dockerfile      | 11 +++++++++
 environment.yml |  1 +
 nsswitch.conf   | 62 +++++++++++++++++++++++++++++++++++++++++++++++++
 xterm-start.sh  |  2 ++
 4 files changed, 76 insertions(+)
 create mode 100644 nsswitch.conf
 create mode 100755 xterm-start.sh

diff --git a/Dockerfile b/Dockerfile
index 49cfa01ab..1f966302b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,8 +14,13 @@ RUN apt-get update --allow-releaseinfo-change && \
     apt-get install gcc-9 g++-9 -y && \
     apt-get install locales -y && \
     apt install libgl1-mesa-glx -y && \
+    apt-get install xterm -y && \
     apt-get clean
 
+RUN apt-get install acl -y && \
+    apt-get install sssd -y
+ADD nsswitch.conf /etc/
+
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
     locale-gen
 ENV LANG en_US.UTF-8
@@ -25,3 +30,9 @@ ENV LC_ALL en_US.UTF-8
 COPY . /home
 
 RUN ./install_shapepipe --develop --vos
+
+# Create entrypoint script for desktop
+RUN mkdir /skaha
+COPY xterm_start.sh /skaha/init.sh
+
+ENTRYPOINT [ "/skaha/init.sh" ]
diff --git a/environment.yml b/environment.yml
index 24f6dfd5f..2e98f669a 100644
--- a/environment.yml
+++ b/environment.yml
@@ -15,6 +15,7 @@ dependencies:
   - numba==0.54.1
   - pandas==1.4.1
   - pip:
+    - jupyter
     - mccd==1.2.3
     - modopt==1.6.0
     - PyQt5==5.15.6
diff --git a/nsswitch.conf b/nsswitch.conf
new file mode 100644
index 000000000..22d8d9949
--- /dev/null
+++ b/nsswitch.conf
@@ -0,0 +1,62 @@
+#
+# /etc/nsswitch.conf
+#
+# An example Name Service Switch config file. This file should be
+# sorted with the most-used services at the beginning.
+#
+# The entry '[NOTFOUND=return]' means that the search for an
+# entry should stop if the search in the previous entry turned
+# up nothing. Note that if the search failed due to some other reason
+# (like no NIS server responding) then the search continues with the
+# next entry.
+#
+# Valid entries include:
+#
+#	nisplus			Use NIS+ (NIS version 3)
+#	nis			Use NIS (NIS version 2), also called YP
+#	dns			Use DNS (Domain Name Service)
+#	files			Use the local files
+#	db			Use the local database (.db) files
+#	compat			Use NIS on compat mode
+#	hesiod			Use Hesiod for user lookups
+#	[NOTFOUND=return]	Stop searching if not found so far
+#
+
+# To use db, put the "db" in front of "files" for entries you want to be
+# looked up first in the databases
+#
+# Example:
+#passwd:    db files nisplus nis
+#shadow:    db files nisplus nis
+#group:     db files nisplus nis
+
+passwd:     sss files
+shadow:     files sss
+group:      sss files
+
+#hosts:     db files nisplus nis dns
+hosts:      files dns
+
+# Example - obey only what nisplus tells us...
+#services:   nisplus [NOTFOUND=return] files
+#networks:   nisplus [NOTFOUND=return] files
+#protocols:  nisplus [NOTFOUND=return] files
+#rpc:        nisplus [NOTFOUND=return] files
+#ethers:     nisplus [NOTFOUND=return] files
+#netmasks:   nisplus [NOTFOUND=return] files
+
+bootparams: nisplus [NOTFOUND=return] files
+
+ethers:     files
+netmasks:   files
+networks:   files
+protocols:  files
+rpc:        files
+services:   files
+
+netgroup:   nisplus
+
+publickey:  nisplus
+
+automount:  files nisplus
+aliases:    files nisplus
diff --git a/xterm-start.sh b/xterm-start.sh
new file mode 100755
index 000000000..007a2516f
--- /dev/null
+++ b/xterm-start.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+xterm

From 5770367157a3e0b50d38472e6f4a6d14e541c445 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 8 Nov 2023 13:12:05 +0100
Subject: [PATCH 005/133] Dockerfile tests

---
 Dockerfile | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1f966302b..2235da657 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,6 +15,7 @@ RUN apt-get update --allow-releaseinfo-change && \
     apt-get install locales -y && \
     apt install libgl1-mesa-glx -y && \
     apt-get install xterm -y && \
+    apt-get install cmake protobuf-compiler -y && \
     apt-get clean
 
 RUN apt-get install acl -y && \
@@ -27,12 +28,22 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
-COPY . /home
+FROM python:3.10-buster
 
-RUN ./install_shapepipe --develop --vos
+RUN pip install poetry==1.7.0
 
-# Create entrypoint script for desktop
-RUN mkdir /skaha
-COPY xterm_start.sh /skaha/init.sh
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
 
-ENTRYPOINT [ "/skaha/init.sh" ]
+COPY . /
+
+COPY pyproject.toml poetry.lock README.rst ./
+RUN touch README.md
+
+RUN poetry install --without dev --no-root && rm -rf $POETRY_CACHE_DIR
+
+COPY shapepipe ./
+
+RUN poetry install --without dev

From 3166e3c39a9ff8bae45c7cfdb2b36188a5bcc0b5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 8 Nov 2023 13:14:08 +0100
Subject: [PATCH 006/133] Added cent-os to CI tests

---
 .github/workflows/ci-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
index cc4ae4e93..ba9fe6cce 100644
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-tests.yml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, centos-latest]
         python-version: [3.9]
 
     steps:

From 04a5315e66800f85f4c874473877661630f40d4b Mon Sep 17 00:00:00 2001
From: Samuel Farrens <samuel.farrens@gmail.com>
Date: Thu, 7 Sep 2023 17:42:12 +0200
Subject: [PATCH 007/133] fixed bug in install_shapepipe

---
 install_shapepipe | 37 ++++++++++++++++++++++---------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/install_shapepipe b/install_shapepipe
index c370d91d0..56fa9be00 100755
--- a/install_shapepipe
+++ b/install_shapepipe
@@ -200,26 +200,28 @@ check_conda() {
     echo -ne "${RED}ERROR: Found Conda version $CONDA_VERSION but require 4.0.0 or greater.${NC}\n"
     exit 1
   fi
-  # Check Conda minor version
-  if [ "$CONDA_VERSION_MINOR" -gt 4 ]
+  # Look for conda.sh file
+  if [ -f "$CONDA_PREFIX_1$CONDA_SH" ]
   then
-    if [ -f "$CONDA_PREFIX_1$CONDA_SH" ]
-    then
-      source "$CONDA_PREFIX_1$CONDA_SH"
-    elif [ -f "$CONDA_PREFIX$CONDA_SH" ]
-    then
-      source "$CONDA_PREFIX$CONDA_SH"
-    else
-      echo -ne "${RED}ERROR: Could not find $CONDA_SH in \$CONDA_PREFIX.${NC}\n"
-      echo -ne "${RED}Activate the base/root Conda environment and try again.${NC}\n"
-      exit 1
-    fi
-    CONDA_ACTIVATE="conda activate"
-    CONDA_DEACTIVATE="conda deactivate"
+    source "$CONDA_PREFIX_1$CONDA_SH"
+  elif [ -f "$CONDA_PREFIX$CONDA_SH" ]
+  then
+    source "$CONDA_PREFIX$CONDA_SH"
   else
+    echo -ne "${RED}ERROR: Could not find $CONDA_SH in \$CONDA_PREFIX.${NC}\n"
+    echo -ne "${RED}Activate the base/root Conda environment and try again.${NC}\n"
+    exit 1
+  fi
+  # Set environment activation command
+  if [ "$USE_CONDA_SOURCE" == TRUE ]
+  then
     CONDA_ACTIVATE="source activate"
     CONDA_DEACTIVATE="source deactivate"
+  else
+    CONDA_ACTIVATE="conda activate"
+    CONDA_DEACTIVATE="conda deactivate"
   fi
+
 }
 
 # Check if a binary executable is already installed in the conda environment
@@ -464,6 +466,7 @@ Options:\n
 \t--pipe-dir\t set the path to the ShapePipe base directory (default is \$PWD)\n
 \t--build-dir\t set the path to the ShapePipe build (default is \$PWD/build)\n
 \t--env-name\t set the name of the conda environment (default is shapepipe)\n
+\t--use-conda-source\t use conda source instead of the default conda activate to activate the shapepipe environment\n
 \t--develop\t option to install additional development packages\n
 \t--vos\t\t option to install virtual observatory software\n
 \t--no-env\t do not build Conda environment\n
@@ -525,6 +528,10 @@ case $i in
     ENV_NAME="${i#*=}"
     shift
     ;;
+    --use-conda-source)
+    USE_CONDA_SOURCE=TRUE
+    shift
+    ;;
     --develop)
     DEVELOP=TRUE
     shift

From 757ef7fad45be8e4315462c66a6642ba70e01b2c Mon Sep 17 00:00:00 2001
From: sfarrens <samuel.farrens@gmail.com>
Date: Wed, 8 Nov 2023 17:34:20 +0100
Subject: [PATCH 008/133] Added development environment.

---
 .github/workflows/ci-dev.yml                  | 50 +++++++++++++++++++
 .../{ci-tests.yml => ci-release.yml}          |  1 -
 environment-dev.yml                           | 37 ++++++++++++++
 environment.yml                               |  6 +++
 install_shapepipe                             | 34 +++++++++++--
 5 files changed, 123 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/ci-dev.yml
 rename .github/workflows/{ci-tests.yml => ci-release.yml} (98%)
 create mode 100644 environment-dev.yml

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
new file mode 100644
index 000000000..a260f6c9d
--- /dev/null
+++ b/.github/workflows/ci-dev.yml
@@ -0,0 +1,50 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+     - develop
+
+jobs:
+
+  test-full:
+    name: Full Test Suite
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python-version: [3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Linux dependencies
+        if: runner.os == 'Linux'
+        run: sudo apt-get install libopenblas-dev
+
+      - name: Install macOS Dependencies
+        shell: bash -l {0}
+        if: runner.os == 'macOS'
+        run: |
+          brew install libomp
+
+      - name: Set up conda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: ${{ matrix.python-version }}
+          auto-activate-base: true
+
+      - name: Install package
+        shell: bash -l {0}
+        run: |
+          ./install_shapepipe --env-dev --develop
+
+      - name: Run tests
+        shell: bash -l {0}
+        run: |
+          conda activate shapepipe
+          python setup.py test
+          shapepipe_run -c example/config.ini
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-release.yml
similarity index 98%
rename from .github/workflows/ci-tests.yml
rename to .github/workflows/ci-release.yml
index cc4ae4e93..ef1f6a013 100644
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-release.yml
@@ -5,7 +5,6 @@ on:
     branches:
      - main
      - master
-     - develop
 
 jobs:
 
diff --git a/environment-dev.yml b/environment-dev.yml
new file mode 100644
index 000000000..b5325d02c
--- /dev/null
+++ b/environment-dev.yml
@@ -0,0 +1,37 @@
+# ShapePipe Development Environment
+# ---------------------------------
+# This environment should only be used for development and testing of  
+# ShapePipe (i.e. nothing that could be published). Packages need only
+# specify the minimum compatible versions. This environment can updated 
+# as needed.
+name: shapepipe-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+  - pip>=21.2.4
+  - astropy>=5.0
+  - autoconf>=2.71
+  - automake>=1.16
+  - cmake>=3.27
+  - galsim>=2.5
+  - joblib>=1.3
+  - libtool>=2.4
+  - matplotlib>=3.8
+  - numba>=0.57
+  - pandas>=2.1
+  - pyqt5-sip>=12.12
+  - pyqtgraph>=0.13
+  - reproject>=0.12
+  - sqlitedict>=2.1
+  - termcolor>=2.3
+  - tqdm>=4.66
+  - treecorr>=4.3
+  - pip:
+    - mccd==1.2.3
+    - modopt==1.6.0
+    - sip_tpv==1.1
+    - sf_tools==2.0.4
+    - git+https://github.com/CEA-COSMIC/pysap@develop
+    - git+https://github.com/aguinot/ngmix@stable_version
+    - git+https://github.com/tobias-liaudat/Stile@v0.1
diff --git a/environment.yml b/environment.yml
index 9279402ef..ec8fae7a1 100644
--- a/environment.yml
+++ b/environment.yml
@@ -1,3 +1,9 @@
+# ShapePipe Production Environment
+# --------------------------------
+# This environment should be used for any production runs of ShapePipe 
+# (i.e. anything that could be published). All packages are pinned to
+# specific versions to maximise the reproducibility of the results.
+# This environment should be updated for each release of ShapePipe.
 name: shapepipe
 channels:
   - conda-forge
diff --git a/install_shapepipe b/install_shapepipe
index 56fa9be00..4c3f512c6 100755
--- a/install_shapepipe
+++ b/install_shapepipe
@@ -81,6 +81,8 @@ NC='\033[0m'
 PIPE_DIR=$PWD
 BUILD_DIR=$PIPE_DIR/build
 ENV_NAME='shapepipe'
+ENV_DEV=FALSE
+USE_CONDA_SOURCE=FALSE
 BUILD_ENV=TRUE
 DEVELOP=FALSE
 VOS=FALSE
@@ -258,6 +260,14 @@ check_binary2() {
   echo ""
 }
 
+# Check the environment name
+check_env_name() {
+if [ "$ENV_DEV" == TRUE ]
+then
+  ENV_NAME='shapepipe-dev'
+fi
+}
+
 # Check if a python site-package is installed correctly in the conda environment
 check_site_pkg() {
   if compgen -G "$CONDA_PREFIX/lib/python3.*/site-packages/$1*" > /dev/null
@@ -343,7 +353,9 @@ package_report() {
   echo 'Package Report'
   echo ''
   packages
-  python -m piplist
+  echo 'Python Package Check List:'
+  echo ''
+  site-packages
   echo ''
   exit_code
 }
@@ -410,6 +422,7 @@ astromatic_setup() {
 
 # Function to uninstall ShapePipe and dependencies
 uninstall() {
+  check_env_name
   read -p "Uninstall $ENV_NAME environment [y/N]: " response
   response=${response:-n}
   if [ $response == "y" ]
@@ -465,8 +478,9 @@ Options:\n
 \t--uninstall\t uninstall ShapePipe\n
 \t--pipe-dir\t set the path to the ShapePipe base directory (default is \$PWD)\n
 \t--build-dir\t set the path to the ShapePipe build (default is \$PWD/build)\n
-\t--env-name\t set the name of the conda environment (default is shapepipe)\n
-\t--use-conda-source\t use conda source instead of the default conda activate to activate the shapepipe environment\n
+\t--env-name\t set the name of the Conda environment (default is shapepipe)\n
+\t--env-dev\t use the developers Conda environment instead of the standard production environment\n
+\t--use-conda-source\t use 'conda source' instead of the default 'conda activate' to activate the shapepipe environment\n
 \t--develop\t option to install additional development packages\n
 \t--vos\t\t option to install virtual observatory software\n
 \t--no-env\t do not build Conda environment\n
@@ -528,6 +542,10 @@ case $i in
     ENV_NAME="${i#*=}"
     shift
     ;;
+    --env-dev)
+    ENV_DEV=TRUE
+    shift
+    ;;
     --use-conda-source)
     USE_CONDA_SOURCE=TRUE
     shift
@@ -620,6 +638,9 @@ start
 # Check if conda is installed
 check_conda
 
+# Check the Conda environment name
+check_env_name
+
 # Find the operating system
 case "$OSTYPE" in
   darwin*)
@@ -652,7 +673,12 @@ setup
 if [ "$BUILD_ENV" == TRUE ]
 then
   report_progress 'ShapePipe Environment'
-  conda env create -f environment.yml -n $ENV_NAME
+  if [ "$ENV_DEV" == TRUE ]
+  then
+    conda env create -f environment-dev.yml -n $ENV_NAME
+  else
+    conda env create -f environment.yml -n $ENV_NAME
+  fi
 fi
 
 # Activate conda environment

From 373722e030ddf83238e9f444880769aeb9f6f63b Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 9 Nov 2023 11:44:05 +0100
Subject: [PATCH 009/133] Added pyproject

---
 pyproject.toml | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 pyproject.toml

diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..bc252da53
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,44 @@
+[tool.poetry]
+name = "shapepipe"
+version = "0.1.0"
+description = "Weak-lensing shape measurement"
+authors = [
+	"Sam Farrens",
+	"Axel Guinot",
+	"Martin Kilbinger <martin.kilbinger@cea.fr>",
+]
+readme = "README.rst"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<3.13"
+cmake = "^3.27.2"
+treecorr = "^4.3.3"
+matplotlib = "^3.7.2"
+jupyter = "^1.0.0"
+pandas = "^2.1.0"
+
+cs-util = "^0.0.5"
+numpy = "^1.19.1"
+autoconf = "^2023.10.23.3"
+galsim = "^2.5.1"
+libtool = "^1.1.0"
+numba = "^0.58.1"
+modopt = "^1.6.1"
+
+#python-pysap = { git = "https://github.com/martinkilbinger/pysap.git", branch = "fix_versions" }
+ngmix = { git = "https://github.com/aguinot/ngmix", branch = "stable_version" }
+stile = { git = "https://github.com/tobias-liaudat/Stile", tag = "v0.1" }
+pywavelets = "^1.4.1"
+
+[tool.poetry.dev-dependencies]
+pytest = "^6.2.5"
+pydocstyle = "^6.3.0"
+pytest-cov = "^4.1.0"
+black = "^23.7.0"
+
+[tool.black]
+line-length = 80
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

From b025b6aad7eebbfab31511ec5d372837acbf4df9 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 10 Nov 2023 15:31:02 +0000
Subject: [PATCH 010/133] canfar pyproject update

---
 pyproject.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index bc252da53..e718d3aeb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,6 +29,8 @@ modopt = "^1.6.1"
 ngmix = { git = "https://github.com/aguinot/ngmix", branch = "stable_version" }
 stile = { git = "https://github.com/tobias-liaudat/Stile", tag = "v0.1" }
 pywavelets = "^1.4.1"
+joblib = "^1.3.2"
+sip-tpv = "^1.1"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"

From ba135f27c7d05d1cfae0ca04233b97680ea3aec7 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 10 Nov 2023 15:32:37 +0000
Subject: [PATCH 011/133] cfis vos configs: missing keywords, DR5 added

---
 example/cfis/config_GitFeGie_vos.ini          | 8 +++++++-
 example/cfis/config_get_tiles_vos_headers.ini | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/example/cfis/config_GitFeGie_vos.ini b/example/cfis/config_GitFeGie_vos.ini
index fd3207bb3..eb06d372b 100644
--- a/example/cfis/config_GitFeGie_vos.ini
+++ b/example/cfis/config_GitFeGie_vos.ini
@@ -66,7 +66,7 @@ NUMBERING_SCHEME =
 
 # Input path where original images are stored. Can be local path or vos url.
 # Single string or list of strings
-INPUT_PATH = vos:cfis/tiles_DR3, vos:cfis/tiles_DR3
+INPUT_PATH = vos:cfis/tiles_DR5, vos:cfis/tiles_DR5
 
 # Input file pattern including tile number as dummy template
 INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight
@@ -98,6 +98,12 @@ FILE_EXT = .fits
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
 
+# Column number of exposure name in FITS header
+COLNUM = 3
+
+# Prefix to remove from exposure name
+EXP_PREFIX = p
+
 
 # Get exposures
 [GET_IMAGES_RUNNER_RUN_2]
diff --git a/example/cfis/config_get_tiles_vos_headers.ini b/example/cfis/config_get_tiles_vos_headers.ini
index e28347ebf..4b79a2493 100644
--- a/example/cfis/config_get_tiles_vos_headers.ini
+++ b/example/cfis/config_get_tiles_vos_headers.ini
@@ -69,7 +69,7 @@ NUMBERING_SCHEME =
 
 # Input path where original images are stored. Can be local path or vos url.
 # Single string or list of strings
-INPUT_PATH = vos:cfis/tiles_DR3
+INPUT_PATH = vos:cfis/tiles_DR5
 
 # Input file pattern including tile number as dummy template
 INPUT_FILE_PATTERN = CFIS.000.000.r

From 07cafb4651db7dee9d372b121c984d3ba27060cb Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 15 Nov 2023 09:55:57 +0100
Subject: [PATCH 012/133] Testing dockerfile with only conda env installed

---
 Dockerfile                                 | 35 ++--------------
 environment.yml                            | 48 ++++++++++------------
 shapepipe/modules/python_example_runner.py |  2 -
 3 files changed, 25 insertions(+), 60 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1f966302b..8cf92b10c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,38 +1,9 @@
 FROM continuumio/miniconda3
 
 LABEL Description="ShapePipe Docker Image"
-WORKDIR /home
 ENV SHELL /bin/bash
 
-ARG CC=gcc-9
-ARG CXX=g++-9
+COPY ./environment.yml ./
+RUN touch ./README.md
 
-RUN apt-get update --allow-releaseinfo-change && \
-    apt-get update && \
-    apt-get upgrade -y && \
-    apt-get install make -y && \
-    apt-get install gcc-9 g++-9 -y && \
-    apt-get install locales -y && \
-    apt install libgl1-mesa-glx -y && \
-    apt-get install xterm -y && \
-    apt-get clean
-
-RUN apt-get install acl -y && \
-    apt-get install sssd -y
-ADD nsswitch.conf /etc/
-
-RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
-    locale-gen
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
-
-COPY . /home
-
-RUN ./install_shapepipe --develop --vos
-
-# Create entrypoint script for desktop
-RUN mkdir /skaha
-COPY xterm_start.sh /skaha/init.sh
-
-ENTRYPOINT [ "/skaha/init.sh" ]
+RUN conda env create --file environment.yml
diff --git a/environment.yml b/environment.yml
index d6e5957b6..0335a4ac7 100644
--- a/environment.yml
+++ b/environment.yml
@@ -1,34 +1,30 @@
-name: shapepipe
+name: shapepipe2
 channels:
   - conda-forge
 dependencies:
   - python=3.9
   - pip>=21.2.4
-  - numpy==1.21.6
-  - astropy==5.0
-  - automake==1.16.2
-  - autoconf==2.69
-  - cmake==3.19.6
-  - galsim==2.2.5
-  - joblib==1.1.0
-  - libtool==2.4.6
-  - matplotlib==3.5.1
-  - numba==0.54.1
-  - pandas==1.4.1
+  - numpy
+  - astropy
+  - automake
+  - autoconf
+  - cmake
+  - galsim
+  - joblib
+  - libtool
+  - matplotlib
+  - numba
   - pip:
-    - jupyter
-    - cs_util==0.0.5
-    - mccd==1.2.3
-    - modopt==1.6.0
-    - PyQt5==5.15.6
-    - pyqtgraph==0.12.4
-    - python-pysap==0.0.6
-    - reproject==0.8
-    - sip_tpv==1.1
-    - sf_tools==2.0.4
-    - sqlitedict==2.0.0
-    - termcolor==1.1.0
-    - tqdm==4.63.0
-    - treecorr==4.2.6
+    - cs_util
+    - modopt
+    - PyQt5
+    - pyqtgraph
+    - reproject
+    - sip_tpv
+    - sf_tools
+    - sqlitedict
+    - termcolor
+    - tqdm
+    - treecorr
     - git+https://github.com/aguinot/ngmix@stable_version
     - git+https://github.com/tobias-liaudat/Stile@v0.1
diff --git a/shapepipe/modules/python_example_runner.py b/shapepipe/modules/python_example_runner.py
index d468c10e0..fc5565ecc 100644
--- a/shapepipe/modules/python_example_runner.py
+++ b/shapepipe/modules/python_example_runner.py
@@ -19,10 +19,8 @@
         'astropy',
         'galsim',
         'joblib',
-        'mccd',
         'ngmix',
         'pandas',
-        'pysap',
         'scipy',
         'sf_tools',
         'sip_tpv',

From 92a999ca5f647637de051c961ef51663a41c98d6 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 15 Nov 2023 09:59:38 +0100
Subject: [PATCH 013/133] Fixing numpy and astropy versions

---
 environment.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/environment.yml b/environment.yml
index 0335a4ac7..bd0c28bce 100644
--- a/environment.yml
+++ b/environment.yml
@@ -4,8 +4,8 @@ channels:
 dependencies:
   - python=3.9
   - pip>=21.2.4
-  - numpy
-  - astropy
+  - numpy==1.21.6
+  - astropy==5.1
   - automake
   - autoconf
   - cmake

From 2b6600de2512b4a91f3f8b0a635deb73c55cc9e0 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 17 Nov 2023 10:39:53 +0000
Subject: [PATCH 014/133] installation on canfar working: removed most versions
 from yml file; added init_canfar.sh script

---
 environment.yml                  | 48 +++++++++++++++-----------------
 example/cfis/config_exp_SpMh.ini |  2 ++
 pyproject.toml                   |  3 ++
 scripts/sh/init_canfar.sh        | 13 +++++++++
 4 files changed, 41 insertions(+), 25 deletions(-)
 create mode 100755 scripts/sh/init_canfar.sh

diff --git a/environment.yml b/environment.yml
index d6e5957b6..489080262 100644
--- a/environment.yml
+++ b/environment.yml
@@ -3,32 +3,30 @@ channels:
   - conda-forge
 dependencies:
   - python=3.9
-  - pip>=21.2.4
-  - numpy==1.21.6
-  - astropy==5.0
-  - automake==1.16.2
-  - autoconf==2.69
-  - cmake==3.19.6
-  - galsim==2.2.5
-  - joblib==1.1.0
-  - libtool==2.4.6
-  - matplotlib==3.5.1
-  - numba==0.54.1
-  - pandas==1.4.1
+  - pip
+  - numpy
+  - astropy
+  - automake
+  - autoconf
+  - cmake
+  - galsim
+  - joblib
+  - libtool
+  - matplotlib
+  - numba
+  - pandas
   - pip:
     - jupyter
-    - cs_util==0.0.5
-    - mccd==1.2.3
-    - modopt==1.6.0
-    - PyQt5==5.15.6
-    - pyqtgraph==0.12.4
-    - python-pysap==0.0.6
-    - reproject==0.8
-    - sip_tpv==1.1
-    - sf_tools==2.0.4
-    - sqlitedict==2.0.0
-    - termcolor==1.1.0
-    - tqdm==4.63.0
-    - treecorr==4.2.6
+    - cs_util
+    - modopt
+    - PyQt5
+    - pyqtgraph
+    - reproject
+    - sip_tpv
+    - sf_tools
+    - sqlitedict
+    - termcolor
+    - tqdm
+    - treecorr
     - git+https://github.com/aguinot/ngmix@stable_version
     - git+https://github.com/tobias-liaudat/Stile@v0.1
diff --git a/example/cfis/config_exp_SpMh.ini b/example/cfis/config_exp_SpMh.ini
index 76f87ddd7..ce37729f3 100644
--- a/example/cfis/config_exp_SpMh.ini
+++ b/example/cfis/config_exp_SpMh.ini
@@ -57,6 +57,8 @@ TIMEOUT = 96:00:00
 
 INPUT_DIR = last:get_images_runner_run_2
 
+FILE_PATTERN = image, weight, flag
+
 # Matches compressed single-exposure files
 FILE_EXT = .fitsfz, .fitsfz, .fitsfz
 
diff --git a/pyproject.toml b/pyproject.toml
index e718d3aeb..59d487316 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,6 +24,8 @@ galsim = "^2.5.1"
 libtool = "^1.1.0"
 numba = "^0.58.1"
 modopt = "^1.6.1"
+# MKDEBUG astropy>5.2: RuntimeError: Invalid TFORM2: 1PE(625) when reading (compressed) tiles weight
+astropy = ">=5.0, <5.2"
 
 #python-pysap = { git = "https://github.com/martinkilbinger/pysap.git", branch = "fix_versions" }
 ngmix = { git = "https://github.com/aguinot/ngmix", branch = "stable_version" }
@@ -31,6 +33,7 @@ stile = { git = "https://github.com/tobias-liaudat/Stile", tag = "v0.1" }
 pywavelets = "^1.4.1"
 joblib = "^1.3.2"
 sip-tpv = "^1.1"
+sqlitedict = "^2.1.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"
diff --git a/scripts/sh/init_canfar.sh b/scripts/sh/init_canfar.sh
new file mode 100755
index 000000000..dd3346f94
--- /dev/null
+++ b/scripts/sh/init_canfar.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "start init canfar"
+
+echo init_canfar > ~/init_canfar.log
+date >> ~/init_canfar.log
+
+. /opt/conda/etc/profile.d/conda.sh
+
+conda activate shapepipe
+
+echo "end init canfar"
+

From b6e5c970db8d6b2e788fa3286b3255c9fcc3bde6 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 17 Nov 2023 11:47:15 +0100
Subject: [PATCH 015/133] Dockerfile for image 0:7, 0:8

---
 Dockerfile                       | 47 ++++++++++++++++++++++++++++++++
 environment.yml                  |  2 +-
 example/cfis/config_exp_SpMh.ini |  2 +-
 scripts/sh/curl_canfar.sh        | 37 +++++++++++++++++++++++++
 4 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100755 scripts/sh/curl_canfar.sh

diff --git a/Dockerfile b/Dockerfile
index 8cf92b10c..dfcef1762 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,54 @@ FROM continuumio/miniconda3
 LABEL Description="ShapePipe Docker Image"
 ENV SHELL /bin/bash
 
+ARG CC=gcc-9
+ARG CXX=g++-9
+
+# gcc < 10 is required to compile ww
+ENV CC=gcc-9
+ENV CXX=g++-9
+
+RUN apt-get update --allow-releaseinfo-change && \
+    apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install apt-utils -y && \
+    apt-get install make -y && \
+    apt-get install automake -y && \
+    apt-get install autoconf -y && \
+    apt-get install gcc-9 g++-9 -y && \
+    apt-get install locales -y && \
+    apt-get install libgl1-mesa-glx -y && \
+    apt-get install xterm -y && \
+    apt-get install cmake protobuf-compiler -y && \
+    apt-get install libtool libtool-bin libtool-doc -y && \
+    apt-get install libfftw3-bin libfftw3-dev -y && \
+    apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \
+    apt-get install vim -y && \
+    apt-get install locate -y && \
+    apt-get install sextractor -y && \
+    apt-get install psfex -y && \
+    apt-get clean
+
+RUN apt-get install acl -y && \
+    apt-get install sssd -y
+ADD nsswitch.conf /etc/
+
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
+    locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
 COPY ./environment.yml ./
+COPY install_shapepipe README.rst setup.py setup.cfg ./
 RUN touch ./README.md
 
+RUN conda update -n base -c defaults conda -c defaults
 RUN conda env create --file environment.yml
+
+COPY shapepipe ./shapepipe
+COPY scripts ./scripts
+
+RUN ls ./scripts/sh
+RUN ./scripts/sh/init_canfar.sh
+#RUN conda activate shapepipe
diff --git a/environment.yml b/environment.yml
index bd0c28bce..8bc1d736a 100644
--- a/environment.yml
+++ b/environment.yml
@@ -1,4 +1,4 @@
-name: shapepipe2
+name: shapepipe
 channels:
   - conda-forge
 dependencies:
diff --git a/example/cfis/config_exp_SpMh.ini b/example/cfis/config_exp_SpMh.ini
index 76f87ddd7..20ac38616 100644
--- a/example/cfis/config_exp_SpMh.ini
+++ b/example/cfis/config_exp_SpMh.ini
@@ -63,7 +63,7 @@ FILE_EXT = .fitsfz, .fitsfz, .fitsfz
 NUMBERING_SCHEME = -0000000
 
 # OUTPUT_SUFFIX, actually file name prefixes.
-# Expected keyword "flag" will lead to a behavior where the data are save as int.
+# Expected keyword "flag" will lead to a behavior where the data are saved as int.
 # The code also expects the image data to use the "image" suffix
 # (default value in the pipeline).
 OUTPUT_SUFFIX = image, weight, flag
diff --git a/scripts/sh/curl_canfar.sh b/scripts/sh/curl_canfar.sh
new file mode 100755
index 000000000..5e898a27d
--- /dev/null
+++ b/scripts/sh/curl_canfar.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/bash
+
+# Usage
+# cat tile_num.txt | xargs -n 1 -P 1 ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.7 shapepipe/scripts/sh/init_canfar.sh
+
+SSL=~/.ssl/cadcproxy.pem
+SESSION=https://ws-uv.canfar.net/skaha/v0/session
+IMAGE=images.canfar.net/unions/shapepipe
+NAME=shapepipe
+
+version=$1
+cmd=$2
+arg=$3
+
+arg_an=`echo "$arg" | tr '_' 'X' | tr '.' 'X'`
+arg_an=X
+
+echo "Start headless container"
+echo "========================"
+ID=`curl -E $SSL $SESSION -d "image=$IMAGE:$version" -d "name=${NAME}${arg_an}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
+#curl -E $SSL $SESSION -d \"name=$NAME\" -d \"image=images.canfar.net/unions/shapepipe\:0.7\" -d \"cmd=$2\" --data-urlencode \"args=$3\"
+echo $ID
+
+echo
+echo "Events (incl. errors)"
+echo "====================="
+cmd="curl -E $SSL $SESSION/$ID?view=events"
+echo $cmd
+$cmd
+
+echo
+echo "Logs (incl. stdout)"
+echo "==================="
+cmd="curl -E $SSL $SESSION/$ID?view=logs"
+echo $cmd
+$cmd
+

From bc66565c8e6ece3c20dee4fe46879fd385725619 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 19 Nov 2023 19:04:19 +0100
Subject: [PATCH 016/133] Installation of sextractor and psfex with conda

---
 Dockerfile                |   3 -
 environment.yml           |   4 +-
 install_shapepipe         | 125 --------------------------------------
 scripts/sh/init_canfar.sh |  13 ++++
 4 files changed, 16 insertions(+), 129 deletions(-)
 create mode 100755 scripts/sh/init_canfar.sh

diff --git a/Dockerfile b/Dockerfile
index dfcef1762..98e6c35fc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,8 +27,6 @@ RUN apt-get update --allow-releaseinfo-change && \
     apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \
     apt-get install vim -y && \
     apt-get install locate -y && \
-    apt-get install sextractor -y && \
-    apt-get install psfex -y && \
     apt-get clean
 
 RUN apt-get install acl -y && \
@@ -51,6 +49,5 @@ RUN conda env create --file environment.yml
 COPY shapepipe ./shapepipe
 COPY scripts ./scripts
 
-RUN ls ./scripts/sh
 RUN ./scripts/sh/init_canfar.sh
 #RUN conda activate shapepipe
diff --git a/environment.yml b/environment.yml
index 8bc1d736a..759813a80 100644
--- a/environment.yml
+++ b/environment.yml
@@ -4,7 +4,8 @@ channels:
 dependencies:
   - python=3.9
   - pip>=21.2.4
-  - numpy==1.21.6
+  - astromatic-psfex==3.21.1
+  - astromatic-source-extractor==2.25.0
   - astropy==5.1
   - automake
   - autoconf
@@ -14,6 +15,7 @@ dependencies:
   - libtool
   - matplotlib
   - numba
+  - numpy==1.21.6
   - pip:
     - cs_util
     - modopt
diff --git a/install_shapepipe b/install_shapepipe
index c370d91d0..96cc89395 100755
--- a/install_shapepipe
+++ b/install_shapepipe
@@ -25,20 +25,6 @@ libpng_ver="1.6.37"
 mpi4py_ver="3.1.3"
 openblas_ver="0.3.18"
 
-# SExtractor Package
-sex_url="https://github.com/astromatic/sextractor/archive/2.25.0.tar.gz"
-sex_tar="2.25.0.tar.gz"
-sex_dir="sextractor-2.25.0"
-sex_bin="sex"
-sex_version=TRUE
-
-# PSFEx Package
-psfex_url="https://github.com/astromatic/psfex/archive/3.21.1.tar.gz"
-psfex_tar="3.21.1.tar.gz"
-psfex_dir="psfex-3.21.1"
-psfex_bin="psfex"
-psfex_version=TRUE
-
 # WeightWatcher Package
 ww_url="http://snapshot.debian.org/archive/debian/20171211T160522Z/pool/main/w/weightwatcher/weightwatcher_1.12.orig.tar.gz"
 ww_tar="weightwatcher_1.12.orig.tar.gz"
@@ -84,8 +70,6 @@ ENV_NAME='shapepipe'
 BUILD_ENV=TRUE
 DEVELOP=FALSE
 VOS=FALSE
-INSTALL_SEX=TRUE
-INSTALL_PSFEX=TRUE
 INSTALL_WW=TRUE
 INSTALL_CDSCLIENT=TRUE
 ATLAS_SEARCH=FALSE
@@ -105,66 +89,6 @@ download_package() {
   rm $2
 }
 
-# Function to build SExtractor with OpenBLAS
-build_sex_blas() {
-  astromatic_setup
-  cd $BUILD_DIR/$1
-  ./autogen.sh
-  ./configure --prefix=$CONDA_PREFIX --enable-openblas --with-fftw-libdir=$FFTW_LIB \
-  --with-fftw-incdir=$FFTW_INC --with-openblas-libdir=$BLAS_LIB \
-  --with-openblas-incdir=$BLAS_INC --quiet
-  make -j --quiet
-  make install
-}
-
-# Function to build SExtractor with ATLAS
-build_sex_atlas() {
-  astromatic_setup
-  cd $BUILD_DIR/$1
-  ./autogen.sh
-  if [ "$ATLAS_SEARCH" == TRUE ]
-  then
-    ./configure --prefix=$CONDA_PREFIX --with-fftw-libdir=$FFTW_LIB \
-    --with-fftw-incdir=$FFTW_INC --quiet
-  else
-    ./configure --prefix=$CONDA_PREFIX --with-fftw-libdir=$FFTW_LIB \
-    --with-fftw-incdir=$FFTW_INC --with-atlas-libdir=$ATLAS_LIB \
-    --with-atlas-incdir=$ATLAS_INC --quiet
-  fi
-  make -j --quiet
-  make install
-}
-
-# Function to build PSFEx with OpenBLAS
-build_psfex_blas() {
-  astromatic_setup
-  cd $BUILD_DIR/$1
-  ./autogen.sh
-  ./configure --prefix=$CONDA_PREFIX --enable-openblas --enable-plplot=no \
-  --with-fftw-libdir=$FFTW_LIB --with-fftw-incdir=$FFTW_INC \
-  --with-openblas-libdir=$BLAS_LIB --with-openblas-incdir=$BLAS_INC --quiet
-  make -j --quiet
-  make install
-}
-
-# Function to build PSFEx with ATLAS
-build_psfex_atlas() {
-  astromatic_setup
-  cd $BUILD_DIR/$1
-  ./autogen.sh
-  if [ "$ATLAS_SEARCH" == TRUE ]
-  then
-    ./configure --prefix=$CONDA_PREFIX --enable-plplot=no \
-    --with-fftw-libdir=$FFTW_LIB --with-fftw-incdir=$FFTW_INC --quiet
-  else
-    ./configure --prefix=$CONDA_PREFIX --enable-plplot=no \
-    --with-fftw-libdir=$FFTW_LIB --with-fftw-incdir=$FFTW_INC \
-    --with-atlas-libdir=$ATLAS_LIB --with-atlas-incdir=$ATLAS_INC --quiet
-  fi
-  make -j --quiet
-  make install
-}
-
 # Function to build a standard package
 build_standard() {
   cd $BUILD_DIR/$1
@@ -290,8 +214,6 @@ start() {
 
 # Function to report package versions
 packages() {
-  check_binary2 $sex_bin "SExtractor" $sex_version $INSTALL_SEX
-  check_binary2 $psfex_bin "PSFEx" $psfex_version $INSTALL_PSFEX
   check_binary2 $ww_bin "WeightWatcher" $ww_version $INSTALL_WW
   check_binary2 $cdsclient_bin "CDSclient" $cdsclient_version $INSTALL_CDSCLIENT
   check_binary2 $mpi_bin "MPI" $mpi_version $USE_MPI $MPI_ROOT
@@ -332,8 +254,6 @@ package_report() {
   then
     activate_shapepipe
   fi
-  INSTALL_SEX=FALSE
-  INSTALL_PSFEX=FALSE
   INSTALL_WW=FALSE
   INSTALL_CDSCLIENT=FALSE
   USE_MPI=FALSE
@@ -381,8 +301,6 @@ setup() {
   check_status 'Operating System:' $SYSOS
   check_status 'Conda Version:' $CONDA_VERSION
   check_status 'Build Conda Environment:' $BUILD_ENV $ENV_NAME
-  check_status 'Install SExtractor:' $INSTALL_SEX
-  check_status 'Install PSFEx:' $INSTALL_PSFEX
   check_status 'Install WeightWatcher:' $INSTALL_WW
   check_status 'Install CDSclient:' $INSTALL_CDSCLIENT
   check_status 'Use MPI:' $USE_MPI $MPI_ROOT
@@ -467,8 +385,6 @@ Options:\n
 \t--develop\t option to install additional development packages\n
 \t--vos\t\t option to install virtual observatory software\n
 \t--no-env\t do not build Conda environment\n
-\t--no-sex\t do not build SExtractor\n
-\t--no-psfex\t do not build PSFEx\n
 \t--no-ww\t\t do not build WeightWatcher\n
 \t--no-cds\t do not build CDSclient\n
 \t--no-exe\t do not build any system executables\n\n
@@ -537,14 +453,6 @@ case $i in
     BUILD_ENV=FALSE
     shift
     ;;
-    --no-sex)
-    INSTALL_SEX=FALSE
-    shift
-    ;;
-    --no-psfex)
-    INSTALL_PSFEX=FALSE
-    shift
-    ;;
     --no-ww)
     INSTALL_WW=FALSE
     shift
@@ -590,8 +498,6 @@ case $i in
     shift
     ;;
     --no-exe)
-    INSTALL_SEX=FALSE
-    INSTALL_PSFEX=FALSE
     INSTALL_WW=FALSE
     INSTALL_CDSCLIENT=FALSE
     shift
@@ -716,37 +622,6 @@ fi
 # BUILD EXTERNAL EXECUTABLES
 ##############################################################################
 
-# Build SExtractor
-if [ "$INSTALL_SEX" == TRUE ] && check_binary $sex_bin
-then
-  report_progress 'SExtractor'
-  conda install -n $ENV_NAME -c conda-forge "fftw==${fftw_ver}" -y
-  download_package $sex_url $sex_tar
-  if [ "$use_atlas" == TRUE ]
-  then
-    build_sex_atlas $sex_dir
-  else
-    conda install -n $ENV_NAME -c conda-forge "openblas==${openblas_ver}" -y
-    build_sex_blas $sex_dir
-  fi
-fi
-
-# Build PSFEx
-if [ "$INSTALL_PSFEX" == TRUE ] && check_binary $psfex_bin
-then
-  report_progress 'PSFEx'
-  conda install "libpng==${libpng_ver}" -y
-  conda install -n $ENV_NAME -c conda-forge "fftw==${fftw_ver}" -y
-  download_package $psfex_url $psfex_tar
-  if [ "$use_atlas" == TRUE ]
-  then
-    build_psfex_atlas $psfex_dir
-  else
-    conda install -n $ENV_NAME -c conda-forge "openblas==${openblas_ver}" -y
-    build_psfex_blas $psfex_dir
-  fi
-fi
-
 # Build WeightWatcher
 if [ "$INSTALL_WW" == TRUE ] && check_binary $ww_bin
 then
diff --git a/scripts/sh/init_canfar.sh b/scripts/sh/init_canfar.sh
new file mode 100755
index 000000000..dd3346f94
--- /dev/null
+++ b/scripts/sh/init_canfar.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "start init canfar"
+
+echo init_canfar > ~/init_canfar.log
+date >> ~/init_canfar.log
+
+. /opt/conda/etc/profile.d/conda.sh
+
+conda activate shapepipe
+
+echo "end init canfar"
+

From 0b3aac6277bedfbacd926e4f82004faf67c66615 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 19 Nov 2023 19:05:02 +0100
Subject: [PATCH 017/133] script to call curl with canfar contained updated,
 adding NCORE as argument

---
 scripts/sh/curl_canfar.sh | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/scripts/sh/curl_canfar.sh b/scripts/sh/curl_canfar.sh
index 5e898a27d..43fb6cd08 100755
--- a/scripts/sh/curl_canfar.sh
+++ b/scripts/sh/curl_canfar.sh
@@ -1,24 +1,30 @@
 #!/usr/bin/bash
 
 # Usage
-# cat tile_num.txt | xargs -n 1 -P 1 ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.7 shapepipe/scripts/sh/init_canfar.sh
+~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.7 shapepipe/scripts/sh/init_canfar.sh 000.000
 
 SSL=~/.ssl/cadcproxy.pem
-SESSION=https://ws-uv.canfar.net/skaha/v0/session
+NCORE=16
+SESSION=https://ws-uv.canfar.net/skaha/v0/session?ram=16&cores=$NCORE
 IMAGE=images.canfar.net/unions/shapepipe
 NAME=shapepipe
 
+# version of image on canfar, e.g. 0:7, 0:8
 version=$1
+
+# command on canfar, e.g. shapepipe/scripts/sh/init_run_canfar.sh
 cmd=$2
-arg=$3
 
-arg_an=`echo "$arg" | tr '_' 'X' | tr '.' 'X'`
+# command line argument, e.g. 181.308
+arg="$3 $NCORE"
+
+#arg_an=`echo "$arg" | tr '_' 'X' | tr '.' 'X'`
 arg_an=X
 
+echo
 echo "Start headless container"
 echo "========================"
 ID=`curl -E $SSL $SESSION -d "image=$IMAGE:$version" -d "name=${NAME}${arg_an}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
-#curl -E $SSL $SESSION -d \"name=$NAME\" -d \"image=images.canfar.net/unions/shapepipe\:0.7\" -d \"cmd=$2\" --data-urlencode \"args=$3\"
 echo $ID
 
 echo

From 69317cb210e5d3b70fbd0cfe64c2d53984b4b98f Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 19 Nov 2023 18:17:38 +0000
Subject: [PATCH 018/133] added init scripts for canfar; job_sp added n_smp for
 further jobs

---
 pyproject.toml                | 49 -----------------------------------
 scripts/sh/init_canfar.sh     |  4 +--
 scripts/sh/init_run_canfar.sh | 21 +++++++++++++++
 scripts/sh/job_sp.bash        |  2 +-
 4 files changed, 24 insertions(+), 52 deletions(-)
 delete mode 100644 pyproject.toml
 create mode 100755 scripts/sh/init_run_canfar.sh

diff --git a/pyproject.toml b/pyproject.toml
deleted file mode 100644
index 59d487316..000000000
--- a/pyproject.toml
+++ /dev/null
@@ -1,49 +0,0 @@
-[tool.poetry]
-name = "shapepipe"
-version = "0.1.0"
-description = "Weak-lensing shape measurement"
-authors = [
-	"Sam Farrens",
-	"Axel Guinot",
-	"Martin Kilbinger <martin.kilbinger@cea.fr>",
-]
-readme = "README.rst"
-
-[tool.poetry.dependencies]
-python = ">=3.9,<3.13"
-cmake = "^3.27.2"
-treecorr = "^4.3.3"
-matplotlib = "^3.7.2"
-jupyter = "^1.0.0"
-pandas = "^2.1.0"
-
-cs-util = "^0.0.5"
-numpy = "^1.19.1"
-autoconf = "^2023.10.23.3"
-galsim = "^2.5.1"
-libtool = "^1.1.0"
-numba = "^0.58.1"
-modopt = "^1.6.1"
-# MKDEBUG astropy>5.2: RuntimeError: Invalid TFORM2: 1PE(625) when reading (compressed) tiles weight
-astropy = ">=5.0, <5.2"
-
-#python-pysap = { git = "https://github.com/martinkilbinger/pysap.git", branch = "fix_versions" }
-ngmix = { git = "https://github.com/aguinot/ngmix", branch = "stable_version" }
-stile = { git = "https://github.com/tobias-liaudat/Stile", tag = "v0.1" }
-pywavelets = "^1.4.1"
-joblib = "^1.3.2"
-sip-tpv = "^1.1"
-sqlitedict = "^2.1.0"
-
-[tool.poetry.dev-dependencies]
-pytest = "^6.2.5"
-pydocstyle = "^6.3.0"
-pytest-cov = "^4.1.0"
-black = "^23.7.0"
-
-[tool.black]
-line-length = 80
-
-[build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
diff --git a/scripts/sh/init_canfar.sh b/scripts/sh/init_canfar.sh
index dd3346f94..d59035728 100755
--- a/scripts/sh/init_canfar.sh
+++ b/scripts/sh/init_canfar.sh
@@ -2,8 +2,8 @@
 
 echo "start init canfar"
 
-echo init_canfar > ~/init_canfar.log
-date >> ~/init_canfar.log
+#echo init_canfar > ~/init_canfar.log
+#date >> ~/init_canfar.log
 
 . /opt/conda/etc/profile.d/conda.sh
 
diff --git a/scripts/sh/init_run_canfar.sh b/scripts/sh/init_run_canfar.sh
new file mode 100755
index 000000000..1945d69fc
--- /dev/null
+++ b/scripts/sh/init_run_canfar.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+echo "start init canfar"
+
+#echo init_canfar > ~/init_canfar.log
+#date >> ~/init_canfar.log
+
+. /opt/conda/etc/profile.d/conda.sh
+
+conda activate shapepipe
+
+cd cosmostat/P3_v2/psfex 
+
+tile_ID=$1
+n_SMP=$2
+echo "tile_ID=$tile_ID n_SMP=$n_SMP"
+
+job_sp $tile_ID -p psfex -j 8 -n $n_SMP
+
+echo "end init canfar"
+
diff --git a/scripts/sh/job_sp.bash b/scripts/sh/job_sp.bash
index 15cb1428a..4d4e6fdcf 100755
--- a/scripts/sh/job_sp.bash
+++ b/scripts/sh/job_sp.bash
@@ -420,7 +420,7 @@ if [[ $do_job != 0 ]]; then
   ### Star detection, selection, PSF model. setools can exit with an error for CCD with insufficient stars,
   ### the script should continue
   STOP=0
-  command_sp "shapepipe_run -c $SP_CONFIG/config_tile_Sx_exp_${psf}.ini" "Run shapepipe (tile detection, exp $psf)"
+  command_cfg_shapepipe "config_tile_Sx_exp_${psf}.ini" "Run shapepipe (tile detection, exp $psf)" $n_smp
   STOP=1
 
 fi

From d35bd694c5d12d554866177c55ed62d50d9b67b0 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 19 Nov 2023 19:21:29 +0100
Subject: [PATCH 019/133] script to call curl for canfar container: added
 resources

---
 scripts/sh/curl_canfar.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scripts/sh/curl_canfar.sh b/scripts/sh/curl_canfar.sh
index 43fb6cd08..89bb4f64e 100755
--- a/scripts/sh/curl_canfar.sh
+++ b/scripts/sh/curl_canfar.sh
@@ -1,11 +1,12 @@
 #!/usr/bin/bash
 
 # Usage
-~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.7 shapepipe/scripts/sh/init_canfar.sh 000.000
+# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.7 shapepipe/scripts/sh/init_canfar.sh 000.000
 
 SSL=~/.ssl/cadcproxy.pem
 NCORE=16
-SESSION=https://ws-uv.canfar.net/skaha/v0/session?ram=16&cores=$NCORE
+SESSION=https://ws-uv.canfar.net/skaha/v0/session
+RESOURCES="ram=16&cores=$NCORE"
 IMAGE=images.canfar.net/unions/shapepipe
 NAME=shapepipe
 
@@ -24,7 +25,7 @@ arg_an=X
 echo
 echo "Start headless container"
 echo "========================"
-ID=`curl -E $SSL $SESSION -d "image=$IMAGE:$version" -d "name=${NAME}${arg_an}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
+ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}${arg_an}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
 echo $ID
 
 echo

From d93a60efa985b0df69849fe771f157f504950ac5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 19 Nov 2023 19:47:15 +0100
Subject: [PATCH 020/133] testing input numbers

---
 shapepipe/pipeline/file_handler.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shapepipe/pipeline/file_handler.py b/shapepipe/pipeline/file_handler.py
index b7479f6f3..037f445b8 100644
--- a/shapepipe/pipeline/file_handler.py
+++ b/shapepipe/pipeline/file_handler.py
@@ -985,6 +985,8 @@ def _save_num_patterns(
                 + 'directory is used.'
             )
 
+        print("MKDEBUG elem", elem)
+
         # Save file list
         np.save(output_file, np.array(final_file_list))
 

From fb83a275d096b710f115008c186f882bf020cf13 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 20 Nov 2023 17:19:01 +0100
Subject: [PATCH 021/133] removed debug msg

---
 shapepipe/pipeline/file_handler.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/shapepipe/pipeline/file_handler.py b/shapepipe/pipeline/file_handler.py
index 037f445b8..b7479f6f3 100644
--- a/shapepipe/pipeline/file_handler.py
+++ b/shapepipe/pipeline/file_handler.py
@@ -985,8 +985,6 @@ def _save_num_patterns(
                 + 'directory is used.'
             )
 
-        print("MKDEBUG elem", elem)
-
         # Save file list
         np.save(output_file, np.array(final_file_list))
 

From e33972b27993a38eacd9d347b0fea57f8bfe3f98 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 20 Nov 2023 22:11:23 +0000
Subject: [PATCH 022/133] numpy error fixed with version upgrade

---
 environment.yml                    | 10 +++++-----
 install_shapepipe                  |  2 +-
 scripts/sh/init_run_tile_canfar.sh | 32 ++++++++++++++++++++++++++++++
 shapepipe/pipeline/file_io.py      |  2 +-
 4 files changed, 39 insertions(+), 7 deletions(-)
 create mode 100755 scripts/sh/init_run_tile_canfar.sh

diff --git a/environment.yml b/environment.yml
index 489080262..90f95717c 100644
--- a/environment.yml
+++ b/environment.yml
@@ -3,9 +3,10 @@ channels:
   - conda-forge
 dependencies:
   - python=3.9
-  - pip
-  - numpy
-  - astropy
+  - pip>=21.2.4
+  - astromatic-psfex==3.21.1
+  - astromatic-source-extractor==2.25.0
+  - astropy==5.1
   - automake
   - autoconf
   - cmake
@@ -14,9 +15,8 @@ dependencies:
   - libtool
   - matplotlib
   - numba
-  - pandas
+  - numpy
   - pip:
-    - jupyter
     - cs_util
     - modopt
     - PyQt5
diff --git a/install_shapepipe b/install_shapepipe
index c370d91d0..644398b50 100755
--- a/install_shapepipe
+++ b/install_shapepipe
@@ -193,7 +193,7 @@ check_conda() {
   CONDA_VERSION_MAJOR=$(cut -d'.' -f1 <<<$CONDA_VERSION)
   CONDA_VERSION_MINOR=$(cut -d'.' -f2 <<<$CONDA_VERSION)
   CONDA_VERSION_PATCH=$(cut -d'.' -f3 <<<$CONDA_VERSION)
-  CONDA_SH=/etc/profile.d/conda.sh
+  CONDA_SH=/opt/conda/etc/profile.d/conda.sh
   # Check Conda major version
   if [ "$CONDA_VERSION_MAJOR" -lt 4 ]
   then
diff --git a/scripts/sh/init_run_tile_canfar.sh b/scripts/sh/init_run_tile_canfar.sh
new file mode 100755
index 000000000..402cd7425
--- /dev/null
+++ b/scripts/sh/init_run_tile_canfar.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+echo "start init run tile canfar"
+
+#echo init_canfar > ~/init_canfar.log
+#date >> ~/init_canfar.log
+
+. /opt/conda/etc/profile.d/conda.sh
+
+conda activate shapepipe
+
+basedir=cosmostat/P3_v2/psfex 
+cd $basedir
+
+tile_ID=$1
+n_SMP=$2
+echo "tile_ID=$tile_ID n_SMP=$n_SMP"
+
+mkdir $tile_ID
+cd $tile_ID
+mkdir output
+cd output
+cp $basedir/log_run_sp.txt .
+for dir in $basedir/run_sp_*; do
+	ln -s $$dir
+done
+cd ..
+
+job_sp $tile_ID -p psfex -j 16 -n $n_SMP
+
+echo "end init run tile canfar"
+
diff --git a/shapepipe/pipeline/file_io.py b/shapepipe/pipeline/file_io.py
index a768f0c06..52e847147 100644
--- a/shapepipe/pipeline/file_io.py
+++ b/shapepipe/pipeline/file_io.py
@@ -1549,7 +1549,7 @@ def _get_fits_col_type(self, col_data):
             col_type = 'D'
         elif type(col_data[0]) is bool:
             col_type = 'L'
-        elif type(col_data[0]) in [str, np.str, np.str_, np.str0]:
+        elif type(col_data[0]) in [str, np.str_, np.str0]:
             col_type = 'A'
         else:
             col_type = 'D'

From 9aa16bebed13ad62453325681c6db3dfcd49cca5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 22 Nov 2023 08:24:36 +0000
Subject: [PATCH 023/133] Getting ready for exclusive-one-tile processing

---
 example/cfis/config_tile_Ma_onthefly.ini |  82 ++++++++++++++++
 example/cfis/config_tile_Sx.ini          | 119 +++++++++++++++++++++++
 scripts/sh/init_run_tile_canfar.sh       |  11 ++-
 3 files changed, 210 insertions(+), 2 deletions(-)
 create mode 100644 example/cfis/config_tile_Ma_onthefly.ini
 create mode 100644 example/cfis/config_tile_Sx.ini

diff --git a/example/cfis/config_tile_Ma_onthefly.ini b/example/cfis/config_tile_Ma_onthefly.ini
new file mode 100644
index 000000000..6f3723a0f
--- /dev/null
+++ b/example/cfis/config_tile_Ma_onthefly.ini
@@ -0,0 +1,82 @@
+# ShapePipe configuration file for masking of tiles
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_tile_Ma
+
+# Add date and time to RUN_NAME, optional, default: False
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = mask_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 8
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+### Mask tiles
+[MASK_RUNNER_RUN]
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# Input file pattern(s), list of strings with length matching number of expected input file types
+# Cannot contain wild cards
+FILE_PATTERN = CFIS_image, CFIS_weight
+
+# FILE_EXT (optional) list of string extensions to identify input files
+FILE_EXT = .fits, .fits
+
+# Path of mask config file
+MASK_CONFIG_PATH = $SP_CONFIG/config_tile_onthefly.mask
+
+# External mask file flag, use if True, otherwise ignore
+USE_EXT_FLAG = False
+
+# External star catalogue flag, use external cat if True,
+# obtain from online catalogue if False
+USE_EXT_STAR = False
+
+# File name suffix for the output flag files (optional)
+PREFIX = pipeline
diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini
new file mode 100644
index 000000000..c5513aa0b
--- /dev/null
+++ b/example/cfis/config_tile_Sx.ini
@@ -0,0 +1,119 @@
+# ShapePipe configuration file for tile detection/
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_tile_Sx
+
+# Add date and time to RUN_NAME, optional, default: True
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = sextractor_runner
+         
+
+# Run mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 16
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+[SEXTRACTOR_RUNNER_RUN]
+
+INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1
+
+INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1
+
+FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag
+
+FILE_EXT = .fits, .fits, .fits
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# SExtractor executable path
+EXEC_PATH = sex
+
+# SExtractor configuration files
+DOT_SEX_FILE = $SP_CONFIG/default_tile.sex
+DOT_PARAM_FILE = $SP_CONFIG/default.param
+DOT_CONV_FILE = $SP_CONFIG/default.conv
+
+# Use input weight image if True
+WEIGHT_IMAGE = True
+
+# Use input flag image if True
+FLAG_IMAGE = True
+
+# Use input PSF file if True
+PSF_FILE = False
+
+# Use distinct image for detection (SExtractor in
+# dual-image mode) if True
+DETECTION_IMAGE = False
+
+# Distinct weight image for detection (SExtractor
+# in dual-image mode)
+DETECTION_WEIGHT = False
+
+ZP_FROM_HEADER = False
+
+BKG_FROM_HEADER = False
+
+# Type of image check (optional), default not used, can be a list of
+# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND,
+# MINIBACK_RMS, -BACKGROUND, #FILTERED,
+# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES
+CHECKIMAGE = BACKGROUND 
+
+# File name suffix for the output sextractor files (optional)
+SUFFIX = sexcat
+
+## Post-processing
+
+# Necessary for tiles, to enable multi-exposure processing
+MAKE_POST_PROCESS = True
+
+# Multi-epoch mode: Path to file with single-exposure WCS header information
+LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite
+
+# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y
+WORLD_POSITION = XWIN_WORLD,YWIN_WORLD
+
+# Number of pixels in x,y of a CCD. Format: Nx,Ny
+CCD_SIZE = 33,2080,1,4612
diff --git a/scripts/sh/init_run_tile_canfar.sh b/scripts/sh/init_run_tile_canfar.sh
index 402cd7425..9f0aa344f 100755
--- a/scripts/sh/init_run_tile_canfar.sh
+++ b/scripts/sh/init_run_tile_canfar.sh
@@ -9,9 +9,11 @@ echo "start init run tile canfar"
 
 conda activate shapepipe
 
-basedir=cosmostat/P3_v2/psfex 
+basedir=cosmostat/P3_v2/psfex
 cd $basedir
 
+cd tile_runs
+
 tile_ID=$1
 n_SMP=$2
 echo "tile_ID=$tile_ID n_SMP=$n_SMP"
@@ -26,7 +28,12 @@ for dir in $basedir/run_sp_*; do
 done
 cd ..
 
-job_sp $tile_ID -p psfex -j 16 -n $n_SMP
+export SP_RUN=.
+export SP_CONFIG=$basedir/cfis
+#shapepipe_run -c $SP_CONFIG/config_tile_Ma.ini
+shapepipe_run -c $SP_CONFIG/config_tile_Sx.iniig_tile_Sx.ini
+
+cd $basedir
 
 echo "end init run tile canfar"
 

From e5d03354346c041b4d220efc738e4f0e25238e83 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 22 Nov 2023 09:29:21 +0100
Subject: [PATCH 024/133] Added command line option to specify exclusive ID for
 processing

---
 shapepipe/pipeline/args.py         |  5 +++++
 shapepipe/pipeline/file_handler.py | 12 +++++++++++-
 shapepipe/pipeline/job_handler.py  |  4 ++++
 shapepipe/run.py                   |  6 +++++-
 4 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/shapepipe/pipeline/args.py b/shapepipe/pipeline/args.py
index e06e44ef9..21eecef84 100644
--- a/shapepipe/pipeline/args.py
+++ b/shapepipe/pipeline/args.py
@@ -128,5 +128,10 @@ def create_arg_parser():
         help='configuration file name',
     )
 
+    optional.add_argument(
+        '-e',
+        '--exclusive',
+        help='exclusive input filer number string',
+    )
     # Return parser
     return parser.parse_args()
diff --git a/shapepipe/pipeline/file_handler.py b/shapepipe/pipeline/file_handler.py
index b7479f6f3..63992df58 100644
--- a/shapepipe/pipeline/file_handler.py
+++ b/shapepipe/pipeline/file_handler.py
@@ -32,12 +32,14 @@ class FileHandler(object):
         List of modules to be run
     config : CustomParser
         Configuaration parser instance
+    exclusive : str, optional
+        Run this file number string exclusively if given, the default is None
     verbose : bool, optional
         Verbose setting, default is True
 
     """
 
-    def __init__(self, run_name, modules, config, verbose=True):
+    def __init__(self, run_name, modules, config, exclusive=None, verbose=True):
 
         self._run_name = run_name
 
@@ -46,6 +48,7 @@ def __init__(self, run_name, modules, config, verbose=True):
             raise ValueError('Invalid module list, check for a trailing comma')
 
         self._config = config
+        self._exclusive = exclusive
         self._verbose = verbose
 
         self.module_runners = get_module_runners(self._module_list)
@@ -1110,6 +1113,13 @@ def _format_process_list(
                     + f'numbering scheme "{num_scheme}".'
                 )
 
+            if self._exclusive is not None:
+                if re.search(number, self._exclusive) is not None:
+                    print(f"Using exclusive number {self._exclusive}")
+                else:
+                    print(f"Skipping {number}, not equal to {self._exclusive}")
+                    continue
+
             if run_method == 'serial':
                 process_items = []
             else:
diff --git a/shapepipe/pipeline/job_handler.py b/shapepipe/pipeline/job_handler.py
index d6baebd89..bc80f35da 100644
--- a/shapepipe/pipeline/job_handler.py
+++ b/shapepipe/pipeline/job_handler.py
@@ -42,6 +42,8 @@ class JobHandler(object):
         Joblib backend, the default is None (which corresponds to 'loky')
     timeout : int, optional
         Timeout limit for a given job in seconds, the default is None
+    exclusive : str, optional
+        Run this file number string exclusively if given, the default is None
     verbose : bool, optional
         Verbose setting, default is True
 
@@ -58,6 +60,7 @@ def __init__(
         batch_size=None,
         backend=None,
         timeout=None,
+        exclusive=None,
         verbose=True,
     ):
 
@@ -72,6 +75,7 @@ def __init__(
         self._module = module
         self._module_runner = self.filehd.module_runners[self._module]
         self.error_count = 0
+        self.exclusive = exclusive
         self._verbose = verbose
 
         # Add the job parameters to the log
diff --git a/shapepipe/run.py b/shapepipe/run.py
index 2443eab50..80bebbda5 100644
--- a/shapepipe/run.py
+++ b/shapepipe/run.py
@@ -51,12 +51,14 @@ def set_up(self):
         self._set_run_name()
         self.modules = self.config.getlist('EXECUTION', 'MODULE')
         self.mode = self.config.get('EXECUTION', 'MODE').lower()
+        self.exclusive=self._args.exclusive
         self.verbose = self.config.getboolean('DEFAULT', 'VERBOSE')
         self.filehd = FileHandler(
             self._run_name,
             self.modules,
             self.config,
-            self.verbose,
+            exclusive=self._args.exclusive,
+            verbose=self.verbose,
         )
         self.error_count = 0
         self._prep_run()
@@ -330,6 +332,7 @@ def run_smp(pipe):
             config=pipe.config,
             log=pipe.log,
             job_type=pipe.run_method[module],
+            exclusive=pipe.exclusive,
             verbose=pipe.verbose,
         )
 
@@ -388,6 +391,7 @@ def run_mpi(pipe, comm):
                 log=pipe.log,
                 job_type=pipe.run_method[module],
                 parallel_mode='mpi',
+                exclusive=pipe.exclusive,
                 verbose=verbose,
             )
 

From d02347ea52885a170ffb09a4e462965fd70f729e Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 24 Nov 2023 10:51:16 +0100
Subject: [PATCH 025/133] curl script exclusive ID

---
 scripts/sh/curl_canfar.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/scripts/sh/curl_canfar.sh b/scripts/sh/curl_canfar.sh
index 89bb4f64e..072097add 100755
--- a/scripts/sh/curl_canfar.sh
+++ b/scripts/sh/curl_canfar.sh
@@ -1,31 +1,33 @@
 #!/usr/bin/bash
 
 # Usage
-# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.7 shapepipe/scripts/sh/init_canfar.sh 000.000
+# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.8 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID exp NCORE
 
 SSL=~/.ssl/cadcproxy.pem
-NCORE=16
+NCORE=2
 SESSION=https://ws-uv.canfar.net/skaha/v0/session
-RESOURCES="ram=16&cores=$NCORE"
+RESOURCES="ram=4&cores=$NCORE"
 IMAGE=images.canfar.net/unions/shapepipe
 NAME=shapepipe
 
 # version of image on canfar, e.g. 0:7, 0:8
 version=$1
 
-# command on canfar, e.g. shapepipe/scripts/sh/init_run_canfar.sh
+# command on canfar, e.g. shapepipe/scripts/sh/init_run_exclusive_canfar.sh
 cmd=$2
 
-# command line argument, e.g. 181.308
-arg="$3 $NCORE"
+# Image ID
+ID=$3
+
+# command line arguments
+arg="$ID $NCORE exp"
 
 #arg_an=`echo "$arg" | tr '_' 'X' | tr '.' 'X'`
-arg_an=X
 
 echo
 echo "Start headless container"
 echo "========================"
-ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}${arg_an}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
+ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
 echo $ID
 
 echo

From 12e6b63aa7a0419317cfafb36c6e84e7853d915c Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 24 Nov 2023 11:28:38 +0100
Subject: [PATCH 026/133] Dockerfile conda -> source activate

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 98e6c35fc..cf9063246 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,5 +49,5 @@ RUN conda env create --file environment.yml
 COPY shapepipe ./shapepipe
 COPY scripts ./scripts
 
-RUN ./scripts/sh/init_canfar.sh
-#RUN conda activate shapepipe
+#RUN ./scripts/sh/init_canfar.sh
+RUN source activate shapepipe

From b05f2a56840b32947b100a5a725563792cb4e72d Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 24 Nov 2023 14:50:37 +0100
Subject: [PATCH 027/133] comment added

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index cf9063246..f24d5484c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,5 +49,5 @@ RUN conda env create --file environment.yml
 COPY shapepipe ./shapepipe
 COPY scripts ./scripts
 
-#RUN ./scripts/sh/init_canfar.sh
+# Activate conda environment using "source"
 RUN source activate shapepipe

From 45942d54503d75e489e768c4b06f5e0187560f98 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 24 Nov 2023 16:19:26 +0100
Subject: [PATCH 028/133] Updated Dockerfile

---
 Dockerfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f24d5484c..d4ea036d8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -39,6 +39,8 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+SHELL ["/bin/bash", "--login", "-c"]
+
 COPY ./environment.yml ./
 COPY install_shapepipe README.rst setup.py setup.cfg ./
 RUN touch ./README.md
@@ -50,4 +52,6 @@ COPY shapepipe ./shapepipe
 COPY scripts ./scripts
 
 # Activate conda environment using "source"
-RUN source activate shapepipe
+#RUN . /opt/conda/etc/profile.d/conda.sh
+#RUN conda init bash
+RUN conda activate shapepipe

From 7d1891e552948980b4bbed2a0d20dba06dd4733c Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 24 Nov 2023 15:35:51 +0000
Subject: [PATCH 029/133] running with exclusive ID

---
 example/cfis/config_exp_psfex.ini       | 149 +++++++
 scripts/sh/init_run_exclusive_canfar.sh |  43 ++
 scripts/sh/job_sp_canfar.bash           | 535 ++++++++++++++++++++++++
 3 files changed, 727 insertions(+)
 create mode 100644 example/cfis/config_exp_psfex.ini
 create mode 100755 scripts/sh/init_run_exclusive_canfar.sh
 create mode 100755 scripts/sh/job_sp_canfar.bash

diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini
new file mode 100644
index 000000000..bfc46675f
--- /dev/null
+++ b/example/cfis/config_exp_psfex.ini
@@ -0,0 +1,149 @@
+# ShapePipe configuration file for single-exposures. PSFex PSF model.
+# Process exposures after masking, from star detection to PSF model.
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_exp_SxSePsf
+
+# Add date and time to RUN_NAME, optional, default: True
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = sextractor_runner, setools_runner, psfex_runner
+         
+
+# Run mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
+INPUT_DIR = $SP_RUN/output
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 16
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+[SEXTRACTOR_RUNNER]
+
+# Input from two modules
+INPUT_MODULE = split_exp_runner, mask_runner
+
+# Read pipeline flag files created by mask module
+FILE_PATTERN = image, weight, pipeline_flag
+
+NUMBERING_SCHEME = -0000000-0
+
+# SExtractor executable path
+EXEC_PATH = sex
+
+# SExtractor configuration files
+DOT_SEX_FILE = $SP_CONFIG/default_exp.sex
+DOT_PARAM_FILE = $SP_CONFIG//default.param
+DOT_CONV_FILE = $SP_CONFIG/default.conv
+
+# Use input weight image if True
+WEIGHT_IMAGE = True
+
+# Use input flag image if True
+FLAG_IMAGE = True
+
+# Use input PSF file if True
+PSF_FILE = False
+
+# Use distinct image for detection (SExtractor in
+# dual-image mode) if True.
+DETECTION_IMAGE = False
+
+# Distinct weight image for detection (SExtractor
+# in dual-image mode)
+DETECTION_WEIGHT = False
+
+# True if photometry zero-point is to be read from exposure image header
+ZP_FROM_HEADER = True
+
+# If ZP_FROM_HEADER is True, zero-point key name
+ZP_KEY = PHOTZP
+
+# Background information from image header.
+# If BKG_FROM_HEADER is True, background value will be read from header.
+# In that case, the value of BACK_TYPE will be set atomatically to MANUAL.
+# This is used e.g. for the LSB images.
+BKG_FROM_HEADER = False
+# LSB images:
+# BKG_FROM_HEADER = True
+
+# If BKG_FROM_HEADER is True, background value key name
+# LSB images:
+#BKG_KEY = IMMODE
+
+# Type of image check (optional), default not used, can be a list of
+# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, 
+# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES
+CHECKIMAGE = BACKGROUND
+
+# File name suffix for the output sextractor files (optional) SUFFIX = tile
+SUFFIX = sexcat
+
+## Post-processing
+
+# Not required for single exposures
+MAKE_POST_PROCESS = FALSE
+
+
+[SETOOLS_RUNNER]
+
+INPUT_DIR = last:sextractor_runner
+
+# Note: Make sure this doe not match the SExtractor background images
+# (sexcat_background*)
+FILE_PATTERN = sexcat
+
+NUMBERING_SCHEME = -0000000-0
+
+# SETools config file
+SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools
+
+
+[PSFEX_RUNNER]
+
+# Use 80% sample for PSF model
+FILE_PATTERN = star_split_ratio_80
+
+NUMBERING_SCHEME = -0000000-0
+
+# Path to executable for the PSF model (optional)
+EXEC_PATH = psfex
+
+# Default psfex configuration file
+DOT_PSFEX_FILE = $SP_CONFIG/default.psfex
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
new file mode 100755
index 000000000..0e4a6fb34
--- /dev/null
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+echo "start init run exclusive canfar"
+
+#echo init_canfar > ~/init_canfar.log
+#date >> ~/init_canfar.log
+
+. /opt/conda/etc/profile.d/conda.sh
+
+conda activate shapepipe
+
+basedir=$HOME/cosmostat/P3_v2/psfex
+cd $basedir
+
+
+ID=$1
+n_SMP=$2
+typ=$3
+echo "ID=$ID n_SMP=$n_SMP type=$typ"
+
+cd ${typ}_runs
+
+mkdir $ID
+cd $ID
+mkdir output
+cd output
+cp $basedir/output/log_run_sp.txt .
+ln -s $basedir/output/log_exp_headers.sqlite
+for dir in $basedir/output/run_sp_*; do
+	ln -s $dir
+done
+cd ..
+
+#export SP_RUN=.
+#export SP_CONFIG=$HOME/shapepipe/example/cfis
+#shapepipe_run -c $SP_CONFIG/config_tile_Ma.ini
+#shapepipe_run -c $SP_CONFIG/config_tile_Sx.ini -e $ID
+job_sp_canfar.bash -p psfex -j 32 -e $ID
+
+cd $basedir
+
+echo "end init run tile canfar"
+
diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
new file mode 100755
index 000000000..cbafd6f19
--- /dev/null
+++ b/scripts/sh/job_sp_canfar.bash
@@ -0,0 +1,535 @@
+#!/usr/bin/env bash
+
+# Name: job_sp_canfar.bash
+# Description: General script to process one or more tiles
+#              with all contributing exposures.
+#              This works as job submission script for
+#              the canfar batch system.
+#              called in interactive mode on a virtual
+#              machine.
+# Author: Martin Kilbinger <martin.kilbinger@cea.fr>
+
+
+# Command line arguments
+## Default values
+job=255
+config_dir=$HOME/shapepipe/example/cfis
+psf='mccd'
+retrieve='vos'
+star_cat_for_mask='onthefly'
+exclusive=''
+results='cosmostat/kilbinger/results_v2'
+n_smp=-1
+nsh_step=-1
+nsh_max=-1
+nsh_jobs=8
+
+## Help string
+usage="Usage: $(basename "$0") [OPTIONS] [TILE_ID]
+\n\nOptions:\n
+   -h\tthis message\n
+   -j, --job JOB\tRunning JOB, bit-coded\n
+   \t   1: retrieve images (online if method=vos)\n
+   \t   2: prepare images (offline)\n
+   \t   4: mask tiles (online if star_cat_for_mask=onthefly)\n
+   \t   8: mask exposures (online if star_cat_for_mask=onthefly)\n
+   \t  16: detection of galaxies on tiles (offline)\n
+   \t  32: processing of stars on exposures (offline)\n
+   \t  64: galaxy selection on tiles (offline)\n
+   \t 128: shapes and morphology (offline)\n
+   \t 256: paste catalogues (offline)\n
+   \t 512: upload results (online)\n
+   -c, --config_dir DIR\n
+   \t config file directory, default='$config_dir'\n
+   -p, --psf MODEL\n
+    \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n
+   -r, --retrieve METHOD\n
+   \tmethod to retrieve images, allowed are 'vos', 'symlink', default='$retrieve'\n
+   -s, --star_cat_for_mask\n
+   \tcatalogue for masking bright stars, allowed are 'onthefly', 'save',\n
+   \tdefault is '${star_cat_for_mask}'\n
+   -e, --exclusive ID\n
+   \texclusive input filer number string ID (default: None)\n
+   -o, --output_dir\n
+   \toutput (upload) directory on vos:cfis, default='$results'\n
+   -n, --n_smp\n
+   \tnumber of jobs (SMP mode only), default from original config files\n
+   --nsh_step NSTEP\n
+   --nsh_jobs NJOB\n
+   \tnumber of shape measurement parallel jobs, default=$nsh_jobs\n
+   \tnumber of objects per parallel shape module call, \n
+   \tdefault: optimal number is computed\n
+   --nsh_max NMAX\n
+   \tmax number of objects per parallel shape module call, \n
+   \tdefault: unlimited; has precedent over --nsh_step\n
+   TILE_ID_i\n
+   \ttile ID(s), e.g. 283.247 214.242\n
+"
+
+## Help if no arguments
+if [ -z $1 ]; then
+        echo -ne $usage
+        exit 1
+fi
+
+## Parse command line
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -h)
+      echo -ne $usage
+      exit 0
+      ;;
+    -j|--job)
+      job="$2"
+      shift
+      ;;
+    -c|--config_dir)
+      config_dir="$2"
+      shift
+      ;;
+    -p|--psf)
+      psf="$2"
+      shift
+      ;;
+    -r|--retrieve)
+      retrieve="$2"
+      shift
+      ;;
+    -s|--star_cat_for_mask)
+      star_cat_for_mask="$2"
+      shift
+      ;;
+    -e|--exclusive)
+      exclusive="$2"
+      shift
+      ;;
+     -o|--output_dir)
+      results="$2"
+      shift
+      ;;
+    -n|--n_smp)
+      n_smp="$2"
+      shift
+      ;;
+    --nsh_max)
+      nsh_max="$2"
+      shift
+      ;;
+    --nsh_step)
+      nsh_step="$2"
+      shift
+      ;;
+    --nsh_jobs)
+      nsh_jobs="$2"
+      shift
+      ;;
+  esac
+  shift
+done
+
+## Check options
+if [ "$psf" != "psfex" ] && [ "$psf" != "mccd" ]; then
+  echo "PSF (option -p) needs to be 'psfex' or 'mccd'"
+  exit 2
+fi
+
+if [ "$star_cat_for_mask" != "onthefly" ] && [ "$star_cat_for_mask" != "save" ]; then
+  echo "Star cat for mask (option -s) needs to be 'onthefly' or 'save'"
+  exit 4
+fi
+
+if [ "$retrieve" != "vos" ] && [ "$retrieve" != "symlink" ]; then
+  echo "method to retrieve images (option -r) needs to be 'vos' or 'symlink'"
+  exit 5
+fi
+
+if [ $nsh_max != -1 ]; then
+  nsh_step=$nsh_max
+fi
+
+# For tar archives. TODO: Should be unique to each job
+export ID="test"
+
+## Paths
+
+## Path variables used in shapepipe config files
+
+# Run path and location of input image directories
+export SP_RUN=`pwd`
+
+# Config file path
+export SP_CONFIG=$SP_RUN/cfis
+export SP_CONFIG_MOD=$SP_RUN/cfis_mod
+
+## Other variables
+
+# Input tile numbers ASCII file
+export TILE_NUMBERS_PATH=tile_numbers.txt
+
+# Output
+OUTPUT=$SP_RUN/output
+
+# For tar archives
+output_rel=`realpath --relative-to=. $OUTPUT`
+
+# Stop on error, default=1
+STOP=1
+
+# Verbose mode (1: verbose, 0: quiet)
+VERBOSE=1
+
+# VCP options
+export CERTFILE=$HOME/.ssl/cadcproxy.pem
+export VCP="vcp --certfile=$CERTFILE"
+
+
+## Functions
+
+# Print string, executes command, and prints return value.
+function command () {
+   cmd=$1
+   str=$2
+
+   RED='\033[0;31m'
+   GREEN='\033[0;32m'
+   NC='\033[0m' # No Color
+   # Color escape characters show up in log files
+   #RED=''
+   #GREEN=''
+   #NC=''
+
+
+   if [ $# == 2 ]; then
+      if [ $VERBOSE == 1 ]; then
+           echo "$str: running '$cmd'"
+      fi
+      $cmd
+   else
+      if [ $VERBOSE == 1 ]; then
+         echo "$str: running '$cmd $4 \"$5 $6\"'"
+      fi
+      $cmd $4 "$5 $6"
+   fi	
+   res=$?
+
+   if [ $VERBOSE == 1 ]; then
+      if [ $res == 0 ]; then
+         echo -e "${GREEN}success, return value = $res${NC}"
+      else
+         echo -e "${RED}error, return value = $res${NC}"
+         if [ $STOP == 1 ]; then
+            echo "${RED}exiting 'canfar_sp.bash', error in command '$cmd'${NC}"
+            exit $res
+         else
+            echo "${RED}continuing 'canfar_sp.bash', error in command '$cmd'${NC}"
+         fi
+      fi
+   fi
+}
+
+# Run shapepipe command. If error occurs, upload sp log files before stopping script.
+function command_sp() {
+   local cmd=$1
+   local str=$2
+
+   command "$1" "$2"
+}
+
+# Set up config file and call shapepipe_run
+function command_cfg_shapepipe() {
+    local config_name=$1
+    local str=$2
+    local _n_smp=$3 
+    local _exclusive=$4
+
+    if [ "$exclusive" != "" ]; then
+      exclusive_flag="-e $_exclusive"
+    else
+      exclusive_flag=""
+    fi
+
+    config_upd=$(set_config_n_smp $config_name $_n_smp)
+    local cmd="shapepipe_run -c $config_upd $exclusive_flag"
+    command_sp "$cmd" "$str"
+}
+
+# Tar and upload files to vos
+function upload() {
+   base=$1
+   shift
+   ID=$1
+   shift
+   verbose=$1
+   shift
+   upl=("$@")
+
+   echo "Counting upload files"
+   n_upl=(`ls -l ${upl[@]} | wc`)
+   if [ $n_upl == 0 ]; then
+      if [ $STOP == 1 ]; then
+         echo "Exiting script, no file found for '$base' tar ball"
+         exit 3
+      fi
+   fi
+   tar czf ${base}_${ID}.tgz ${upl[@]}
+   command "$VCP ${base}_${ID}.tgz vos:cfis/$results" "Upload tar ball"
+}
+
+# Upload log files
+function upload_logs() {
+   id=$1
+   verbose=$2
+
+   upl="$output_rel/*/*/logs $output_rel/*/logs"
+   upload "logs" "$id" "$verbose" "${upl[@]}"
+}
+
+function set_config_n_smp() {
+  local config_name=$1
+  local _n_smp=$2
+
+  local config_orig="$SP_CONFIG/$config_name"
+
+  if [[ $_n_smp != -1 ]]; then
+    # Update SMP batch size
+    local config_upd="$SP_CONFIG_MOD/$config_name"
+    update_config $config_orig $config_upd "SMP_BATCH_SIZE" $_n_smp
+  else
+    # Keep original config file
+    local config_upd=$config_orig
+  fi
+
+  # Set "return" value (stdout)
+  echo "$config_upd"
+}
+
+# Update config file
+function update_config() {
+  local config_orig=$1
+  local config_upd=$2
+  local key=$3
+  local val_upd=$4
+
+  cat $config_orig \
+    | perl -ane 's/'$key'\s+=.+/'$key' = '$val_upd'/; print' > $config_upd
+}
+
+### Start ###
+
+echo "Start processing"
+
+# Create input and output directories
+mkdir -p $SP_RUN
+cd $SP_RUN
+mkdir -p $OUTPUT
+mkdir -p $SP_CONFIG_MOD
+
+# Processing
+
+## Check for input tile list
+#if [ ! -e $TILE_NUMBERS_PATH ]; then
+  #echo "Tile numbers file $TILE_NUMBERS_PATH not found, exiting"
+#fi
+
+### Retrieve config files
+if [[ $config_dir == *"vos:"* ]]; then
+  command_sp "$VCP $config_dir ." "Retrieve shapepipe config files"
+else
+  if [[ ! -L cfis ]]; then
+    command_sp "ln -s $config_dir cfis" "Retrieve shapepipe config files"
+  fi
+fi
+
+
+## Retrieve config files and images (online if retrieve=vos)
+## Retrieve and save star catalogues for masking (if star_cat_for_mask=save)
+(( do_job= $job & 1 ))
+if [[ $do_job != 0 ]]; then
+
+  ### Retrieve files
+  command_sp "shapepipe_run -c $SP_CONFIG/config_GitFeGie_$retrieve.ini" "Retrieve images"
+
+  ### Retrieve and save star catalogues for masking
+  if [ "$star_cat_for_mask" == "save" ]; then
+    #### For tiles
+    mkdir $SP_RUN/star_cat_tiles
+    command_sp "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_1/output $SP_RUN/star_cat_tiles" "Save star cats for masking (tile)"
+
+    #### For single-exposures
+    mkdir $SP_RUN/star_cat_exp
+    command_sp "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_2/output $SP_RUN/star_cat_exp exp" "Save star cats for masking (exp)"
+  fi
+
+fi
+
+## Prepare images (offline)
+(( do_job= $job & 2 ))
+if [[ $do_job != 0 ]]; then
+
+  ### Uncompress tile weights
+  command_cfg_shapepipe "config_tile_Uz.ini" "Run shapepipe (uncompress tile weights)" $n_smp $exclusive
+
+  ### Split images into single-HDU files, merge headers for WCS info
+  command_cfg_shapepipe "config_exp_SpMh.ini" "Run shapepipe (split images, merge headers)" $n_smp $exclusive
+
+fi
+
+## Mask tiles: add star, halo, and Messier object masks (online if "star_cat_for_mask" is "onthefly")
+(( do_job= $job & 4 ))
+if [[ $do_job != 0 ]]; then
+
+  ### Mask tiles
+  command_cfg_shapepipe "config_tile_Ma_$star_cat_for_mask.ini" "Run shapepipe (mask tiles)" $n_smp $exclusive
+
+fi
+
+## Mask exposures: add star, halo, and Messier object masks (online if "star_cat_for_mask" is "onthefly")
+(( do_job= $job & 8 ))
+if [[ $do_job != 0 ]]; then
+
+  ### Mask exposures
+  command_cfg_shapepipe "config_ext_Ma_$star_cat_for_mask.ini" "Run shapepipe (mask exposures)" $n_smp $exclusive
+
+fi
+
+
+## Remaining exposure processing (offline)
+(( do_job= $job & 16 ))
+if [[ $do_job != 0 ]]; then
+
+  ### Object detection on tiles
+  command_cfg_shapepipe "config_tile_Sx.ini" "Run shapepipe (tile detection)" $n_smp $exclusive
+
+fi
+
+## Exposure processing (offline)
+(( do_job= $job & 32 ))
+if [[ $do_job != 0 ]]; then
+
+  ### Star detection, selection, PSF model. setools can exit with an error for CCD with insufficient stars,
+  ### the script should continue
+  STOP=0
+  command_cfg_shapepipe "config_exp_${psf}.ini" "Run shapepipe (exp $psf)" $n_smp $exclusive
+  STOP=1
+
+fi
+
+## Process tiles up to shape measurement
+(( do_job= $job & 64 ))
+if [[ $do_job != 0 ]]; then
+
+  ### PSF model letter: 'P' (psfex) or 'M' (mccd)
+  letter=${psf:0:1}
+  Letter=${letter^}
+  command_sp "shapepipe_run -c $SP_CONFIG/config_tile_${Letter}iViSmVi.ini" "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)"
+
+fi
+
+## Shape measurement (offline)
+(( do_job= $job & 128 ))
+if [[ $do_job != 0 ]]; then
+
+  ### Prepare config files
+  n_min=0
+  if [[ $nsh_step == -1 ]]; then
+    n_obj=`get_number_objects.py`
+    nsh_step=`echo "$(($n_obj/$nsh_jobs))"`
+  fi
+
+  n_max=$((nsh_step - 1))
+  for k in $(seq 1 $nsh_jobs); do
+    cat $SP_CONFIG/config_tile_Ng_template.ini | \
+      perl -ane \
+        's/(ID_OBJ_MIN =) X/$1 '$n_min'/; s/(ID_OBJ_MAX =) X/$1 '$n_max'/; s/NgXu/Ng'$k'u/; s/X_interp/'$psf'_interp/g; print' \
+        > $SP_CONFIG_MOD/config_tile_Ng${k}u.ini
+    n_min=$((n_min + nsh_step))
+    if [ "$k" == $((nsh_jobs - 1)) ] && [ $nsh_max == -1 ]; then
+      n_max=-1
+    else
+      n_max=$((n_min + nsh_step - 1))
+    fi
+  done
+
+  ### Shapes, run $nsh_jobs parallel processes
+  VERBOSE=0
+  for k in $(seq 1 $nsh_jobs); do
+      command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" "Run shapepipe (tile: ngmix+galsim $k)" &
+  done
+  wait
+  VERBOSE=1
+
+fi
+
+## Create final catalogues (offline)
+(( do_job= $job & 256 ))
+if [[ $do_job != 0 ]]; then
+
+  cat $SP_CONFIG/config_merge_sep_cats_template.ini | \
+    perl -ane \
+      's/(N_SPLIT_MAX =) X/$1 '$nsh_jobs'/; print' \
+      > $SP_CONFIG_MOD/config_merge_sep_cats.ini
+ 
+  ### Merge separated shapes catalogues
+  command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_merge_sep_cats.ini" "Run shapepipe (tile: merge sep cats)" "$VERBOSE" "$ID"
+
+  ### Merge all relevant information into final catalogue
+  command_sp "shapepipe_run -c $SP_CONFIG/config_make_cat_$psf.ini" "Run shapepipe (tile: create final cat $psf)" "$VERBOSE" "$ID"
+
+fi
+
+## Upload results (online)
+(( do_job= $job & 512 ))
+if [[ $do_job != 0 ]]; then
+
+  ### module and pipeline log files
+  upload_logs "$ID" "$VERBOSE"
+
+  ### Final shape catalog
+  ### pipeline_flags are the tile masks, for random cats
+  ### SETools masks (selection), stats and plots
+  ### ${psf}_interp_exp for diagnostics, validation with leakage,
+  ### validation with residuals, rho stats
+
+  NAMES=(
+    "final_cat"
+    "pipeline_flag"
+    "setools_mask"
+    "setools_stat"
+    "setools_plot"
+  )
+  DIRS=(
+    "*/make_cat_runner/output"
+    "*/mask_runner_run_1/output"
+    "*/setools_runner/output/mask"
+    "*/setools_runner/output/stat"
+    "*/setools_runner/output/plot"
+  )
+  PATTERNS=(
+    "final_cat-*"
+    "pipeline_flag-???-???*"
+    "*"
+    "*"
+    "*"
+  )
+
+  # PSF validation
+  pattern="validation_psf-*"
+  if [ "$psf" == "psfex" ]; then
+    name="psfex_interp_exp"
+    dir="*/psfex_interp_runner/output"
+  else
+    name="mccd_fit_val_runner"
+    dir="*/mccd_fit_val_runner/output"
+  fi
+  upl=$output_rel/$dir/$pattern
+  upload "$name" "$ID" "$VERBOSE" "${upl[@]}"
+
+  for n in "${!NAMES[@]}"; do
+      name=${NAMES[$n]}
+      dir=${DIRS[$n]}
+      pattern=${PATTERNS[$n]}
+      upl=$output_rel/$dir/$pattern
+      upload "$name" "$ID" "$VERBOSE" "${upl[@]}"
+  done
+
+fi

From 02affa88cd0f02651182f90b1f77ea835b8ea31c Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sat, 25 Nov 2023 08:07:39 +0000
Subject: [PATCH 030/133] exp runs

---
 example/cfis/config_tile_Sx.ini         |  6 ++--
 scripts/sh/init_canfar.sh               |  7 +----
 scripts/sh/init_run_exclusive_canfar.sh |  4 +--
 scripts/sh/init_run_tile_canfar.sh      | 39 -------------------------
 scripts/sh/job_sp.bash                  | 22 --------------
 shapepipe/pipeline/file_handler.py      | 11 +++++--
 6 files changed, 14 insertions(+), 75 deletions(-)
 delete mode 100755 scripts/sh/init_run_tile_canfar.sh

diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini
index c5513aa0b..09eb921c2 100644
--- a/example/cfis/config_tile_Sx.ini
+++ b/example/cfis/config_tile_Sx.ini
@@ -35,7 +35,7 @@ LOG_NAME = log_sp
 RUN_LOG_NAME = log_run_sp
 
 # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
-INPUT_DIR = .
+INPUT_DIR = $SP_RUN/output
 
 # Output directory
 OUTPUT_DIR = $SP_RUN/output
@@ -45,7 +45,7 @@ OUTPUT_DIR = $SP_RUN/output
 [JOB]
 
 # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
-SMP_BATCH_SIZE = 16
+SMP_BATCH_SIZE = 2
 
 # Timeout value (optional), default is None, i.e. no timeout limit applied
 TIMEOUT = 96:00:00
@@ -53,7 +53,7 @@ TIMEOUT = 96:00:00
 
 ## Module options
 
-[SEXTRACTOR_RUNNER_RUN]
+[SEXTRACTOR_RUNNER]
 
 INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1
 
diff --git a/scripts/sh/init_canfar.sh b/scripts/sh/init_canfar.sh
index 8f5a955c9..8229aff23 100755
--- a/scripts/sh/init_canfar.sh
+++ b/scripts/sh/init_canfar.sh
@@ -2,17 +2,12 @@
 
 echo "start init canfar"
 
-<<<<<<< HEAD
 #echo init_canfar > ~/init_canfar.log
 #date >> ~/init_canfar.log
-=======
-echo init_canfar > ~/init_canfar.log
-date >> ~/init_canfar.log
->>>>>>> origin/exclusive
 
 . /opt/conda/etc/profile.d/conda.sh
 
-conda activate shapepipe
+source activate shapepipe
 
 echo "end init canfar"
 
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 0e4a6fb34..7f9f428e1 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -33,9 +33,9 @@ cd ..
 
 #export SP_RUN=.
 #export SP_CONFIG=$HOME/shapepipe/example/cfis
-#shapepipe_run -c $SP_CONFIG/config_tile_Ma.ini
 #shapepipe_run -c $SP_CONFIG/config_tile_Sx.ini -e $ID
-job_sp_canfar.bash -p psfex -j 32 -e $ID
+
+job_sp_canfar.bash -p psfex -j 32 -e $ID -n $n_SMP
 
 cd $basedir
 
diff --git a/scripts/sh/init_run_tile_canfar.sh b/scripts/sh/init_run_tile_canfar.sh
deleted file mode 100755
index 9f0aa344f..000000000
--- a/scripts/sh/init_run_tile_canfar.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-
-echo "start init run tile canfar"
-
-#echo init_canfar > ~/init_canfar.log
-#date >> ~/init_canfar.log
-
-. /opt/conda/etc/profile.d/conda.sh
-
-conda activate shapepipe
-
-basedir=cosmostat/P3_v2/psfex
-cd $basedir
-
-cd tile_runs
-
-tile_ID=$1
-n_SMP=$2
-echo "tile_ID=$tile_ID n_SMP=$n_SMP"
-
-mkdir $tile_ID
-cd $tile_ID
-mkdir output
-cd output
-cp $basedir/log_run_sp.txt .
-for dir in $basedir/run_sp_*; do
-	ln -s $$dir
-done
-cd ..
-
-export SP_RUN=.
-export SP_CONFIG=$basedir/cfis
-#shapepipe_run -c $SP_CONFIG/config_tile_Ma.ini
-shapepipe_run -c $SP_CONFIG/config_tile_Sx.iniig_tile_Sx.ini
-
-cd $basedir
-
-echo "end init run tile canfar"
-
diff --git a/scripts/sh/job_sp.bash b/scripts/sh/job_sp.bash
index 4d4e6fdcf..2a207d807 100755
--- a/scripts/sh/job_sp.bash
+++ b/scripts/sh/job_sp.bash
@@ -8,11 +8,6 @@
 #              called in interactive mode on a virtual
 #              machine.
 # Author: Martin Kilbinger <martin.kilbinger@cea.fr>
-# Date: v1.0 11/2020
-#       v1.1 01/2021
-
-# MKDEBUG TODO:
-# Option to change SMP_BATCH_SIZE, not for MPI
 
 
 # VM home, required for canfar run.
@@ -294,23 +289,6 @@ function upload_logs() {
    upload "logs" "$id" "$verbose" "${upl[@]}"
 }
 
-# Print script variables
-function print_env() {
-   echo "*** Environment ***"
-   echo "Data:"
-   echo " TILE_ARR=${TILE_ARR[@]}"
-   echo "Paths:"
-   echo " VM_HOME=$VM_HOME"
-   echo " SP_RUN=$SP_RUN"
-   echo " TILE_NUMBERS_PATH=$TILE_NUMBERS_PATH"
-   echo " OUTPUT=$OUTPUT"
-   echo " SP_CONFIG=$SP_CONFIG"
-   echo "Other variables:"
-   echo " VCP=$VCP"
-   echo " CERTFILE=$CERTFILE"
-   echo "***"
-}
-
 function set_config_n_smp() {
   local config_name=$1
   local _n_smp=$2
diff --git a/shapepipe/pipeline/file_handler.py b/shapepipe/pipeline/file_handler.py
index 63992df58..51b02cdd7 100644
--- a/shapepipe/pipeline/file_handler.py
+++ b/shapepipe/pipeline/file_handler.py
@@ -1113,11 +1113,16 @@ def _format_process_list(
                     + f'numbering scheme "{num_scheme}".'
                 )
 
+            # If "exclusive" options is set: discard all non-matching IDs
             if self._exclusive is not None:
-                if re.search(number, self._exclusive) is not None:
-                    print(f"Using exclusive number {self._exclusive}")
+                id_to_test = f"-{self._exclusive.replace('.', '-')}"
+                if number == id_to_test:
+                    if self._verbose:
+                        print(f"Using exclusive number {self._exclusive} ({id_to_test})")
                 else:
-                    print(f"Skipping {number}, not equal to {self._exclusive}")
+                    if self._verbose:
+                        #print(f"Skipping {number}, not equal to {self._exclusive} ({id_to_test})")
+                        pass
                     continue
 
             if run_method == 'serial':

From d8ad3280f5270b8a874bb3216d58765788a832e9 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sat, 25 Nov 2023 14:59:47 +0100
Subject: [PATCH 031/133] local curl script NCORES -> 1

---
 scripts/sh/curl_canfar.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/sh/curl_canfar.sh b/scripts/sh/curl_canfar.sh
index 072097add..7b9e1de91 100755
--- a/scripts/sh/curl_canfar.sh
+++ b/scripts/sh/curl_canfar.sh
@@ -4,7 +4,7 @@
 # ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.8 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID exp NCORE
 
 SSL=~/.ssl/cadcproxy.pem
-NCORE=2
+NCORE=1
 SESSION=https://ws-uv.canfar.net/skaha/v0/session
 RESOURCES="ram=4&cores=$NCORE"
 IMAGE=images.canfar.net/unions/shapepipe

From 60c3cd1b25ce28d99e271e00ce1af76e79559421 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 28 Nov 2023 18:23:57 +0100
Subject: [PATCH 032/133] Renamed science-portal scripts (local/remote)

---
 .../{curl_canfar.sh => curl_canfar_local.sh}  | 21 +----------------
 scripts/sh/curl_canfar_monitor_local.sh       | 23 +++++++++++++++++++
 .../{init_canfar.sh => init_canfar_remote.sh} |  0
 3 files changed, 24 insertions(+), 20 deletions(-)
 rename scripts/sh/{curl_canfar.sh => curl_canfar_local.sh} (64%)
 create mode 100755 scripts/sh/curl_canfar_monitor_local.sh
 rename scripts/sh/{init_canfar.sh => init_canfar_remote.sh} (100%)

diff --git a/scripts/sh/curl_canfar.sh b/scripts/sh/curl_canfar_local.sh
similarity index 64%
rename from scripts/sh/curl_canfar.sh
rename to scripts/sh/curl_canfar_local.sh
index 7b9e1de91..c875d4cb3 100755
--- a/scripts/sh/curl_canfar.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -22,25 +22,6 @@ ID=$3
 # command line arguments
 arg="$ID $NCORE exp"
 
-#arg_an=`echo "$arg" | tr '_' 'X' | tr '.' 'X'`
-
 echo
-echo "Start headless container"
-echo "========================"
 ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
-echo $ID
-
-echo
-echo "Events (incl. errors)"
-echo "====================="
-cmd="curl -E $SSL $SESSION/$ID?view=events"
-echo $cmd
-$cmd
-
-echo
-echo "Logs (incl. stdout)"
-echo "==================="
-cmd="curl -E $SSL $SESSION/$ID?view=logs"
-echo $cmd
-$cmd
-
+echo $ID >> IDs.txt
diff --git a/scripts/sh/curl_canfar_monitor_local.sh b/scripts/sh/curl_canfar_monitor_local.sh
new file mode 100755
index 000000000..412e33c01
--- /dev/null
+++ b/scripts/sh/curl_canfar_monitor_local.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/bash
+
+# --false-start --tcp-fastopen  faster
+
+# --http2   slower
+# --tlsv1.2 slower?
+# --insecure ?
+
+# -H "Accept-Encoding: gzip"    faster?
+
+SSL=~/.ssl/cadcproxy.pem
+SESSION=https://ws-uv.canfar.net/skaha/v0/session
+
+type=$1
+
+echo "type=$type"
+
+while [ 1 ]; do
+  ID=`tail -n 1 IDs.txt`
+  cmd="curl -E $SSL $SESSION/$ID?view=$type"
+  echo $cmd
+  $cmd
+done
diff --git a/scripts/sh/init_canfar.sh b/scripts/sh/init_canfar_remote.sh
similarity index 100%
rename from scripts/sh/init_canfar.sh
rename to scripts/sh/init_canfar_remote.sh

From d331171728891660662eb3c682029738afee4356 Mon Sep 17 00:00:00 2001
From: Samuel Farrens <samuel.farrens@gmail.com>
Date: Thu, 30 Nov 2023 17:48:04 +0100
Subject: [PATCH 033/133] added BigMac depdendency for macOS

---
 .github/workflows/ci-dev.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
index a260f6c9d..984a9e7d6 100644
--- a/.github/workflows/ci-dev.yml
+++ b/.github/workflows/ci-dev.yml
@@ -28,7 +28,8 @@ jobs:
         shell: bash -l {0}
         if: runner.os == 'macOS'
         run: |
-          brew install libomp
+          brew tap sfarrens/sf
+          brew install bigmac libomp
 
       - name: Set up conda
         uses: conda-incubator/setup-miniconda@v2

From e335922cdbf5de302ff2f98c5c89bf0667fb4f16 Mon Sep 17 00:00:00 2001
From: sfarrens <samuel.farrens@gmail.com>
Date: Fri, 1 Dec 2023 10:05:55 +0100
Subject: [PATCH 034/133] activate dev environment

---
 .github/workflows/ci-dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
index 984a9e7d6..9e666f776 100644
--- a/.github/workflows/ci-dev.yml
+++ b/.github/workflows/ci-dev.yml
@@ -46,6 +46,6 @@ jobs:
       - name: Run tests
         shell: bash -l {0}
         run: |
-          conda activate shapepipe
+          conda activate shapepipe-dev
           python setup.py test
           shapepipe_run -c example/config.ini

From 0737c0f2c5d35b1cf172432f604cc16607a03ec6 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 1 Dec 2023 10:14:31 +0000
Subject: [PATCH 035/133] trying to run Pi

---
 example/cfis/config_exp_Pi.ini             | 82 ++++++++++++++++++++++
 scripts/sh/init_run_sp_exclusive_canfar.sh | 25 +++++++
 2 files changed, 107 insertions(+)
 create mode 100644 example/cfis/config_exp_Pi.ini
 create mode 100755 scripts/sh/init_run_sp_exclusive_canfar.sh

diff --git a/example/cfis/config_exp_Pi.ini b/example/cfis/config_exp_Pi.ini
new file mode 100644
index 000000000..bf98c962b
--- /dev/null
+++ b/example/cfis/config_exp_Pi.ini
@@ -0,0 +1,82 @@
+# ShapePipe configuration file for single-HDU single exposure images..
+# PSFex PSF model; interpolation for validation.
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_exp_Pi
+
+# Add date and time to RUN_NAME, optional, default: True
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = psfex_interp_runner
+         
+
+# Run mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 8
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+[PSFEX_INTERP_RUNNER]
+
+# Use 20% sample for PSF validation
+FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat
+
+FILE_EXT = .psf, .fits, .cat
+
+NUMBERING_SCHEME = -0000000-0
+
+# Run mode for psfex interpolation:
+#  CLASSIC: 'classical' run, interpolate to object positions
+#  MULTI-EPOCH: interpolate for multi-epoch images
+#  VALIDATION: validation for single-epoch images
+MODE = VALIDATION
+
+# Column names of position parameters
+POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE
+
+# If True, measure and store ellipticity of the PSF (using moments)
+GET_SHAPES = True
+
+# Minimum number of stars per CCD for PSF model to be computed
+STAR_THRESH = 22
+
+# Maximum chi^2 for PSF model to be computed on CCD
+CHI2_THRESH = 2
diff --git a/scripts/sh/init_run_sp_exclusive_canfar.sh b/scripts/sh/init_run_sp_exclusive_canfar.sh
new file mode 100755
index 000000000..3d8e25c79
--- /dev/null
+++ b/scripts/sh/init_run_sp_exclusive_canfar.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+echo "start init run sp exclusive canfar"
+
+. /opt/conda/etc/profile.d/conda.sh
+
+conda activate shapepipe
+
+basedir=$HOME/cosmostat/P3_v2/psfex
+cd $basedir
+
+
+ID=$1
+n_SMP=$2
+typ=$3
+echo "ID=$ID n_SMP=$n_SMP type=$typ"
+
+export SP_RUN=.
+export SP_CONFIG=$HOME/shapepipe/example/cfis
+shapepipe_run -c $SP_CONFIG/config_exp_Pi.ini -e $ID
+
+cd $basedir
+
+echo "end init run sp exclusive canfar"
+

From c65d75a58c2451aea214f8c009e4822419b142d8 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 1 Dec 2023 11:39:52 +0100
Subject: [PATCH 036/133] numpy -> 1.22 to avoid asscalar bug

---
 environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/environment.yml b/environment.yml
index 759813a80..ea42dda00 100644
--- a/environment.yml
+++ b/environment.yml
@@ -15,7 +15,7 @@ dependencies:
   - libtool
   - matplotlib
   - numba
-  - numpy==1.21.6
+  - numpy==1.22
   - pip:
     - cs_util
     - modopt

From 2a7fab4824cf109933976d7bd428fb3af739fd26 Mon Sep 17 00:00:00 2001
From: sfarrens <samuel.farrens@gmail.com>
Date: Fri, 1 Dec 2023 13:11:59 +0100
Subject: [PATCH 037/133] added cs_util to dev env

---
 environment-dev.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/environment-dev.yml b/environment-dev.yml
index b5325d02c..cd2977ab8 100644
--- a/environment-dev.yml
+++ b/environment-dev.yml
@@ -28,6 +28,7 @@ dependencies:
   - tqdm>=4.66
   - treecorr>=4.3
   - pip:
+    - cs_util==0.0.5
     - mccd==1.2.3
     - modopt==1.6.0
     - sip_tpv==1.1

From 1e7b63359ec258dea8e3db6033fdb2897d7b149e Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 3 Dec 2023 17:36:53 +0000
Subject: [PATCH 038/133] aux script to create links for exposure output runs
 for tile

---
 example/cfis/config_exp_Pi.ini               |   2 +-
 example/cfis/config_exp_psfex.ini            |  31 +-
 example/cfis/config_tile_PiViSmVi_canfar.ini | 188 ++++++++++++
 scripts/python/link_to_exp_for_tile.py       | 305 +++++++++++++++++++
 scripts/sh/init_run_exclusive_canfar.sh      |  33 +-
 scripts/sh/job_sp_canfar.bash                |   2 +-
 6 files changed, 542 insertions(+), 19 deletions(-)
 create mode 100644 example/cfis/config_tile_PiViSmVi_canfar.ini
 create mode 100755 scripts/python/link_to_exp_for_tile.py

diff --git a/example/cfis/config_exp_Pi.ini b/example/cfis/config_exp_Pi.ini
index bf98c962b..ae510058e 100644
--- a/example/cfis/config_exp_Pi.ini
+++ b/example/cfis/config_exp_Pi.ini
@@ -46,7 +46,7 @@ OUTPUT_DIR = $SP_RUN/output
 [JOB]
 
 # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
-SMP_BATCH_SIZE = 8
+SMP_BATCH_SIZE = 2
 
 # Timeout value (optional), default is None, i.e. no timeout limit applied
 TIMEOUT = 96:00:00
diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini
index bfc46675f..178aaee58 100644
--- a/example/cfis/config_exp_psfex.ini
+++ b/example/cfis/config_exp_psfex.ini
@@ -9,7 +9,7 @@
 VERBOSE = True
 
 # Name of run (optional) default: shapepipe_run
-RUN_NAME = run_sp_exp_SxSePsf
+RUN_NAME = run_sp_exp_SxSePsfPi
 
 # Add date and time to RUN_NAME, optional, default: True
 ; RUN_DATETIME = False
@@ -19,7 +19,7 @@ RUN_NAME = run_sp_exp_SxSePsf
 [EXECUTION]
 
 # Module name, single string or comma-separated list of valid module runner names
-MODULE = sextractor_runner, setools_runner, psfex_runner
+MODULE = sextractor_runner, setools_runner, psfex_runner,psfex_interp_runner
          
 
 # Run mode, SMP or MPI
@@ -147,3 +147,30 @@ EXEC_PATH = psfex
 
 # Default psfex configuration file
 DOT_PSFEX_FILE = $SP_CONFIG/default.psfex
+
+[PSFEX_INTERP_RUNNER]
+
+# Use 20% sample for PSF validation
+FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat
+
+FILE_EXT = .psf, .fits, .cat
+
+NUMBERING_SCHEME = -0000000-0
+
+# Run mode for psfex interpolation:
+#  CLASSIC: 'classical' run, interpolate to object positions
+#  MULTI-EPOCH: interpolate for multi-epoch images
+#  VALIDATION: validation for single-epoch images
+MODE = VALIDATION
+
+# Column names of position parameters
+POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE
+
+# If True, measure and store ellipticity of the PSF (using moments)
+GET_SHAPES = True
+
+# Minimum number of stars per CCD for PSF model to be computed
+STAR_THRESH = 22
+
+# Maximum chi^2 for PSF model to be computed on CCD
+CHI2_THRESH = 2
diff --git a/example/cfis/config_tile_PiViSmVi_canfar.ini b/example/cfis/config_tile_PiViSmVi_canfar.ini
new file mode 100644
index 000000000..67dfa153e
--- /dev/null
+++ b/example/cfis/config_tile_PiViSmVi_canfar.ini
@@ -0,0 +1,188 @@
+# ShapePipe configuration file for tile, from detection  up to shape measurement.
+# PSFEx PSF model.
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_tile_PsViSmVi
+
+# Add date and time to RUN_NAME, optional, default: False
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+#MODULE = psfex_interp_runner,
+
+MODULE = psfex_interp_runner, vignetmaker_runner, spread_model_runner,
+         vignetmaker_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 16
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+[PSFEX_INTERP_RUNNER]
+
+INPUT_DIR = last:sextractor_runner
+
+FILE_PATTERN = sexcat
+
+FILE_EXT = .fits
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# Run mode for psfex interpolation:
+#  CLASSIC: 'classical' run, interpolate to object positions
+#  MULTI-EPOCH: interpolate for multi-epoch images
+#  VALIDATION: validation for single-epoch images
+MODE = MULTI-EPOCH
+
+# Column names of position parameters
+POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD
+
+# If True, measure and store ellipticity of the PSF
+GET_SHAPES = True
+
+# Number of stars threshold
+STAR_THRESH = 20
+
+# chi^2 threshold
+CHI2_THRESH = 2
+
+# Multi-epoch mode parameters
+
+ME_DOT_PSF_DIR = psfex_runner
+
+# Input psf file pattern
+ME_DOT_PSF_PATTERN = star_split_ratio_80
+
+# Multi-epoch mode: Path to file with single-exposure WCS header information
+ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite
+
+
+# Create vignets for tiles weights
+[VIGNETMAKER_RUNNER_RUN_1]
+
+INPUT_DIR = last:sextractor_runner, last:uncompress_fits_runner
+
+FILE_PATTERN = sexcat, CFIS_weight
+
+FILE_EXT = .fits, .fits
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+MASKING = False
+MASK_VALUE = 0
+
+# Run mode for psfex interpolation:
+#  CLASSIC: 'classical' run, interpolate to object positions
+#  MULTI-EPOCH: interpolate for multi-epoch images
+#  VALIDATION: validation for single-epoch images
+MODE = CLASSIC
+
+# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates)
+COORD = PIX
+POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE
+
+# Vignet size in pixels
+STAMP_SIZE = 51
+
+# Output file name prefix, file name is <PREFIX>_vignet.fits
+PREFIX = weight
+
+
+[SPREAD_MODEL_RUNNER]
+
+INPUT_DIR = last:sextractor_runner, last:psfex_interp_runner, last:vignetmaker_runner_run_1
+
+FILE_PATTERN = sexcat, galaxy_psf, weight_vignet
+
+FILE_EXT = .fits, .sqlite, .fits
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# Pixel scale in arcsec
+PIXEL_SCALE = 0.186
+
+# Output mode:
+#  new: create a new catalog with: [number, mag, sm, sm_err]
+#  add: create a copy of the input SExtractor with the column sm and sm_err
+OUTPUT_MODE = new
+
+
+[VIGNETMAKER_RUNNER_RUN_2]
+
+# Create multi-epoch vignets for tiles corresponding to
+# positions on single-exposures
+
+INPUT_DIR = last:sextractor_runner
+
+FILE_PATTERN = sexcat
+
+FILE_EXT = .fits
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+MASKING = False
+MASK_VALUE = 0
+
+# Run mode for psfex interpolation:
+#  CLASSIC: 'classical' run, interpolate to object positions
+#  MULTI-EPOCH: interpolate for multi-epoch images
+#  VALIDATION: validation for single-epoch images
+MODE = MULTI-EPOCH
+
+# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates)
+COORD = SPHE
+POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD
+
+# Vignet size in pixels
+STAMP_SIZE = 51
+
+# Output file name prefix, file name is <PREFIX_>vignet.fits
+PREFIX =
+
+# Additional parameters for path and file pattern corresponding to single-exposure
+# run outputs
+ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_2
+ME_IMAGE_PATTERN = flag, image, weight, background
+ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite
diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
new file mode 100755
index 000000000..40d74e352
--- /dev/null
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -0,0 +1,305 @@
+#!/usr/bin/env python
+
+# -*- coding: utf-8 -*-
+
+"""Script link_to_esp_for_tile.py
+
+:Author: Martin Kilbinger
+
+"""
+
+import os
+import sys
+import re
+import copy
+
+from optparse import OptionParser
+
+
+class param:
+    """General class to store (default) variables
+
+    """
+    def __init__(self, **kwds):
+        self.__dict__.update(kwds)
+
+    def print(self, **kwds):
+        print(self.__dict__)
+
+    def var_list(self, **kwds):
+        return vars(self)
+
+
+def params_default():
+    """Params Default.
+    
+    Set default parameter values.
+
+    Returns
+    -------
+    class param
+        parameter values
+
+    """
+    p_def = param(
+        tile_base_dir  = '.',
+        exp_base_dir = '.',
+    )
+
+    return p_def
+
+
+def parse_options(p_def):
+    """Parse Options.
+    
+    Parse command line options.
+
+    Parameters
+    ----------
+    p_def: class param
+        parameter values
+
+    Returns
+    -------
+    list
+        command line options
+        command line str
+
+    """
+    usage  = "%prog [OPTIONS]"
+    parser = OptionParser(usage=usage)
+
+    # IO
+    parser.add_option(
+        '-i',
+        '--input_tile_dir',
+        dest='tile_base_dir',
+        type='string',
+        default=p_def.tile_base_dir,
+        help=f'input tile base directory, default=\'{p_def.tile_base_dir}\''
+    )
+    parser.add_option(
+        '-t',
+        '--tile_ID',
+        dest='tile_ID',
+        type='string',
+        help=f"input tile ID",
+    )
+    parser.add_option(
+        '-I',
+        '--input_exp_dir',
+        dest='exp_base_dir',
+        type='string',
+        default=p_def.exp_base_dir,
+        help=f'input exposure base directory, default=\'{p_def.exp_base_dir}\''
+    )
+    parser.add_option(
+        '-v',
+        '--verbose',
+        dest='verbose',
+        action='store_true',
+        help='verbose output'
+    )
+
+    options, args = parser.parse_args()
+
+    return options, args
+
+
+def check_options(options):
+    """Check Options.
+    
+    Check command line options.
+
+    Parameters
+    ----------
+    options: tuple
+        Command line options
+
+    Returns
+    -------
+    bool
+        Result of option check. False if invalid option value.
+
+    """
+    return True
+
+
+def update_param(p_def, options):
+    """Update Param.
+    
+    Return default parameter, updated and complemented according to options.
+
+    Parameters
+    ----------
+    p_def:  class param
+        parameter values
+    optiosn: tuple
+        command line options
+
+    Returns
+    -------
+    class param
+        updated paramter values
+
+    """
+    param = copy.copy(p_def)
+
+    # Update keys in param according to options values
+    for key in vars(param):
+        if key in vars(options):
+            setattr(param, key, getattr(options, key))
+
+    # Add remaining keys from options to param
+    for key in vars(options):
+        if not key in vars(param):
+            setattr(param, key, getattr(options, key))
+
+    # Do extra stuff if necessary
+
+    return param
+
+
+def matching_subdirs(base_dir, pattern):
+
+    # Find all matching subdirectories
+    subdirs = []
+    for entry in os.listdir(base_dir):
+        full_path = os.path.join(base_dir, entry)
+        if os.path.isdir(full_path) and entry.startswith(pattern):
+            subdirs.append(full_path)
+
+    # Sort according to creation date
+    subdirs.sort(key=os.path.getctime)
+
+    return subdirs
+
+
+def get_tile_out_dir(tile_base_dir, tile_ID):
+
+    tile_out_dir = f"{tile_base_dir}/{tile_ID}/output"
+
+    return tile_out_dir
+
+
+def get_exp_IDs(tile_base_dir, tile_ID, verbose=False):
+
+    tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID)
+
+    subdirs = matching_subdirs(tile_out_dir, "run_sp_GitFeGie")
+
+    if len(subdirs) != 1:
+        raise IOError(f"Exactly one matching directory in {tile_out_dir} expected, not {len(subdirs)}")
+
+    # Replace dot with dash in tile ID
+    tile_ID_sp = re.sub(r"\.", "-", tile_ID)
+    exp_ID_file = f"{subdirs[0]}/find_exposures_runner/output/exp_numbers-{tile_ID_sp}.txt"
+
+    exp_IDs = []
+    with open(exp_ID_file) as f_in:
+        for line in f_in:
+            name = line.strip()
+            # Remove any letter
+            ID = re.sub("[a-zA-Z]", "", name) 
+            exp_IDs.append(ID)
+
+    return exp_IDs
+
+
+def get_exp_single_HDU_IDs(exp_IDs, n_CPU):
+
+    exp_shdu_IDs = []
+    for exp_ID in exp_IDs:
+        for idx in range(n_CPU):
+            ID = f"{exp_ID}-{idx}"
+            exp_shdu_IDs.append(ID)
+
+    return exp_shdu_IDs
+
+    
+def get_paths(exp_base_dir, exp_shdu_IDs):
+
+    number = {}
+    paths = []
+    for exp_shdu_ID in exp_shdu_IDs:
+        name = f"{exp_base_dir}/{exp_shdu_ID}/output"
+        path = os.path.abspath(name)
+        subdirs = matching_subdirs(path, "run_sp_exp_SxSePsf")
+        n_subdirs = len(subdirs)
+
+        if n_subdirs not in number:
+            number[n_subdirs] = 1
+        else:
+            number[n_subdirs] += 1
+
+        if n_subdirs != 1:
+            msg = f"Exactly one matching directory in {path} expected, not {n_subdirs}"
+            print(msg)
+            if n_subdirs == 0:
+                continue
+        paths.append(f"{subdirs[0]}")
+
+    return paths, number
+
+
+def create_links_paths(tile_base_dir, tile_ID, paths):
+
+    tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID)
+
+    for path in paths:
+
+        head, tail = os.path.split(path)
+        src = path
+        dst = f"{tile_out_dir}/{tail}"
+        if os.path.exists(dst):
+            print("Warning: {dst} already exists, no link created")
+        else:
+            print(f"ln -s {src} {dst}")
+            os.symlink(src, dst)
+
+
+def update_log_files(tile_base_dir, tile_ID, paths):
+
+   tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID)
+
+   for path in paths:
+
+        head, tail = os.path.split(path)
+        src = path
+        dst = f"{tile_out_dir}/{tail}"
+            os.symlink(src, dst)
+
+
+
+def main(argv=None):
+
+    # Set default parameters
+    p_def = params_default()
+
+    # Command line options
+    options, args = parse_options(p_def)
+
+    if check_options(options) is False:
+        return 1
+
+    param = update_param(p_def, options)
+
+    tile_base_dir = param.tile_base_dir
+    exp_base_dir = param.exp_base_dir
+    tile_ID = param.tile_ID
+    n_CPU = 40
+    verbose = param.verbose
+
+    exp_IDs = get_exp_IDs(tile_base_dir, tile_ID, verbose=verbose)
+    exp_shdu_IDs = get_exp_single_HDU_IDs(exp_IDs, n_CPU)
+    paths, number = get_paths(exp_base_dir, exp_shdu_IDs)
+    print(number)
+
+    create_links_paths(tile_base_dir, tile_ID, paths)
+
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 7f9f428e1..42e8f5824 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -20,22 +20,25 @@ echo "ID=$ID n_SMP=$n_SMP type=$typ"
 
 cd ${typ}_runs
 
-mkdir $ID
+#mkdir $ID
+
 cd $ID
-mkdir output
-cd output
-cp $basedir/output/log_run_sp.txt .
-ln -s $basedir/output/log_exp_headers.sqlite
-for dir in $basedir/output/run_sp_*; do
-	ln -s $dir
-done
-cd ..
-
-#export SP_RUN=.
-#export SP_CONFIG=$HOME/shapepipe/example/cfis
-#shapepipe_run -c $SP_CONFIG/config_tile_Sx.ini -e $ID
-
-job_sp_canfar.bash -p psfex -j 32 -e $ID -n $n_SMP
+
+#mkdir output
+#cd output
+pwd
+#cp $basedir/output/log_run_sp.txt .
+#ln -s $basedir/output/log_exp_headers.sqlite
+#for dir in $basedir/output/run_sp_*; do
+	#ln -s $dir
+#done
+#cd ..
+
+export SP_RUN=.
+export SP_CONFIG=$HOME/shapepipe/example/cfis
+shapepipe_run -c $SP_CONFIG/config_exp_Pi.ini -e $ID
+
+#job_sp_canfar.bash -p psfex -j 32 -e $ID -n $n_SMP
 
 cd $basedir
 
diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
index cbafd6f19..621700f9a 100755
--- a/scripts/sh/job_sp_canfar.bash
+++ b/scripts/sh/job_sp_canfar.bash
@@ -421,7 +421,7 @@ if [[ $do_job != 0 ]]; then
   ### PSF model letter: 'P' (psfex) or 'M' (mccd)
   letter=${psf:0:1}
   Letter=${letter^}
-  command_sp "shapepipe_run -c $SP_CONFIG/config_tile_${Letter}iViSmVi.ini" "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)"
+  command_cfg_shapepipe "config_tile_${Letter}iViSmVi_canfar.ini" "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)" $n_smp $exclusive
 
 fi
 

From 85f0c99ad3d1d2433455a0d5124ab699e000636f Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 3 Dec 2023 18:58:42 +0100
Subject: [PATCH 039/133] Added aux script to update runs log file

---
 scripts/python/link_to_exp_for_tile.py |  1 +
 scripts/python/update_runs_log_file.py | 80 ++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 scripts/python/update_runs_log_file.py

diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index 40d74e352..64e83995b 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -160,6 +160,7 @@ class param
     return param
 
 
+# TODO: move to cs_util                                                          
 def matching_subdirs(base_dir, pattern):
 
     # Find all matching subdirectories
diff --git a/scripts/python/update_runs_log_file.py b/scripts/python/update_runs_log_file.py
new file mode 100644
index 000000000..c371eedfd
--- /dev/null
+++ b/scripts/python/update_runs_log_file.py
@@ -0,0 +1,80 @@
+import sys
+import os
+
+# TODO: move to cs_util
+def matching_subdirs(base_dir, pattern, where="start", tail=False):                                         
+                                                                                 
+    # Find all matching subdirectories                                           
+    subdirs = []                                                                 
+    for entry in os.listdir(base_dir):                                           
+        full_path = os.path.join(base_dir, entry)                                
+        if os.path.isdir(full_path):
+            found = False
+
+            # Look for pattern at start or end
+            if where == "start" and entry.startswith(pattern):               
+                found = True
+            if where == "end" and entry.endswith(pattern):
+                found = True
+
+            # Append for return
+            if found:
+                if not tail:
+                    path = full_path
+                else:
+                    head, tail = os.path.split(full_path) 
+                    path = tail
+                subdirs.append(path)
+                                                                                 
+    # Sort according to creation date                                            
+    if not tail:
+        subdirs.sort(key=os.path.getctime)                                           
+                                                                                 
+    return subdirs
+
+
+def get_module_runs(subdirs):
+
+    all_runs = {}
+    for subdir in subdirs:
+        runs = matching_subdirs(subdir, "_runner", where="end", tail=True)
+        all_runs[subdir] = runs
+
+    return all_runs
+
+
+def update_log_file(module_runs, log_name):
+
+    with open(log_name, "w") as f_out: 
+        for key in module_runs:
+            print(key, file=f_out, end=" ")
+            print(",".join(module_runs[key]), file=f_out)
+
+
+def main(argv=None):                                                             
+                                                                                 
+    # Set default parameters                                                     
+    #p_def = params_default()                                                     
+                                                                                 
+    # Command line options                                                       
+    #options, args = parse_options(p_def)                                         
+                                                                                 
+    #if check_options(options) is False:                                          
+        #return 1                                                                 
+                                                                                 
+    #param = update_param(p_def, options)                                         
+                                                                                 
+    base_dir = "./output"
+    pattern = "run_sp_"
+    log_name = f"{base_dir}/log_run_sp.txt"
+
+    subdirs = matching_subdirs(base_dir, pattern)
+    module_runs = get_module_runs(subdirs)
+    #save_prev(log_name)
+    update_log_file(module_runs, log_name)
+                                                                                 
+    return 0                                                                     
+                                                                                 
+                                                                                 
+if __name__ == "__main__":                                                       
+    sys.exit(main(sys.argv))

From d0c3d9fb9a091aac5765539f47894989079b8f15 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 3 Dec 2023 19:23:16 +0100
Subject: [PATCH 040/133] update runs log file script: deal with multiple runs
 of same module

---
 scripts/python/update_runs_log_file.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/scripts/python/update_runs_log_file.py b/scripts/python/update_runs_log_file.py
index c371eedfd..0ee5870ff 100644
--- a/scripts/python/update_runs_log_file.py
+++ b/scripts/python/update_runs_log_file.py
@@ -1,8 +1,9 @@
+import re
 import sys
 import os
 
 # TODO: move to cs_util
-def matching_subdirs(base_dir, pattern, where="start", tail=False):                                         
+def matching_subdirs(base_dir, pattern, tail=False):                                         
                                                                                  
     # Find all matching subdirectories                                           
     subdirs = []                                                                 
@@ -12,18 +13,19 @@ def matching_subdirs(base_dir, pattern, where="start", tail=False):
             found = False
 
             # Look for pattern at start or end
-            if where == "start" and entry.startswith(pattern):               
-                found = True
-            if where == "end" and entry.endswith(pattern):
-                found = True
+            if pattern in entry:               
 
-            # Append for return
-            if found:
+                # Get full path or last part ("tail")
                 if not tail:
                     path = full_path
                 else:
                     head, tail = os.path.split(full_path) 
                     path = tail
+
+                # Remove postfix in case of multiple runs of same module
+                path = re.sub("_run_.\d?", "", path)
+
+                # Append to result
                 subdirs.append(path)
                                                                                  
     # Sort according to creation date                                            
@@ -37,7 +39,7 @@ def get_module_runs(subdirs):
 
     all_runs = {}
     for subdir in subdirs:
-        runs = matching_subdirs(subdir, "_runner", where="end", tail=True)
+        runs = matching_subdirs(subdir, "_runner", tail=True)
         all_runs[subdir] = runs
 
     return all_runs

From 323d44a97e024cc2f78c13c56a2c221d243e07ab Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 4 Dec 2023 11:53:28 +0100
Subject: [PATCH 041/133] run_log: added function get_all_dirs

---
 scripts/sh/curl_canfar_local.sh |  2 +-
 shapepipe/pipeline/run_log.py   | 35 +++++++++++++++++++++++++++++----
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index c875d4cb3..e410c8d30 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -4,7 +4,7 @@
 # ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.8 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID exp NCORE
 
 SSL=~/.ssl/cadcproxy.pem
-NCORE=1
+NCORE=2
 SESSION=https://ws-uv.canfar.net/skaha/v0/session
 RESOURCES="ram=4&cores=$NCORE"
 IMAGE=images.canfar.net/unions/shapepipe
diff --git a/shapepipe/pipeline/run_log.py b/shapepipe/pipeline/run_log.py
index 115e87de0..d0f9af608 100644
--- a/shapepipe/pipeline/run_log.py
+++ b/shapepipe/pipeline/run_log.py
@@ -170,6 +170,32 @@ def get_last(runs, module):
     return last_run.split(' ')[0]
 
 
+def get_all_dirs(run_log_file, module):
+    """Get All Dirs.
+
+    Return directory paths corresponding to all runs of given module.
+
+    Parameters
+    ----------
+    run_log_file : str
+        Run log file name
+    module : str
+        Module name
+
+    Returns
+    -------
+    list
+        Directory names of all module runs
+
+    """
+    runs = get_list(run_log_file)
+    all_runs = get_all(runs, module)
+
+    all_dirs = [f"{run}/{module}/output" for run in all_runs[0].split(" ")]
+ 
+    return all_dirs
+
+
 def get_last_dir(run_log_file, module):
     """Get Last Dir.
 
@@ -188,9 +214,10 @@ def get_last_dir(run_log_file, module):
         Directory name of last module run
 
     """
-    runs = get_list(run_log_file)
-    all_runs = get_all(runs, module)
-    last_run = all_runs[0].split(' ')[0]
-    last_dir = f'{last_run}/{module}/output'
+    all_dirs = get_all_dirs(run_log_file, module)
+    last_dir = all_dirs[0]
 
     return last_dir
+
+
+

From dedbc3f7017e93953a4f4ba4c7e6f8ef835aa559 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 4 Dec 2023 13:22:31 +0100
Subject: [PATCH 042/133] Fixed new function get_all_dirs

---
 scripts/python/link_to_exp_for_tile.py | 13 -------------
 shapepipe/pipeline/run_log.py          |  5 ++++-
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index 64e83995b..55fc09d03 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -259,19 +259,6 @@ def create_links_paths(tile_base_dir, tile_ID, paths):
             os.symlink(src, dst)
 
 
-def update_log_files(tile_base_dir, tile_ID, paths):
-
-   tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID)
-
-   for path in paths:
-
-        head, tail = os.path.split(path)
-        src = path
-        dst = f"{tile_out_dir}/{tail}"
-            os.symlink(src, dst)
-
-
-
 def main(argv=None):
 
     # Set default parameters
diff --git a/shapepipe/pipeline/run_log.py b/shapepipe/pipeline/run_log.py
index d0f9af608..c182eeac7 100644
--- a/shapepipe/pipeline/run_log.py
+++ b/shapepipe/pipeline/run_log.py
@@ -191,7 +191,10 @@ def get_all_dirs(run_log_file, module):
     runs = get_list(run_log_file)
     all_runs = get_all(runs, module)
 
-    all_dirs = [f"{run}/{module}/output" for run in all_runs[0].split(" ")]
+    all_dirs = []
+    for run in all_runs:
+        dir_name = run.split(" ")[0]
+        all_dirs.append(f"{dir_name}/{module}/output")
  
     return all_dirs
 

From e67f549889f23ce92ba60efc631b40f102ce8f26 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 4 Dec 2023 12:24:53 +0000
Subject: [PATCH 043/133] Fixed (as for MCCD) FITS key bug

---
 example/cfis/config_tile_PiViSmVi_canfar.ini  | 10 +++----
 scripts/python/link_to_exp_for_tile.py        | 28 ++++++-------------
 .../psfex_interp_package/psfex_interp.py      |  8 ++++--
 shapepipe/modules/vignetmaker_runner.py       |  6 ++--
 shapepipe/pipeline/file_handler.py            |  2 +-
 5 files changed, 24 insertions(+), 30 deletions(-)

diff --git a/example/cfis/config_tile_PiViSmVi_canfar.ini b/example/cfis/config_tile_PiViSmVi_canfar.ini
index 67dfa153e..ed9e3d339 100644
--- a/example/cfis/config_tile_PiViSmVi_canfar.ini
+++ b/example/cfis/config_tile_PiViSmVi_canfar.ini
@@ -58,7 +58,7 @@ TIMEOUT = 96:00:00
 
 [PSFEX_INTERP_RUNNER]
 
-INPUT_DIR = last:sextractor_runner
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner
 
 FILE_PATTERN = sexcat
 
@@ -99,7 +99,7 @@ ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite
 # Create vignets for tiles weights
 [VIGNETMAKER_RUNNER_RUN_1]
 
-INPUT_DIR = last:sextractor_runner, last:uncompress_fits_runner
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:uncompress_fits_runner
 
 FILE_PATTERN = sexcat, CFIS_weight
 
@@ -130,7 +130,7 @@ PREFIX = weight
 
 [SPREAD_MODEL_RUNNER]
 
-INPUT_DIR = last:sextractor_runner, last:psfex_interp_runner, last:vignetmaker_runner_run_1
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:psfex_interp_runner, last:vignetmaker_runner_run_1
 
 FILE_PATTERN = sexcat, galaxy_psf, weight_vignet
 
@@ -153,7 +153,7 @@ OUTPUT_MODE = new
 # Create multi-epoch vignets for tiles corresponding to
 # positions on single-exposures
 
-INPUT_DIR = last:sextractor_runner
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner
 
 FILE_PATTERN = sexcat
 
@@ -183,6 +183,6 @@ PREFIX =
 
 # Additional parameters for path and file pattern corresponding to single-exposure
 # run outputs
-ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner_run_2
+ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner
 ME_IMAGE_PATTERN = flag, image, weight, background
 ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite
diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index 64e83995b..de18f92e8 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -218,14 +218,14 @@ def get_exp_single_HDU_IDs(exp_IDs, n_CPU):
     return exp_shdu_IDs
 
     
-def get_paths(exp_base_dir, exp_shdu_IDs):
+def get_paths(exp_base_dir, exp_shdu_IDs, pattern):
 
     number = {}
     paths = []
     for exp_shdu_ID in exp_shdu_IDs:
         name = f"{exp_base_dir}/{exp_shdu_ID}/output"
         path = os.path.abspath(name)
-        subdirs = matching_subdirs(path, "run_sp_exp_SxSePsf")
+        subdirs = matching_subdirs(path, pattern)
         n_subdirs = len(subdirs)
 
         if n_subdirs not in number:
@@ -253,25 +253,12 @@ def create_links_paths(tile_base_dir, tile_ID, paths):
         src = path
         dst = f"{tile_out_dir}/{tail}"
         if os.path.exists(dst):
-            print("Warning: {dst} already exists, no link created")
+            print(f"Warning: {dst} already exists, no link created")
         else:
             print(f"ln -s {src} {dst}")
             os.symlink(src, dst)
 
 
-def update_log_files(tile_base_dir, tile_ID, paths):
-
-   tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID)
-
-   for path in paths:
-
-        head, tail = os.path.split(path)
-        src = path
-        dst = f"{tile_out_dir}/{tail}"
-            os.symlink(src, dst)
-
-
-
 def main(argv=None):
 
     # Set default parameters
@@ -293,10 +280,13 @@ def main(argv=None):
 
     exp_IDs = get_exp_IDs(tile_base_dir, tile_ID, verbose=verbose)
     exp_shdu_IDs = get_exp_single_HDU_IDs(exp_IDs, n_CPU)
-    paths, number = get_paths(exp_base_dir, exp_shdu_IDs)
-    print(number)
 
-    create_links_paths(tile_base_dir, tile_ID, paths)
+    patterns = ["run_sp_exp_SxSePsf", "run_sp_exp_Pi"]
+    for pattern in patterns:
+        paths, number = get_paths(exp_base_dir, exp_shdu_IDs, pattern)
+        print(number)
+
+        create_links_paths(tile_base_dir, tile_ID, paths)
 
 
     return 0
diff --git a/shapepipe/modules/psfex_interp_package/psfex_interp.py b/shapepipe/modules/psfex_interp_package/psfex_interp.py
index 3ac1b25ea..ca7d6bea3 100644
--- a/shapepipe/modules/psfex_interp_package/psfex_interp.py
+++ b/shapepipe/modules/psfex_interp_package/psfex_interp.py
@@ -568,13 +568,15 @@ def _interpolate_me(self):
 
         all_id = np.copy(cat.get_data()['NUMBER'])
         key_ne = 'N_EPOCH'
-        if key_ne not in cat.get_data():
+        if key_ne not in cat.get_data().dtype.names:
+            print("MKDEBUG ", cat.get_data())
             raise KeyError(
-                f'Key {key_ne} not found in input galaxy catalogue, needed for'
+                f'Key {key_ne} not found in input galaxy catalogue'
+                + f'{self._galcat_path}, needed for'
                 + ' PSF interpolation to multi-epoch data; run previous module'
                 + ' (SExtractor) in multi-epoch mode'
             )
-        n_epoch = np.copy(cat.get_data()[key_me])
+        n_epoch = np.copy(cat.get_data()[key_ne])
 
         list_ext_name = cat.get_ext_name()
         hdu_ind = [
diff --git a/shapepipe/modules/vignetmaker_runner.py b/shapepipe/modules/vignetmaker_runner.py
index 972e54e16..afd6ed66b 100644
--- a/shapepipe/modules/vignetmaker_runner.py
+++ b/shapepipe/modules/vignetmaker_runner.py
@@ -93,8 +93,10 @@ def vignetmaker_runner(
             modules = config.getlist(module_config_sec, 'ME_IMAGE_DIR')
             image_dir = []
             for module in modules:
-                last_dir = get_last_dir(run_dirs['run_log'], module)
-                image_dir.append(last_dir)
+                #last_dir = get_last_dir(run_dirs['run_log'], module)
+                all_dirs = get_all_dirs(run_dirs['run_log'], module)
+                for my_dir in all_dirs:
+                    image_dir.append(my_dir)
             image_pattern = config.getlist(
                 module_config_sec,
                 'ME_IMAGE_PATTERN',
diff --git a/shapepipe/pipeline/file_handler.py b/shapepipe/pipeline/file_handler.py
index 51b02cdd7..6ce2192a1 100644
--- a/shapepipe/pipeline/file_handler.py
+++ b/shapepipe/pipeline/file_handler.py
@@ -1118,7 +1118,7 @@ def _format_process_list(
                 id_to_test = f"-{self._exclusive.replace('.', '-')}"
                 if number == id_to_test:
                     if self._verbose:
-                        print(f"Using exclusive number {self._exclusive} ({id_to_test})")
+                        print(f"-- Using exclusive number {self._exclusive} ({id_to_test})")
                 else:
                     if self._verbose:
                         #print(f"Skipping {number}, not equal to {self._exclusive} ({id_to_test})")

From b6cb27d788347a72a2926434872acc13d8caea26 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 4 Dec 2023 15:15:17 +0000
Subject: [PATCH 044/133] vignet makers: can use last and all in additional
 input inage directories

---
 .../vignetmaker_package/vignetmaker.py        | 56 +++++++++++--------
 shapepipe/modules/vignetmaker_runner.py       | 23 +++++---
 2 files changed, 47 insertions(+), 32 deletions(-)

diff --git a/shapepipe/modules/vignetmaker_package/vignetmaker.py b/shapepipe/modules/vignetmaker_package/vignetmaker.py
index 047de8a6b..a7e2b2857 100644
--- a/shapepipe/modules/vignetmaker_package/vignetmaker.py
+++ b/shapepipe/modules/vignetmaker_package/vignetmaker.py
@@ -7,6 +7,7 @@
 """
 
 import re
+import os
 
 import numpy as np
 from astropy.wcs import WCS
@@ -183,22 +184,22 @@ def _get_stamp(self, img_path, pos, rad):
 
         return vign
 
-    def _get_stamp_me(self, image_dir, image_pattern):
+    def _get_stamp_me(self, image_dirs, image_pattern):
         """Get Stamp Multi-Epoch.
 
         Get stamps for multi-epoch data.
 
         Parameters
         ----------
-        image_dir : str
-            Path to the directory where the images are
+        image_dirs : list
+            Path to directories to search for input images
         image_pattern : str
             Common part of the file names
 
         Returns
         -------
         dict
-            Directory containing object id and vignets for each epoch
+            Dictionary containing object id and vignets for each epoch
 
         """
         cat = file_io.FITSCatalogue(self._galcat_path, SEx_catalogue=True)
@@ -226,10 +227,19 @@ def _get_stamp_me(self, image_dir, image_pattern):
                 if ccd == -1:
                     continue
 
-                img_path = (
-                    image_dir + '/' + image_pattern + '-'
-                    + exp_name + '-' + str(ccd) + '.fits'
-                )
+                # Look for input image
+                found = False
+                image_name = f"{image_pattern}-{exp_name}-{ccd}.fits"
+                for image_dir in image_dirs:
+                    img_path = f"{image_dir}/{image_name}"
+                    if os.path.exists(img_path):
+                        found = True
+                        break
+                if not found:
+                    raise FileNotFoundError(
+                        f"Could not find image {image_name}"
+                    )
+
                 ind_obj = np.where(cat.get_data(hdu_index)['CCD_N'] == ccd)[0]
                 obj_id = all_id[ind_obj]
 
@@ -292,23 +302,23 @@ def _get_stamp_me(self, image_dir, image_pattern):
 
         return output_dict
 
-    def process_me(self, image_dir, image_pattern, f_wcs_path, rad):
+    def process_me(self, image_dirs, image_pattern, f_wcs_path, rad):
         """Process Multi-Epoch.
 
         Main function to create the stamps in the multi-epoch case.
 
         Parameters
         ----------
-        image_dir : list
-            List of directories where the image are; ff ``len(image_dir) == 1``
-            -> all images are in the same directory, else ``len(image_dir)``
-            must match ``len(image_pattern)``
+        image_dirs : list
+            Directories of image locations.
+            Each list item contains sublist in which images are searched.
+            Length of outer list has to match image_pattern, or be single item.
         image_pattern : list
             Common part of each kind of file names
         f_wcs_path : str
             Path to the log file containing the WCS for each CCDs
         rad : int
-            Radius of the stamp, must be odd
+            Radius of the stamp, must be an odd integer
 
         """
         self._f_wcs_file = SqliteDict(f_wcs_path)
@@ -316,17 +326,15 @@ def process_me(self, image_dir, image_pattern, f_wcs_path, rad):
 
         for idx in range(len(image_pattern)):
 
-            if len(image_dir) != len(image_pattern):
-                output_dict = self._get_stamp_me(
-                    image_dir[0],
-                    image_pattern[idx],
-                )
-
+            if len(image_dirs) != len(image_pattern):
+                index = 0
             else:
-                output_dict = self._get_stamp_me(
-                    image_dir[idx],
-                    image_pattern[idx],
-                )
+                index = idx
+
+            output_dict = self._get_stamp_me(
+                image_dirs[index],
+                image_pattern[idx],
+            )
 
             self._save_vignet_me(output_dict, image_pattern[idx])
 
diff --git a/shapepipe/modules/vignetmaker_runner.py b/shapepipe/modules/vignetmaker_runner.py
index afd6ed66b..9a83f1cd6 100644
--- a/shapepipe/modules/vignetmaker_runner.py
+++ b/shapepipe/modules/vignetmaker_runner.py
@@ -2,14 +2,14 @@
 
 Module runner for ``vignetmaker``.
 
-:Author: Axel Guinot
+:Authors: Axel Guinot, Martin Kilbinger
 
 """
 
 from shapepipe.modules.module_decorator import module_runner
 from shapepipe.modules.vignetmaker_package import vignetmaker as vm
 
-from shapepipe.pipeline.run_log import get_last_dir
+from shapepipe.pipeline.run_log import get_last_dir, get_all_dirs
 
 
 @module_runner(
@@ -91,12 +91,19 @@ def vignetmaker_runner(
         elif mode == 'MULTI-EPOCH':
             # Fetch image directory and patterns
             modules = config.getlist(module_config_sec, 'ME_IMAGE_DIR')
-            image_dir = []
+            image_dirs = []
             for module in modules:
-                #last_dir = get_last_dir(run_dirs['run_log'], module)
-                all_dirs = get_all_dirs(run_dirs['run_log'], module)
-                for my_dir in all_dirs:
-                    image_dir.append(my_dir)
+                module_name = module.split(":")[-1]
+                if "last" in module:
+                    dirs = [get_last_dir(run_dirs['run_log'], module_name)]
+                elif "all" in module:
+                    dirs = get_all_dirs(run_dirs['run_log'], module_name)
+                else:
+                    raise ValueError(
+                        "Expected qualifier 'last:' or 'all' before module"
+                        + f" '{module}' in config entry 'ME_IMAGE_DIR'")
+                image_dirs.append(dirs)
+
             image_pattern = config.getlist(
                 module_config_sec,
                 'ME_IMAGE_PATTERN',
@@ -105,7 +112,7 @@ def vignetmaker_runner(
             f_wcs_path = config.getexpanded(module_config_sec, 'ME_LOG_WCS')
 
             # Process inputs
-            vm_inst.process_me(image_dir, image_pattern, f_wcs_path, radius)
+            vm_inst.process_me(image_dirs, image_pattern, f_wcs_path, radius)
 
         # Invalid mode
         else:

From 096af60f2debf296d30dca8885ae72f852af7379 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 8 Dec 2023 09:30:12 +0100
Subject: [PATCH 045/133] Dockerfile + jupyter, activate

---
 Dockerfile                        | 4 +++-
 scripts/python/merge_final_cat.py | 2 --
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index d4ea036d8..6cd1923e7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -54,4 +54,6 @@ COPY scripts ./scripts
 # Activate conda environment using "source"
 #RUN . /opt/conda/etc/profile.d/conda.sh
 #RUN conda init bash
-RUN conda activate shapepipe
+RUN ./scripts/sh/init_canfar.sh
+RUN source activate shapepipe
+RUN pip install jupyter
diff --git a/scripts/python/merge_final_cat.py b/scripts/python/merge_final_cat.py
index 1a160c449..076f5c935 100755
--- a/scripts/python/merge_final_cat.py
+++ b/scripts/python/merge_final_cat.py
@@ -20,8 +20,6 @@
 
 from optparse import OptionParser
 
-from tqdm import tqdm
-
 from shapepipe.utilities import cfis
 
 

From 3c372506c39bb41493f54aaa20e66f386f26aef6 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 8 Dec 2023 13:41:52 +0000
Subject: [PATCH 046/133] file handler raises error if no process

---
 shapepipe/pipeline/file_handler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/shapepipe/pipeline/file_handler.py b/shapepipe/pipeline/file_handler.py
index 6ce2192a1..66baf12e6 100644
--- a/shapepipe/pipeline/file_handler.py
+++ b/shapepipe/pipeline/file_handler.py
@@ -1135,6 +1135,9 @@ def _format_process_list(
             ])
             process_list.append(process_items)
 
+        if len(process_list) == 0:
+            raise ValueError("Empty process list")
+
         return process_list
 
     def _save_process_list(

From 667eb7bd871b949882a0fdf256323add65bb7152 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 8 Dec 2023 13:42:12 +0000
Subject: [PATCH 047/133] combine mask outputs

---
 example/cfis/config_MaMa_onthefly.ini        |  2 +-
 example/cfis/config_exp_psfex.ini            |  9 +++--
 scripts/python/update_runs_log_file.py       |  2 ++
 scripts/sh/init_run_exclusive_canfar.sh      | 38 ++++++++++++--------
 scripts/sh/job_sp_canfar.bash                |  2 +-
 scripts/sh/prepare_tiles_for_final.bash      |  4 +--
 shapepipe/modules/mask_package/mask.py       | 15 ++++++++
 shapepipe/modules/setools_package/setools.py |  5 ++-
 shapepipe/modules/sextractor_runner.py       |  2 +-
 9 files changed, 55 insertions(+), 24 deletions(-)

diff --git a/example/cfis/config_MaMa_onthefly.ini b/example/cfis/config_MaMa_onthefly.ini
index bde813940..84f117e65 100644
--- a/example/cfis/config_MaMa_onthefly.ini
+++ b/example/cfis/config_MaMa_onthefly.ini
@@ -44,7 +44,7 @@ OUTPUT_DIR = $SP_RUN/output
 [JOB]
 
 # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
-SMP_BATCH_SIZE = 8
+SMP_BATCH_SIZE = 16
 
 # Timeout value (optional), default is None, i.e. no timeout limit applied
 TIMEOUT = 96:00:00
diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini
index 178aaee58..11f375456 100644
--- a/example/cfis/config_exp_psfex.ini
+++ b/example/cfis/config_exp_psfex.ini
@@ -9,7 +9,8 @@
 VERBOSE = True
 
 # Name of run (optional) default: shapepipe_run
-RUN_NAME = run_sp_exp_SxSePsfPi
+#RUN_NAME = run_sp_exp_SxSePsfPi
+RUN_NAME = run_sp_exp_SxSePsf
 
 # Add date and time to RUN_NAME, optional, default: True
 ; RUN_DATETIME = False
@@ -19,7 +20,8 @@ RUN_NAME = run_sp_exp_SxSePsfPi
 [EXECUTION]
 
 # Module name, single string or comma-separated list of valid module runner names
-MODULE = sextractor_runner, setools_runner, psfex_runner,psfex_interp_runner
+MODULE = sextractor_runner, setools_runner, psfex_runner
+#psfex_interp_runner
          
 
 # Run mode, SMP or MPI
@@ -57,7 +59,8 @@ TIMEOUT = 96:00:00
 [SEXTRACTOR_RUNNER]
 
 # Input from two modules
-INPUT_MODULE = split_exp_runner, mask_runner
+INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2
+#INPUT_MODULE = split_exp_runner, mask_runner_run_2
 
 # Read pipeline flag files created by mask module
 FILE_PATTERN = image, weight, pipeline_flag
diff --git a/scripts/python/update_runs_log_file.py b/scripts/python/update_runs_log_file.py
index 0ee5870ff..b244c28ec 100644
--- a/scripts/python/update_runs_log_file.py
+++ b/scripts/python/update_runs_log_file.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 import re
 import sys
 import os
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 42e8f5824..02c218271 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -24,21 +24,29 @@ cd ${typ}_runs
 
 cd $ID
 
-#mkdir output
-#cd output
-pwd
-#cp $basedir/output/log_run_sp.txt .
-#ln -s $basedir/output/log_exp_headers.sqlite
-#for dir in $basedir/output/run_sp_*; do
-	#ln -s $dir
-#done
-#cd ..
-
-export SP_RUN=.
-export SP_CONFIG=$HOME/shapepipe/example/cfis
-shapepipe_run -c $SP_CONFIG/config_exp_Pi.ini -e $ID
-
-#job_sp_canfar.bash -p psfex -j 32 -e $ID -n $n_SMP
+if [ ! -d "output" ]; then
+  mkdir output
+fi
+
+  cd output
+  #ln -s $basedir/output/log_exp_headers.sqlite
+  # Remove potentially obsolete link
+  rm run_sp_exp_SpMh*
+  for dir in $basedir/output/run_sp_*; do
+	  ln -s $dir
+  done
+  rm  run_sp_MaMa_*
+  ln -s $basedir/output/run_sp_combined_flag
+  cd ..
+  update_runs_log_file.py
+
+  pwd
+
+#export SP_RUN=.
+#export SP_CONFIG=$HOME/shapepipe/example/cfis
+#shapepipe_run -c $SP_CONFIG/config_exp_Pi.ini -e $ID
+
+job_sp_canfar.bash -p psfex -j 32 -e $ID -n $n_SMP
 
 cd $basedir
 
diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
index 621700f9a..daf997378 100755
--- a/scripts/sh/job_sp_canfar.bash
+++ b/scripts/sh/job_sp_canfar.bash
@@ -388,7 +388,7 @@ fi
 if [[ $do_job != 0 ]]; then
 
   ### Mask exposures
-  command_cfg_shapepipe "config_ext_Ma_$star_cat_for_mask.ini" "Run shapepipe (mask exposures)" $n_smp $exclusive
+  command_cfg_shapepipe "config_exp_Ma_$star_cat_for_mask.ini" "Run shapepipe (mask exposures)" $n_smp $exclusive
 
 fi
 
diff --git a/scripts/sh/prepare_tiles_for_final.bash b/scripts/sh/prepare_tiles_for_final.bash
index c26f8e8c2..454d8fd77 100755
--- a/scripts/sh/prepare_tiles_for_final.bash
+++ b/scripts/sh/prepare_tiles_for_final.bash
@@ -70,7 +70,7 @@ if [ "$cat" == "final" ]; then
   INPUT="$pwd/$out_base/run_sp_Mc_*"
 elif [ "$cat" == "flag" ]; then
   run_dir="run_sp_combined_flag"
-  INPUT="$pwd/$out_base/run_sp_tile_Ma_*"
+  INPUT="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_2"
 else
   run_dir="run_sp_combined_image"
   INPUT="$pwd/$out_base/run_sp_Git_*"
@@ -90,7 +90,7 @@ if [ "$cat" == "final" ]; then
   )
 elif [ "$cat" == "flag" ]; then
   DIRS=(
-	  "mask_runner"
+	  "mask_runner_run_2"
   )
   PATTERNS=(
 	  "pipeline_flag-*"
diff --git a/shapepipe/modules/mask_package/mask.py b/shapepipe/modules/mask_package/mask.py
index 69a1c3c97..230f2e897 100644
--- a/shapepipe/modules/mask_package/mask.py
+++ b/shapepipe/modules/mask_package/mask.py
@@ -113,6 +113,7 @@ def __init__(
         # Set error flag
         self._err = False
 
+
     def _get_config(self):
         """Get Config.
 
@@ -307,6 +308,20 @@ def make_mask(self):
         Main function to create the mask.
 
         """
+        output_file_name = (
+            f'{self._img_prefix}'
+            + f'{self._outname_base}{self._img_number}.fits'
+        )
+        if (
+            os.path.exists(f"output/run_sp_MaMa_2023-11-21_06-11-21/mask_runner_run_1/output/{output_file_name}")
+            or os.path.exists(f"output/run_sp_MaMa_2023-11-21_06-11-21/mask_runner_run_2/output/{output_file_name}")
+        ):
+            print("MKDEBUG skipping ", output_file_name)
+            return None, None
+        else:
+            print("MKDEBUG processing ", output_file_name)
+
+
         if self._config['MD']['make']:
             self.missing_data()
 
diff --git a/shapepipe/modules/setools_package/setools.py b/shapepipe/modules/setools_package/setools.py
index 78feae3d1..3d4d3eb9b 100644
--- a/shapepipe/modules/setools_package/setools.py
+++ b/shapepipe/modules/setools_package/setools.py
@@ -58,7 +58,10 @@ def __init__(
                 SEx_catalogue=True,
             )
             cat_file.open()
-            self._data = cat_file.get_data()
+            try:
+                self._data = cat_file.get_data()
+            except:
+                raise IOError(f"Could not load catalogue data from {cat}")
             cat_file.close()
 
         else:
diff --git a/shapepipe/modules/sextractor_runner.py b/shapepipe/modules/sextractor_runner.py
index 9e3bac7d3..e65374983 100644
--- a/shapepipe/modules/sextractor_runner.py
+++ b/shapepipe/modules/sextractor_runner.py
@@ -2,7 +2,7 @@
 
 Module runner for ``sextractor``.
 
-:Author: Axel Guinot
+:Author:s Axel Guinot, Martin Kilbinger
 
 """
 

From bc20ab2ba8f3b24b1334d8c6520845ea887a9199 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Dec 2023 09:46:43 +0100
Subject: [PATCH 048/133] canfar curl command: added kind (tile, exp) as option

---
 scripts/sh/curl_canfar_local.sh         | 13 ++++++++-----
 scripts/sh/prepare_tiles_for_final.bash |  2 +-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index e410c8d30..befb77356 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/bash
 
 # Usage
-# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.8 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID exp NCORE
+# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.9 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID ind
 
 SSL=~/.ssl/cadcproxy.pem
 NCORE=2
@@ -16,12 +16,15 @@ version=$1
 # command on canfar, e.g. shapepipe/scripts/sh/init_run_exclusive_canfar.sh
 cmd=$2
 
-# Image ID
-ID=$3
+# Kind ("tile" or "exp")
+kind=$3
+
+# Image ID; has to be last argument to work with xargs
+ID=$4
+
 
 # command line arguments
-arg="$ID $NCORE exp"
+arg="$ID $NCORE $kind"
 
-echo
 ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
 echo $ID >> IDs.txt
diff --git a/scripts/sh/prepare_tiles_for_final.bash b/scripts/sh/prepare_tiles_for_final.bash
index c26f8e8c2..8407dd2d8 100755
--- a/scripts/sh/prepare_tiles_for_final.bash
+++ b/scripts/sh/prepare_tiles_for_final.bash
@@ -90,7 +90,7 @@ if [ "$cat" == "final" ]; then
   )
 elif [ "$cat" == "flag" ]; then
   DIRS=(
-	  "mask_runner"
+	  "mask_runner_run_2"
   )
   PATTERNS=(
 	  "pipeline_flag-*"

From 1b4b3bf66c38f1cffd839e39794bff5947493f04 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Dec 2023 17:24:25 +0000
Subject: [PATCH 049/133] curl remote job script init_run_exclusive_canfar:
 command line options properly

---
 example/cfis/config_tile_PiViSmVi_canfar.ini |   2 +-
 scripts/python/link_to_exp_for_tile.py       |  32 ++++--
 scripts/sh/init_run_exclusive_canfar.sh      | 104 ++++++++++++++++---
 scripts/sh/prepare_tiles_for_final.bash      |   4 +-
 4 files changed, 119 insertions(+), 23 deletions(-)

diff --git a/example/cfis/config_tile_PiViSmVi_canfar.ini b/example/cfis/config_tile_PiViSmVi_canfar.ini
index ed9e3d339..71bdc699e 100644
--- a/example/cfis/config_tile_PiViSmVi_canfar.ini
+++ b/example/cfis/config_tile_PiViSmVi_canfar.ini
@@ -183,6 +183,6 @@ PREFIX =
 
 # Additional parameters for path and file pattern corresponding to single-exposure
 # run outputs
-ME_IMAGE_DIR = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner
+ME_IMAGE_DIR = last:split_exp_runner, last:split_exp_runner, last:split_exp_runner, all:sextractor_runner
 ME_IMAGE_PATTERN = flag, image, weight, background
 ME_LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite
diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index de18f92e8..e6f93756f 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -165,10 +165,13 @@ def matching_subdirs(base_dir, pattern):
 
     # Find all matching subdirectories
     subdirs = []
-    for entry in os.listdir(base_dir):
-        full_path = os.path.join(base_dir, entry)
-        if os.path.isdir(full_path) and entry.startswith(pattern):
-            subdirs.append(full_path)
+    if os.path.exists(base_dir):
+        for entry in os.listdir(base_dir):
+            full_path = os.path.join(base_dir, entry)
+            if os.path.isdir(full_path) and entry.startswith(pattern):
+                subdirs.append(full_path)
+    else:
+        print(f"Warning: {base_dir} does not exist, continuing...")
 
     # Sort according to creation date
     subdirs.sort(key=os.path.getctime)
@@ -189,6 +192,8 @@ def get_exp_IDs(tile_base_dir, tile_ID, verbose=False):
 
     subdirs = matching_subdirs(tile_out_dir, "run_sp_GitFeGie")
 
+    if len(subdirs) == 0:
+        raise IOError(f"No matching directory '{pattern}' in {tile_out_dir} found")
     if len(subdirs) != 1:
         raise IOError(f"Exactly one matching directory in {tile_out_dir} expected, not {len(subdirs)}")
 
@@ -253,9 +258,24 @@ def create_links_paths(tile_base_dir, tile_ID, paths):
         src = path
         dst = f"{tile_out_dir}/{tail}"
         if os.path.exists(dst):
-            print(f"Warning: {dst} already exists, no link created")
+            src_ex = os.readlink(dst)
+            if src_ex == src:
+                print(f"Warning: {dst} already exists, no link created")
+                continue
+            else:
+                idx = 1
+                dst_orig = dst
+                while True:
+                    dst = f"{dst_orig}_{idx}"
+                    if os.path.exists(dst):
+                        idx += 1
+                    else:
+                        #print("MKDEBUG new ", dst)
+                        #print(f"ln -s {src} {dst}")
+                        os.symlink(src, dst)
+                        break
         else:
-            print(f"ln -s {src} {dst}")
+            #print(f"ln -s {src} {dst}")
             os.symlink(src, dst)
 
 
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 02c218271..292b999a1 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -1,9 +1,74 @@
 #!/bin/bash
 
-echo "start init run exclusive canfar"
+# init_run_exclusive_canfar.sh
+
+# Command line arguments
+## Default values
+job=-1
+exclusive=-1
+n_smp=1
+kind=-1
+
+## Help string
+usage="Usage: $(basename "$0") -j JOB -e ID  -k KIND [OPTIONS]
+\n\nOptions:\n
+   -h\tthis message\n
+   -j, --job JOB\tRUnning JOB, bit-coded\n
+   -e, --exclusive ID\timage ID\n
+   -p, --psf MODEL\n
+    \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n
+   -k, --kind KIND\n
+    \timage kind, allowed are 'tile' and 'exp'\n
+   -n, --n_smp N_SMOp\n
+    \tnumber of jobs (SMP mode only), default from original config files\n
+"
+
+## Help if no arguments                                                         
+if [ -z $1 ]; then                                                              
+        echo -ne $usage                                                         
+        exit 1                                                                  
+fi 
+
+## Parse command line                                                           
+while [ $# -gt 0 ]; do                                                          
+  case "$1" in                                                                  
+    -h)                                                                         
+      echo -ne $usage                                                           
+      exit 0                                                                    
+      ;;                                                                        
+    -j|--job)                                                                   
+      job="$2"                                                                  
+      shift                                                                     
+      ;; 
+    -e|--exclusive)                                                             
+      exclusive="$2"                                                            
+      shift                                                                     
+      ;;
+    -n|--n_smp)                                                                 
+      n_smp="$2"                                                                
+      shift                                                                     
+      ;;                                                                        
+  esac                                                                          
+  shift                                                                         
+done
+
+# Check options
+if [ "$job" == "-1" ]; then
+  echo "No job indicated, use option -j"
+  exit 2
+fi
+
+if [ "$exclusive" == "-1" ]; then
+  echo "No image ID indicated, use option -e"
+  exit 3
+fi
 
-#echo init_canfar > ~/init_canfar.log
-#date >> ~/init_canfar.log
+if [ "kind" == "-1" ]; then
+  echo "No image kind indicated, use option -k"
+  exit 4
+fi
+
+echo "start init run exclusive canfar"
 
 . /opt/conda/etc/profile.d/conda.sh
 
@@ -13,14 +78,17 @@ basedir=$HOME/cosmostat/P3_v2/psfex
 cd $basedir
 
 
-ID=$1
-n_SMP=$2
-typ=$3
-echo "ID=$ID n_SMP=$n_SMP type=$typ"
+echo "ID=$ID n_SMP=$n_SMP kind=$kind"
 
-cd ${typ}_runs
+if [ "$kind" == "tile" ]; then
+  link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs
+fi
 
-#mkdir $ID
+cd ${kind}_runs
+
+if [ ! -d "$ID" ]; then
+  mkdir $ID
+fi
 
 cd $ID
 
@@ -28,25 +96,33 @@ if [ ! -d "output" ]; then
   mkdir output
 fi
 
+if [ "1" == "1" ]; then
   cd output
-  #ln -s $basedir/output/log_exp_headers.sqlite
+
+  if [ ! -f log_exp_headers.sqlite ]; then
+    ln -s $basedir/output/log_exp_headers.sqlite
+  fi
+
   # Remove potentially obsolete link
-  rm run_sp_exp_SpMh*
+  #rm run_sp_exp_SpMh*
+  #rm  run_sp_MaMa_*
+
   for dir in $basedir/output/run_sp_*; do
-	  ln -s $dir
+	  ln -sf $dir
   done
-  rm  run_sp_MaMa_*
   ln -s $basedir/output/run_sp_combined_flag
+
   cd ..
   update_runs_log_file.py
 
   pwd
+fi
 
 #export SP_RUN=.
 #export SP_CONFIG=$HOME/shapepipe/example/cfis
 #shapepipe_run -c $SP_CONFIG/config_exp_Pi.ini -e $ID
 
-job_sp_canfar.bash -p psfex -j 32 -e $ID -n $n_SMP
+job_sp_canfar.bash -p psfex -j 64 -e $ID -n $n_SMP
 
 cd $basedir
 
diff --git a/scripts/sh/prepare_tiles_for_final.bash b/scripts/sh/prepare_tiles_for_final.bash
index 454d8fd77..c74b67ea7 100755
--- a/scripts/sh/prepare_tiles_for_final.bash
+++ b/scripts/sh/prepare_tiles_for_final.bash
@@ -70,7 +70,7 @@ if [ "$cat" == "final" ]; then
   INPUT="$pwd/$out_base/run_sp_Mc_*"
 elif [ "$cat" == "flag" ]; then
   run_dir="run_sp_combined_flag"
-  INPUT="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_2"
+  INPUT="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
 else
   run_dir="run_sp_combined_image"
   INPUT="$pwd/$out_base/run_sp_Git_*"
@@ -90,7 +90,7 @@ if [ "$cat" == "final" ]; then
   )
 elif [ "$cat" == "flag" ]; then
   DIRS=(
-	  "mask_runner_run_2"
+	  "mask_runner_run_1"
   )
   PATTERNS=(
 	  "pipeline_flag-*"

From bd82f95ae96043c2de6ac577dae311c74a03e133 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 14 Dec 2023 21:35:06 +0100
Subject: [PATCH 050/133] curl local command: added -k kind

---
 scripts/sh/curl_canfar_local.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index befb77356..e7d64aada 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -4,9 +4,9 @@
 # ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.9 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID ind
 
 SSL=~/.ssl/cadcproxy.pem
-NCORE=2
+N_SMP=2
 SESSION=https://ws-uv.canfar.net/skaha/v0/session
-RESOURCES="ram=4&cores=$NCORE"
+RESOURCES="ram=4&cores=$N_SMP"
 IMAGE=images.canfar.net/unions/shapepipe
 NAME=shapepipe
 
@@ -22,9 +22,9 @@ kind=$3
 # Image ID; has to be last argument to work with xargs
 ID=$4
 
-
-# command line arguments
-arg="$ID $NCORE $kind"
+# command line arguments for remote script:
+# collect into string
+arg="-j $JOB -e $ID -n $N_SMP -k $kind"
 
 ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
 echo $ID >> IDs.txt

From 33ea4a18c30cedcdb35bcac5d226bda4b6a0f3b0 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 15 Dec 2023 09:33:55 +0000
Subject: [PATCH 051/133] summary missing ID 32 fixed (?)

---
 scripts/python/summary_run.py | 744 ++++++++++++++++++++++++++++++++++
 1 file changed, 744 insertions(+)
 create mode 100755 scripts/python/summary_run.py

diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
new file mode 100755
index 000000000..35d42b39f
--- /dev/null
+++ b/scripts/python/summary_run.py
@@ -0,0 +1,744 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+
+import logging
+
+from collections import Counter
+
+from tqdm import tqdm
+
+
+def get_IDs_from_file(path):
+    """Get IDs From File.
+
+    Return IDs from text file. Removes letters and replaces
+    dots "." with dashes "-".
+
+    Parameters
+    ----------
+    path: str
+        input file path
+
+    Returns
+    --------
+    list
+        IDs
+
+    """
+    numbers = []
+    with open(path) as f_in:
+        for line in f_in:
+            entry = line.rstrip()
+            number = re.sub("[a-zA-Z]", "", entry)
+            numbers.append(number)
+
+    return numbers
+
+
+def replace_dot_dash(numbers):
+
+    results = [re.sub("\.", "-", number) for number in numbers]
+
+    return results
+
+
+def replace_dash_dot_if_tile(numbers):
+
+    pattern = re.compile(r"(?<!\d{3}-\d{3})-")
+    results = [pattern.sub(".", number) for number in numbers]
+    
+    return results
+
+
+def replace_dot_dash(numbers):
+
+    results = [re.sub("\.", "-", number) for number in numbers]
+
+    return results
+
+
+
+def get_all_exposures(exp_number_file_list, verbose=False):
+    """Get All Exposures.
+
+    Return all exposure names from a list of text files.
+
+    Parameters
+    ----------
+    exp_number_list: list
+        input file names
+
+    """
+    exposures = set()
+    for idx, path in enumerate(exp_number_file_list):
+        exps = get_IDs_from_file(path)
+        exposures.update(exps)
+
+    return list(exposures)
+
+
+def get_all_shdus(exposures, n_CCD):
+    """Get All SHDUs.
+
+    Return all single-exposure single-HDU (CCD) IDs.
+
+    Parameters
+    ----------
+    exposures: list
+        exposure names
+    n_CCD: int
+        number of CCDs per exposure
+
+    Returns
+    --------
+    list
+        single-exposure single-HDU IDs
+
+    """
+    shdus = []
+    for exposure in exposures:
+        for idx_CCD in range(n_CCD):
+            shdus.append(f"{exposure}-{idx_CCD}")
+
+    return shdus
+
+
+def set_as_list(item=None, n=None, default=1):
+    """Set As List.
+
+    Return input as list.
+
+    Parameters
+    -----------
+    item: str, int, or list, optional
+        input item(s); default is None, in which
+        case the return is [1] * n
+    n: int, optional
+        number of list items to return, default is None,
+        in which case the number will be set to 1. If item and
+        n are not None, n has to be equal to len(item)
+    default: int, optional
+        value to return if item is not given;
+        default is 1
+
+    Raises
+    -------
+    IndexError
+        if n != len(item)
+
+    Returns
+    -------
+    list
+        input item(s) as list
+    """
+    my_n = n or 1
+
+    if not item:
+        result = [default] * my_n
+    elif not isinstance(item, list):
+        result = [item] * my_n
+    else:
+        result = item
+        if len(item) != my_n:
+            raise IndexError(f"item has length {len(item)} != {n}")
+
+    return result
+
+
+def check_special(module, paths_in_dir, names_in_dir):
+
+    if module == "setools_runner":
+        inds_special = []
+        for idx in range(len(paths_in_dir)):
+            base_path = paths_in_dir[idx].replace(names_in_dir[idx], "")
+
+            stats_dir = f"{base_path}/../stat"
+            stats_files = os.listdir(stats_dir)
+            if len(stats_files) != 1:
+                raise ValueError(
+                    f"Expected exactly one stats file in {stats_dir}, not"
+                    + f" {len(stats_files)}"
+                )
+
+            stats_path = os.path.join(stats_dir, stats_files[0])
+            with open(stats_path) as f_in:
+                lines = f_in.readlines()
+                for line in lines:
+                    entry = line.rstrip()
+                    m = re.search(line, "Nb stars = (\S*)")
+                    if m:
+                        value = int(m[2])
+                        if value == 0:
+                            inds_special.append(idx)
+                        else:
+                            print(f"b stars = {value}, not special")
+                        break
+
+        print(inds_special)
+        for idx in inds_special:
+            paths_in_dir.pop(idx)
+            names_in_dir.pop(idx)
+
+        return paths_in_dir, names_in_dir, len(inds_special)
+
+
+class job_data(object):
+    """Job Data.
+
+    Class to handle a job.
+
+    Parameters
+    ----------
+    bit: int
+        bit-coded job number
+    run_dir: str or list
+        run directory(ies)
+    modules: list
+        module names
+    key_expected: int or str
+        number of expected output files; if str: will be updated
+        with runtime value
+    n_mult: int or list, optional
+        multiplicity of output files, default `None`, in which
+        case it is set to 1
+    pattern: list, optional
+        if not None, file pattern to match; defafult is `None`
+    output_dir: str, optional
+        output directory, defaul is "./output"
+    output_subdirs: str, optional
+        output subdirectories if not `None`; default is `None`
+    output_subdirs_suffix: str, optional
+        output subdir suffix if not `None`; default is `None`
+    verbose: bool, optional
+        verbose output if True; default is False
+
+    """
+    def __init__(
+        self,
+        bit,
+        run_dir,
+        modules,
+        key_expected,
+        n_mult=None,
+        pattern=None,
+        output_dir="./output",
+        output_subdirs=None,
+        output_subdirs_suffix=None,
+        verbose=False,
+    ):
+        self._bit = bit
+        self._run_dir = set_as_list(item=run_dir, n=len(modules))
+        self._modules = modules
+        self._key_expected = set_as_list(item=key_expected, n=len(modules))
+        self._n_mult = set_as_list(item=n_mult, n=len(modules))
+        self._pattern = set_as_list(item=pattern, n=len(modules), default="")
+        self._output_dir = output_dir
+        self._output_subdirs = output_subdirs or [""]
+        self._output_subdirs_suffix = set_as_list(
+            output_subdirs_suffix, len(modules), default="."
+        )
+        self._verbose = verbose
+
+    def print_intro(self):
+        """Print Intro.
+
+        Print header line for job statistics.
+
+        """
+        logging.info(f" (Job {self._bit})")
+
+    @classmethod
+    def print_stats_header(self):
+        """Print Stats Header.
+
+        Print overall header information for stats output.
+
+        """
+        logging.info(
+            "module                          expected     found   miss_expl"
+            + " missing uniq_miss  fr_found"
+        )
+        logging.info("=" * 50)
+
+    @classmethod
+    def print_stats(
+        self,
+        module,
+        n_expected,
+        n_found,
+        n_missing_explained,
+        n_missing,
+        n_mult,
+    ):
+        """Print Stats.
+
+        Print output file statistics.
+
+        Parameters
+        ----------
+        module: str
+            module name
+        n_expected: int
+            number of expected files
+        n_found: int
+            number of found files
+        n_missing_explained: int
+            number of missing but explained files
+        n_missing: int
+            number of missing files
+        n_mult: int
+            multipicity
+
+        """
+        if n_expected > 0:
+            fraction_found = n_found / n_expected
+        else:
+            fraction_found = 1
+
+        n_missing_per_mult = n_missing / n_mult
+
+        logging.info(
+            f"{module:30s} {n_expected:9d} {n_found:9d}"
+            + f" {n_missing_explained:9d} {n_missing:9d}"
+            + f" {n_missing_per_mult:9.1f} {fraction_found:9.1%}"
+        )
+
+    @classmethod
+    def is_ID_in_str(self, ID, path):
+        if ID in path:
+            return True
+        #if re.sub("\.", "-", ID) in path:
+            #return True
+        #return False
+
+    @classmethod
+    def is_not_in_any(self, ID, list_str):
+        return not any(ID in string for string in list_str)
+
+    @classmethod
+    def get_unique(self, names):
+        n_all = len(names)
+        names_unique = list(set(names))
+        n_unique = len(names_unique)
+
+        if n_all != n_unique:
+            if True:  # self._verbose:
+                logging.warning(
+                    f"{n_all - n_unique} duplicates removed from {n_all} IDs"
+                )
+
+        return names_unique
+
+    def output_missing(
+        self, module, key_expected, names_in_dir, n_mult, par_runtime=None
+    ):
+        output_path = f"missing_job_{self._bit}_{module}.txt"
+
+        list_expected = get_par_runtime(par_runtime, key_expected, kind="list")
+
+        pattern = re.compile(r"\d+[\d-]+")
+        IDs = []
+        for name in names_in_dir:
+            match = pattern.search(name)
+            if match:
+                IDs.append(match.group())
+            else:
+                raise ValueError(f"No ID found in {name}")
+
+        ID_counts = Counter(IDs)
+        missing_IDs = [
+            ID for ID, count in ID_counts.items()
+            if count < n_mult
+        ]
+
+        n_all = len(missing_IDs)
+        missing_IDs_unique = self.get_unique(missing_IDs)
+        n_unique = len(missing_IDs_unique)
+
+        if n_unique > 0:
+            IDs_dot = replace_dash_dot_if_tile(missing_IDs_unique)
+            with open(output_path, "w") as f_out:
+                for ID in IDs_dot:
+                    print(ID, file=f_out)
+
+        return missing_IDs_unique
+
+    def output_missing_job(self, missing_IDs):
+        output_path = f"missing_job_{self._bit}_all.txt"
+
+        missing_IDs_all = set(missing_IDs)
+
+        if len(missing_IDs_all) > 0:
+            with open(output_path, "w") as f_out:
+                for ID in missing_IDs_all:
+                    print(ID, file=f_out)
+        else:
+            logging.warning("no missing IDs in output_missing_job")
+
+    def get_names_in_dir(self, iterable, module, idx):
+
+        # Initialise output file names
+        names_in_dir = []
+        paths_in_dir = []
+
+        # Loop over subdirs
+        for jdx, subdir in enumerate(iterable):
+            base_and_subdir = (
+                f"{self._output_dir}/{subdir}/"
+                + f"{self._output_subdirs_suffix[idx]}"
+            )
+            if os.path.isdir(base_and_subdir):
+
+                matches = []
+
+                # Loop over entries (files and dirs)
+                with os.scandir(base_and_subdir) as entries:
+                    for entry in entries:
+
+                        # Append directory name if matches module
+                        if entry.name.startswith(self._run_dir[idx]):
+                            matches.append(entry)
+
+                    # This entry does not match module -> next
+                    if not matches:
+                        continue
+
+                    # Sort according to creation time
+                    matches_sorted = sorted(
+                        matches,
+                        key=lambda entry: entry.name,
+                    )
+
+                    # Get most recent one
+                    last = matches_sorted[-1]
+
+                    # Get full path
+                    full_path = os.path.join(base_and_subdir, last.name)
+
+                    # Get module output directory
+                    directory = f"{full_path}/{module}/output"
+
+                    # Some modules have special requirements 
+                    if module == "setools_runner":
+                        directory = f"{directory}/rand_split"
+
+                    #if os.path.exists(directory):
+                    try:
+                        with os.scandir(directory) as entries2:
+                            # if entry2.is_file()
+                            files = [
+                                entry2.name
+                                for entry2 in entries2
+                                if entry2.name.startswith(self._pattern[idx])
+                            ]
+                            names_in_dir.extend(files)
+                            paths_in_dir.extend(
+                                [os.path.join(directory, file)
+                                for file in files]
+                            )
+                    except FileNotFoundError:
+                        pass
+                    except Exeption as e:
+                        print(f"Unknown error {e}")
+
+        return names_in_dir, paths_in_dir
+
+    def check_numbers(self, par_runtime=None, indices=None):
+        """Check Numbers.
+
+        Check output file numbers and IDs.
+
+        Parameters
+        ----------
+        par_runtime : dict, optional
+            runtime parameter. default is None
+        indices: list, optional
+            if not None (default), only check modules corresponding
+            to indices
+
+        """
+        # Update subdirs if not already set as list
+        if not isinstance(self._output_subdirs, list):
+            self._output_subdirs = get_par_runtime(
+                par_runtime, self._output_subdirs, kind="list"
+            )
+
+        self._paths_in_dir = {}
+        self._missing_IDs_unique = []
+
+        n_missing_job = 0
+
+        # Loop over modules
+        for idx, module in enumerate(self._modules):
+            if indices is not None and idx not in indices:
+                continue
+
+            # Look over subdirs
+            iterable = self._output_subdirs
+            if len(iterable) > 1 and self._verbose:
+                iterable = tqdm(iterable, desc="subdirs", leave=True)
+
+            # Get output file names and paths
+            names_in_dir, paths_in_dir = self.get_names_in_dir(
+                iterable,
+                module,
+                idx,
+            )
+
+            self._paths_in_dir[idx] = paths_in_dir
+
+            # If expected is string: Update parameter with runtime value
+            # and set as integer
+            if isinstance(self._key_expected[idx], str):
+                n_expected_base = get_par_runtime(
+                    par_runtime, self._key_expected[idx], kind="n"
+                )
+            else:
+                n_expected_base = self._key_expected[idx]
+
+            # Get some numbers
+            n_found = len(names_in_dir)
+            n_expected = n_expected_base * self._n_mult[idx]
+            n_missing = n_expected - n_found
+
+            n_missing_explained = 0
+            if False and n_missing > 0:
+                if module == "setools_runner":
+                    (
+                        paths_in_dir,
+                        names_in_dir,
+                        n_missing_explained,
+                    ) = check_special(module, paths_in_dir, names_in_dir)
+
+                n_missing = n_missing - n_missing_explained
+
+            # Print statistics
+            self.print_stats(
+                module,
+                n_expected,
+                n_found,
+                n_missing_explained,
+                n_missing,
+                self._n_mult[idx],
+            )
+
+            # Write missing IDs for module to file
+            if n_missing > 0:
+                missing_IDs_unique = self.output_missing(
+                    module,
+                    self._key_expected[idx],
+                    names_in_dir,
+                    self._n_mult[idx],
+                    par_runtime=par_runtime,
+                )
+                n_missing_job += n_missing
+                self._missing_IDs_unique.extend(missing_IDs_unique)
+
+        # Write missing IDs for entire job to file
+        if n_missing_job > 0:
+            self.output_missing_job(self._missing_IDs_unique)
+
+
+def get_par_runtime(par_runtime, key, kind="n"):
+    """Get Par RunTime.
+
+    Return runtime parameter value.
+
+    Parameters
+    ----------
+    par_runtime: dict
+        runtime parameter
+    key: str
+        key
+
+    """
+    combined_key = f"{kind}_{key}"
+    return par_runtime[combined_key]
+
+
+def print_par_runtime(par_runtime, verbose=True):
+    # Print runtime parameter values
+    if verbose:
+        logging.info("")
+        logging.info("===========")
+        logging.info("par_runtime")
+        logging.info("-----------")
+        for key, value in par_runtime.items():
+            if not key.startswith("list"):
+                logging.info(f"{key:30s} {value:6d}")
+            else:
+                logging.info(f"{key:29s} [{len(value):6d}]")
+        logging.info("===========")
+        logging.info("")
+
+
+def main(argv=None):
+    # Set default parameters
+    # p_def = params_default()
+
+    # Command line options
+    # options, args = parse_options(p_def)
+
+    # if check_options(options) is False:
+    # return 1
+
+    # param = update_param(p_def, options)
+
+    verbose = True
+    log_file_name = "summary_log.txt"
+    handlers = [logging.FileHandler(log_file_name), logging.StreamHandler()]
+    logging.basicConfig(
+        level=logging.INFO, format="%(message)s", handlers=handlers
+    )
+
+    main_dir = "."
+    retrieve = "vos"
+    tile_ID_path = f"{main_dir}/tile_numbers.txt"
+
+    # tile IDs with dots
+    list_tile_IDs_dot = get_IDs_from_file(tile_ID_path)
+
+    # tile IDs with dashes
+    list_tile_IDs = replace_dot_dash(list_tile_IDs_dot)
+    n_tile_IDs = len(list_tile_IDs)
+    n_CCD = 40
+
+    par_runtime = {}
+
+    par_runtime["n_tile_IDs"] = n_tile_IDs
+    par_runtime["list_tile_IDs"] = list_tile_IDs
+
+    jobs = {}
+
+    if retrieve == "vos":
+        n_link = 2
+    else:
+        n_link = 1
+
+    jobs["1"] = job_data(
+        1,
+        "run_sp_GitFeGie_",
+        [
+            "get_images_runner_run_1",
+            "find_exposures_runner",
+            "get_images_runner_run_2",
+        ],
+        ["tile_IDs", "tile_IDs", "exposures"],
+        pattern=["CFIS_", "", ""],
+        n_mult=[1 * n_link, 1, 3],
+        output_dir=f"{main_dir}/output",
+        verbose=verbose,
+    )
+
+    jobs["2"] = job_data(
+        2,
+        ["run_sp_Uz", "run_sp_exp_SpMh", "run_sp_exp_SpMh_2023-12"],
+        ["uncompress_fits_runner", "merge_headers_runner", "split_exp_runner"],
+        ["tile_IDs", 0, "3*n_shdus+n_exposures"],
+        n_mult=[1, 1, 1],
+        output_dir=f"{main_dir}/output",
+        verbose=verbose,
+    )
+
+    # TODO: rename run dirs to run_sp_tile_Ma, run_sp_exp_Ma
+    jobs["4"] = job_data(
+        4,
+        "run_sp_combined_flag",
+        ["mask_runner_run_1"],
+        ["tile_IDs"],
+        output_dir=f"{main_dir}/output",
+        verbose=verbose,
+    )
+
+    jobs["8"] = job_data(
+        8,
+        "run_sp_combined_flag",
+        ["mask_runner_run_2"],
+        ["shdus"],
+        output_dir=f"{main_dir}/output",
+        verbose=verbose,
+    )
+
+    jobs["16"] = job_data(
+        16,
+        "run_sp_tile_Sx",
+        ["sextractor_runner"],
+        ["tile_IDs"],
+        n_mult=2,
+        output_dir=f"{main_dir}/tile_runs",
+        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
+        verbose=verbose,
+    )
+
+    # TODO setools_runner output/mask
+    jobs["32"] = job_data(
+        32,
+        [
+            "run_sp_exp_SxSePsf",
+            "run_sp_exp_SxSePsf",
+            "run_sp_exp_SxSePsf",
+        ],  # "run_sp_exp_Pi"],
+        [
+            "sextractor_runner",
+            "setools_runner",
+            "psfex_runner",
+        ],  # "psfex_interp_runner"],
+        "shdus",
+        n_mult=[2, 2, 2],  # 1],
+        output_dir=f"{main_dir}/exp_runs",
+        output_subdirs="shdus",
+        output_subdirs_suffix="output",
+        verbose=verbose,
+    )
+
+    jobs["64"] = job_data(
+        "64",
+        "run_sp_tile_PsViSmVi",
+        [
+            "psfex_interp_runner",
+            "vignetmaker_runner_run_1",
+            "spread_model_runner",
+            "vignetmaker_runner_run_2",
+        ],
+        "tile_IDs",
+        n_mult=[1, 1, 1, 4],
+        output_dir=f"{main_dir}/tile_runs",
+        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
+        verbose=verbose,
+    )
+
+    job_data.print_stats_header()
+
+    for key in "1":
+        job = jobs[key]
+        job.print_intro()
+        job.check_numbers(par_runtime=par_runtime, indices=[0, 1])
+
+        all_exposures = get_all_exposures(job._paths_in_dir[1], verbose=True)
+        par_runtime["n_exposures"] = len(all_exposures)
+
+        job.check_numbers(par_runtime, indices=[2])
+
+    # Update runtime parameter
+    par_runtime["n_shdus"] = get_par_runtime(par_runtime, "exposures") * n_CCD
+    par_runtime["n_3*n_shdus+n_exposures"] = 3 * get_par_runtime(
+        par_runtime, "shdus"
+    ) + get_par_runtime(par_runtime, "exposures")
+    par_runtime["list_shdus"] = get_all_shdus(all_exposures, n_CCD)
+
+    print_par_runtime(par_runtime, verbose=verbose)
+
+    #for key in ["2", "4", "8", "16", "32", "64"]:
+    for key in ["32", "64"]:
+        job = jobs[key]
+        job.print_intro()
+        job.check_numbers(par_runtime=par_runtime)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

From ba17ebcd515131e5e19895cab56bf52135bda15d Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 15 Dec 2023 10:37:55 +0100
Subject: [PATCH 052/133] curl canfar local script added job, kind

---
 scripts/sh/curl_canfar_local.sh | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index e7d64aada..c0add5152 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/bash
 
 # Usage
-# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.9 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID ind
+# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.9 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID kind job
 
 SSL=~/.ssl/cadcproxy.pem
 N_SMP=2
@@ -19,12 +19,16 @@ cmd=$2
 # Kind ("tile" or "exp")
 kind=$3
 
+# Job number
+job=$4
+
 # Image ID; has to be last argument to work with xargs
-ID=$4
+ID=$5
 
 # command line arguments for remote script:
 # collect into string
-arg="-j $JOB -e $ID -n $N_SMP -k $kind"
+arg="-j $job -e $ID -n $N_SMP -k $kind"
 
+echo curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"
 ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
 echo $ID >> IDs.txt

From 9ac9dc1831faf827b92f4a3e5b325e79467e1d0d Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 17 Dec 2023 12:50:08 +0000
Subject: [PATCH 053/133] SP ngmxix (job 128) running on canfar

---
 example/cfis/config_tile_Ng_template.ini |  24 +---
 scripts/python/get_number_objects.py     |  17 ++-
 scripts/python/link_to_exp_for_tile.py   |  29 ++--
 scripts/python/summary_run.py            | 160 +++++++++++++++--------
 scripts/sh/init_run_exclusive_canfar.sh  |  99 +++++++++-----
 scripts/sh/job_sp_canfar.bash            |   4 +-
 6 files changed, 201 insertions(+), 132 deletions(-)

diff --git a/example/cfis/config_tile_Ng_template.ini b/example/cfis/config_tile_Ng_template.ini
index f9d733d71..5ad7aaf58 100644
--- a/example/cfis/config_tile_Ng_template.ini
+++ b/example/cfis/config_tile_Ng_template.ini
@@ -55,7 +55,7 @@ TIMEOUT = 96:00:00
 # Model-fitting shapes with ngmix
 [NGMIX_RUNNER]
 
-INPUT_DIR = last:sextractor_runner_run_1,last:X_interp_runner,last:vignetmaker_runner_run_2
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner,last:X_interp_runner,last:vignetmaker_runner_run_2
 
 FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet
 
@@ -75,25 +75,3 @@ PIXEL_SCALE = 0.186
 
 ID_OBJ_MIN = X
 ID_OBJ_MAX = X
-
-
-# Moment-based (KSB) shapes with galsim
-[GALSIM_SHAPES_V2_RUNNER]
-
-INPUT_DIR = last:sextractor_runner_run_2, last:vignetmaker_runner_run_1, last:X_interp_runner,last:vignetmaker_runner_run_2
-
-FILE_PATTERN = sexcat, weight_vignet, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet
-
-FILE_EXT = .fits, .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite
-
-# NUMBERING_SCHEME (optional) string with numbering pattern for input files
-NUMBERING_SCHEME = -000-000
-
-# Multi-epoch mode: Path to file with single-exposure WCS header information
-LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite
-
-# Magnitude zero-point
-MAG_ZP = 30.0
-
-ID_OBJ_MIN = X
-ID_OBJ_MAX = X
diff --git a/scripts/python/get_number_objects.py b/scripts/python/get_number_objects.py
index 67cdb294d..0f3ae41b1 100755
--- a/scripts/python/get_number_objects.py
+++ b/scripts/python/get_number_objects.py
@@ -17,7 +17,7 @@
 from optparse import OptionParser                                               
 from astropy.io import fits
 
-from shapepipe.pipeline.run_log import get_last_dir                             
+from shapepipe.pipeline.run_log import get_last_dir, get_all_dirs
 from shapepipe.utilities import cfis                                            
 
 
@@ -281,10 +281,21 @@ def main(argv=None):
     # Save command line arguments to log file
     f_log = cfis.log_command(argv, close_no_return=False)
 
-    module = 'sextractor_runner_run_1'
     pattern = 'sexcat'
     run_log_file = 'output/log_run_sp.txt'
-    last_dir = get_last_dir(run_log_file, module)
+
+    # For v1
+    #module = 'sextractor_runner_run_1'
+
+    # For v2
+    module = "sextractor_runner"
+    all_dir = get_all_dirs(run_log_file, module)
+    paths = []
+    for path in all_dir:
+        if "run_sp_tile_Sx" in path:
+            paths.append(path)
+    paths = sorted(paths)
+    last_dir = paths[-1]
 
     file_list = glob.glob(f'{last_dir}/{pattern}*.fits')
     if len(file_list) == 0:
diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index d750b97b7..f6e01021b 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -218,6 +218,8 @@ def get_exp_IDs(tile_base_dir, tile_ID, verbose=False):
             ID = re.sub("[a-zA-Z]", "", name) 
             exp_IDs.append(ID)
 
+    if verbose:
+        print("Exposures: ", exp_IDs)
     return exp_IDs
 
 
@@ -237,6 +239,7 @@ def get_paths(exp_base_dir, exp_shdu_IDs, pattern):
     number = {}
     paths = []
     for exp_shdu_ID in exp_shdu_IDs:
+
         name = f"{exp_base_dir}/{exp_shdu_ID}/output"
         path = os.path.abspath(name)
         subdirs = matching_subdirs(path, pattern)
@@ -253,14 +256,18 @@ def get_paths(exp_base_dir, exp_shdu_IDs, pattern):
                 + f"  not {n_subdirs}"
             )
             print(msg)
+            # More than one match: sort according to name = creation time 
+            subdirs = sorted(subdirs)
             if n_subdirs == 0:
                 continue
-        paths.append(f"{subdirs[0]}")
+
+        # Append matching subdir; if more than one append newest
+        paths.append(f"{subdirs[-1]}")
 
     return paths, number
 
 
-def create_links_paths(tile_base_dir, tile_ID, paths):
+def create_links_paths(tile_base_dir, tile_ID, paths, verbose=False):
 
     tile_out_dir = get_tile_out_dir(tile_base_dir, tile_ID)
 
@@ -272,8 +279,9 @@ def create_links_paths(tile_base_dir, tile_ID, paths):
         if os.path.exists(dst):
             src_existing = os.readlink(dst)
             if src_existing == src:
-                print(src, dst)
-                    #f"Warning: {src} <- {dst} already exists, no link created"
+                if verbose:
+                    #print("link {src} <- {dst}")
+                    f"Warning: {src} <- {dst} already exists, no link created"
                 #)
                 continue
             else:
@@ -284,12 +292,13 @@ def create_links_paths(tile_base_dir, tile_ID, paths):
                     if os.path.exists(dst):
                         idx += 1
                     else:
-                        #print("MKDEBUG new ", dst)
-                        #print(f"ln -s {src} {dst}")
+                        if verbose:
+                            print(f"link {src} <- {dst}")
                         os.symlink(src, dst)
                         break
         else:
-            #print(f"ln -s {src} {dst}")
+            if verbose:
+                print(f"link {src} <- {dst}")
             os.symlink(src, dst)
 
 
@@ -319,12 +328,8 @@ def main(argv=None):
     for pattern in patterns:
         paths, number = get_paths(exp_base_dir, exp_shdu_IDs, pattern)
         print(number)
-        if "2159307-31" in paths:
-            print("found")
-        else:
-            print("not found")
 
-        create_links_paths(tile_base_dir, tile_ID, paths)
+        create_links_paths(tile_base_dir, tile_ID, paths, verbose=verbose)
 
 
     return 0
diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index aedf090f6..e8cc4f0d5 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -206,12 +206,13 @@ class job_data(object):
         case it is set to 1
     pattern: list, optional
         if not None, file pattern to match; defafult is `None`
-    output_dir: str, optional
-        output directory, defaul is "./output"
+    path_lef: str, optional
+        left (first) part of output directory, defaul is "./output"
     output_subdirs: str, optional
         output subdirectories if not `None`; default is `None`
-    output_subdirs_suffix: str, optional
-        output subdir suffix if not `None`; default is `None`
+    path_right: str, optional
+        right (last) part of output subdir suffix if not `None`;
+        default is `None`
     verbose: bool, optional
         verbose output if True; default is False
 
@@ -224,9 +225,9 @@ def __init__(
         key_expected,
         n_mult=None,
         pattern=None,
-        output_dir="./output",
+        path_left="./output",
         output_subdirs=None,
-        output_subdirs_suffix=None,
+        path_right=None,
         verbose=False,
     ):
         self._bit = bit
@@ -235,10 +236,10 @@ def __init__(
         self._key_expected = set_as_list(item=key_expected, n=len(modules))
         self._n_mult = set_as_list(item=n_mult, n=len(modules))
         self._pattern = set_as_list(item=pattern, n=len(modules), default="")
-        self._output_dir = output_dir
+        self._path_left = path_left
         self._output_subdirs = output_subdirs or [""]
-        self._output_subdirs_suffix = set_as_list(
-            output_subdirs_suffix, len(modules), default="."
+        self._path_right = set_as_list(
+            path_right, len(modules), default="."
         )
         self._verbose = verbose
 
@@ -333,7 +334,13 @@ def get_unique(self, names):
         return names_unique
 
     def output_missing(
-        self, module, key_expected, names_in_dir, paths_in_dir, n_mult, par_runtime=None
+        self,
+        module,
+        key_expected,
+        names_in_dir,
+        paths_in_dir,
+        n_mult,
+        par_runtime=None,
     ):
         output_path = f"missing_job_{self._bit}_{module}.txt"
 
@@ -348,9 +355,6 @@ def output_missing(
             match = pattern.search(name)
             if match:
                 IDs.append(match.group())
-
-                if match.group() == '2079613-1':
-                    print(name, path)
             else:
                 raise ValueError(f"No ID found in {name}")
 
@@ -358,9 +362,6 @@ def output_missing(
         ## Count occurences
         ID_counts = Counter(IDs)
 
-        #import pdb
-        #pdb.set_trace()
-
         ## Add to missing if ocurence less than n_mult
         missing_IDs = []
         for ID in list_expected:
@@ -379,10 +380,10 @@ def output_missing(
 
         return missing_IDs_unique
 
-    def output_missing_job(self, missing_IDs):
+    def output_missing_job(self):
         output_path = f"missing_job_{self._bit}_all.txt"
 
-        missing_IDs_all = set(missing_IDs)
+        missing_IDs_all = set(self._missing_IDs_job)
 
         if len(missing_IDs_all) > 0:
             with open(output_path, "w") as f_out:
@@ -391,6 +392,42 @@ def output_missing_job(self, missing_IDs):
         else:
             logging.warning("no missing IDs in output_missing_job")
 
+    @classmethod
+    def get_last_full_path(self, base_and_subdir, matches):
+        """Get Last Full Path
+
+        Return full path of last file in list.
+
+        """
+        # Sort according to creation time
+        matches_sorted = sorted(
+            matches,
+            key=lambda entry: entry.name,
+        )
+
+        # Get most recent one
+        last = matches_sorted[-1]
+
+        # Get full path
+        full_path = os.path.join(base_and_subdir, last.name)
+
+        return full_path
+
+    @classmethod
+    def get_module_output_dir(self, full_path, module):
+        """Get Module Output Dir.
+
+        Return output directory name for given module.
+
+        """
+        directory = f"{full_path}/{module}/output"
+
+        # Some modules have special requirements 
+        if module == "setools_runner":
+            directory = f"{directory}/rand_split"
+
+        return directory
+
     def get_names_in_dir(self, iterable, module, idx):
 
         # Initialise output file names
@@ -400,8 +437,8 @@ def get_names_in_dir(self, iterable, module, idx):
         # Loop over subdirs
         for jdx, subdir in enumerate(iterable):
             base_and_subdir = (
-                f"{self._output_dir}/{subdir}/"
-                + f"{self._output_subdirs_suffix[idx]}"
+                f"{self._path_left}/{subdir}/"
+                + f"{self._path_right[idx]}"
             )
             if os.path.isdir(base_and_subdir):
 
@@ -419,25 +456,16 @@ def get_names_in_dir(self, iterable, module, idx):
                     if not matches:
                         continue
 
-                    # Sort according to creation time
-                    matches_sorted = sorted(
-                        matches,
-                        key=lambda entry: entry.name,
+                    full_path = self.get_last_full_path(
+                        base_and_subdir, matches
                     )
 
-                    # Get most recent one
-                    last = matches_sorted[-1]
-
-                    # Get full path
-                    full_path = os.path.join(base_and_subdir, last.name)
-
                     # Get module output directory
-                    directory = f"{full_path}/{module}/output"
-
-                    # Some modules have special requirements 
-                    if module == "setools_runner":
-                        directory = f"{directory}/rand_split"
+                    directory = self.get_module_output_dir(
+                        full_path, module
+                    )
 
+                    # Look over files
                     # os.path.whether exists is twice faster than try/except
                     if os.path.exists(directory):
                         with os.scandir(directory) as entries2:
@@ -446,6 +474,8 @@ def get_names_in_dir(self, iterable, module, idx):
                                 for entry2 in entries2
                                 if entry2.name.startswith(self._pattern[idx])
                             ]
+
+                            # Append matching files
                             names_in_dir.extend(files)
                             paths_in_dir.extend(
                                 [os.path.join(directory, file)
@@ -454,6 +484,17 @@ def get_names_in_dir(self, iterable, module, idx):
 
         return names_in_dir, paths_in_dir
 
+    def update_subdirs(self, par_runtime):
+        """Update Subdirs.
+
+        Update subdir names with runtime information if required.
+
+        """
+        if not isinstance(self._output_subdirs, list):
+            self._output_subdirs = get_par_runtime(
+                par_runtime, self._output_subdirs, kind="list"
+            )
+
     def check_numbers(self, par_runtime=None, indices=None):
         """Check Numbers.
 
@@ -469,14 +510,11 @@ def check_numbers(self, par_runtime=None, indices=None):
 
         """
         # Update subdirs if not already set as list
-        if not isinstance(self._output_subdirs, list):
-            self._output_subdirs = get_par_runtime(
-                par_runtime, self._output_subdirs, kind="list"
-            )
+        self.update_subdirs(par_runtime)
 
+        # Initialise variables
         self._paths_in_dir = {}
-        self._missing_IDs_unique = []
-
+        self._missing_IDs_job = []
         n_missing_job = 0
 
         # Loop over modules
@@ -513,8 +551,8 @@ def check_numbers(self, par_runtime=None, indices=None):
             n_missing = n_expected - n_found
 
             n_missing_explained = 0
-            if False and n_missing > 0:
-                if module == "setools_runner":
+            if n_missing > 0:
+                if False and module == "setools_runner":
                     (
                         paths_in_dir,
                         names_in_dir,
@@ -535,7 +573,7 @@ def check_numbers(self, par_runtime=None, indices=None):
 
             # Write missing IDs for module to file
             if n_missing > 0:
-                missing_IDs_unique = self.output_missing(
+                missing_IDs = self.output_missing(
                     module,
                     self._key_expected[idx],
                     names_in_dir,
@@ -544,11 +582,11 @@ def check_numbers(self, par_runtime=None, indices=None):
                     par_runtime=par_runtime,
                 )
                 n_missing_job += n_missing
-                self._missing_IDs_unique.extend(missing_IDs_unique)
+                self._missing_IDs_job.extend(missing_IDs)
 
         # Write missing IDs for entire job to file
         if n_missing_job > 0:
-            self.output_missing_job(self._missing_IDs_unique)
+            self.output_missing_job()
 
 
 def get_par_runtime(par_runtime, key, kind="n"):
@@ -638,7 +676,7 @@ def main(argv=None):
         ["tile_IDs", "tile_IDs", "exposures"],
         pattern=["CFIS_", "", ""],
         n_mult=[1 * n_link, 1, 3],
-        output_dir=f"{main_dir}/output",
+        path_left=f"{main_dir}/output",
         verbose=verbose,
     )
 
@@ -648,7 +686,7 @@ def main(argv=None):
         ["uncompress_fits_runner", "merge_headers_runner", "split_exp_runner"],
         ["tile_IDs", 0, "3*n_shdus+n_exposures"],
         n_mult=[1, 1, 1],
-        output_dir=f"{main_dir}/output",
+        path_left=f"{main_dir}/output",
         verbose=verbose,
     )
 
@@ -657,7 +695,7 @@ def main(argv=None):
         "run_sp_combined_flag",
         ["mask_runner_run_1"],
         ["tile_IDs"],
-        output_dir=f"{main_dir}/output",
+        path_left=f"{main_dir}/output",
         verbose=verbose,
     )
 
@@ -666,7 +704,7 @@ def main(argv=None):
         "run_sp_combined_flag",
         ["mask_runner_run_2"],
         ["shdus"],
-        output_dir=f"{main_dir}/output",
+        path_left=f"{main_dir}/output",
         verbose=verbose,
     )
 
@@ -676,7 +714,7 @@ def main(argv=None):
         ["sextractor_runner"],
         ["tile_IDs"],
         n_mult=2,
-        output_dir=f"{main_dir}/tile_runs",
+        path_left=f"{main_dir}/tile_runs",
         output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
         verbose=verbose,
     )
@@ -696,9 +734,9 @@ def main(argv=None):
         ],  # "psfex_interp_runner"],
         "shdus",
         n_mult=[2, 2, 2],  # 1],
-        output_dir=f"{main_dir}/exp_runs",
+        path_left=f"{main_dir}/exp_runs",
         output_subdirs="shdus",
-        output_subdirs_suffix="output",
+        path_right="output",
         verbose=verbose,
     )
 
@@ -713,7 +751,17 @@ def main(argv=None):
         ],
         "tile_IDs",
         n_mult=[1, 1, 1, 4],
-        output_dir=f"{main_dir}/tile_runs",
+        path_left=f"{main_dir}/tile_runs",
+        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
+        verbose=verbose,
+    )
+
+    jobs["128"] = job_data(
+        "128",
+        "run_sp_tile_ngmix_Ng1u",
+        ["ngmix_runner"],
+        "tile_IDs",
+        path_left=f"{main_dir}/tile_runs",
         output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
         verbose=verbose,
     )
@@ -739,8 +787,8 @@ def main(argv=None):
 
     print_par_runtime(par_runtime, verbose=verbose)
 
-    #for key in ["2", "4", "8", "16", "32", "64"]:
-    for key in ["32", "64"]:
+    for key in ["2", "4", "8", "16", "32", "64", "128"]:
+    #for key in ["128"]:
         job = jobs[key]
         job.print_intro()
         job.check_numbers(par_runtime=par_runtime)
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 6b8903710..ff8c6d50e 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -6,8 +6,9 @@
 ## Default values
 job=-1
 ID=-1
-n_SMP=1
+N_SMP=1
 kind=-1
+dry_run=0
 
 ## Help string
 usage="Usage: $(basename "$0") -j JOB -e ID  -k KIND [OPTIONS]
@@ -20,8 +21,10 @@ usage="Usage: $(basename "$0") -j JOB -e ID  -k KIND [OPTIONS]
     \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n
    -k, --kind KIND\n
     \timage kind, allowed are 'tile' and 'exp'\n
-   -n, --n_SMP N_SMOp\n
+   -N, --N_SMP N_SMOp\n
     \tnumber of jobs (SMP mode only), default from original config files\n
+   -n, --dry_run\n
+    \tdry run, no actuall processing\n
 "
 
 ## Help if no arguments                                                         
@@ -45,7 +48,7 @@ while [ $# -gt 0 ]; do
       ID="$2"                                                            
       shift                                                                     
       ;;
-    -n|--n_SMP)                                                                 
+    -N|--N_SMP)                                                                 
       n_SMP="$2"                                                                
       shift                                                                     
       ;;                                                                        
@@ -53,6 +56,9 @@ while [ $# -gt 0 ]; do
       kind="$2"                                                                
       shift                                                                     
       ;;                                                                        
+    -n|--dry_run)
+      dry_run=1
+      ;;
   esac                                                                          
   shift                                                                         
 done
@@ -75,6 +81,10 @@ fi
 
 echo "start init run exclusive canfar"
 
+if [ "$dry_run" == 1 ]; then
+  echo "in dry run mode"
+fi
+
 . /opt/conda/etc/profile.d/conda.sh
 
 conda activate shapepipe
@@ -83,47 +93,59 @@ basedir=$HOME/cosmostat/P3_v2/psfex
 cd $basedir
 
 
-echo "ID=$ID n_SMP=$n_SMP kind=$kind"
+if [ "$dry_run" == "0" ]; then
 
-if [ "$kind" == "tile" ]; then
-  rm -rf tile_runs/$ID/output/run_exp_SxSePsf*
-  link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs
-fi
+  if [ "$kind" == "tile" ]; then
+    rm -rf tile_runs/$ID/output/run_exp_SxSePsf*
+    link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs
+  fi
 
-cd ${kind}_runs
+  cd ${kind}_runs
 
-if [ ! -d "$ID" ]; then
-  mkdir $ID
-fi
+  if [ ! -d "$ID" ]; then
+    mkdir $ID
+  fi
 
-cd $ID
+  cd $ID
+  pwd
 
-if [ ! -d "output" ]; then
-  mkdir output
-fi
+  if [ ! -d "output" ]; then
+    mkdir output
+  fi
 
-cd output
+  cd output
 
-if [ ! -f log_exp_headers.sqlite ]; then
-  ln -s $basedir/output/log_exp_headers.sqlite
-fi
+  if [ ! -f log_exp_headers.sqlite ]; then
+    ln -s $basedir/output/log_exp_headers.sqlite
+  fi
 
-# Remove potentially obsolete link
-#rm run_sp_exp_SpMh*
-#rm  run_sp_MaMa_*
+  # Remove potentially obsolete link
+  #rm run_sp_exp_SpMh*
+  #rm  run_sp_MaMa_*
 
-for dir in $basedir/output/run_sp_*; do
- ln -sf $dir
-done
-if [ ! -e run_sp_combined_flag ]; then
-  ln -s $basedir/output/run_sp_combined_flag
-fi
+  for dir in $basedir/output/run_sp_*; do
+  ln -sf $dir
+  done
+  if [ ! -e run_sp_combined_flag ]; then
+    ln -s $basedir/output/run_sp_combined_flag
+  fi
+
+  # Remove unfinished ngmix dirs
+  if [ -e "run_sp_tile_ngmix_Ng1u/ngmix_runner" ]; then
+    echo "previous empty ngmix run found, removing"
+    n_out=`ls -rlt run_sp_tile_ngmix_Ng7u/ngmix_runner/output | wc -l`
+    if [ "$n_out" == "1" ]; then
+      rm -rf run_sp_tile_ngmix_Ng*u
+    fi
+  fi
 
-cd ..
-update_runs_log_file.py
+  cd ..
+  update_runs_log_file.py
 
-echo -n "pwd: "
-pwd
+  echo -n "pwd: "
+  pwd
+
+fi
 
 #export SP_RUN=.
 #export SP_CONFIG=$HOME/shapepipe/example/cfis
@@ -132,10 +154,15 @@ pwd
 echo -n "environment: "
 echo $CONDA_PREFIX
 
-cmd="job_sp_canfar.bash -p psfex -j $job -e $ID -n $n_SMP"
-echo "Running commnd $cmd ..."
+cmd="job_sp_canfar.bash -p psfex -j $job -e $ID --n_smp $N_SMP"
+echo -n "Running commnd '$cmd' ..."
 
-$cmd
+if [ "$dry_run" == 1 ]; then
+  echo " dry run"
+else
+  echo
+  $cmd
+fi
 
 cd $basedir
 
diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
index 78bcfc58f..97280f374 100755
--- a/scripts/sh/job_sp_canfar.bash
+++ b/scripts/sh/job_sp_canfar.bash
@@ -55,8 +55,8 @@ usage="Usage: $(basename "$0") [OPTIONS] [TILE_ID]
    -n, --n_smp N_SMP\n
    \tnumber of jobs (SMP mode only), default from original config files\n
    --nsh_step NSTEP\n
-   --nsh_jobs NJOB\n
    \tnumber of shape measurement parallel jobs, default=$nsh_jobs\n
+   --nsh_jobs NJOB\n
    \tnumber of objects per parallel shape module call, \n
    \tdefault: optimal number is computed\n
    --nsh_max NMAX\n
@@ -432,7 +432,7 @@ if [[ $do_job != 0 ]]; then
   ### Prepare config files
   n_min=0
   if [[ $nsh_step == -1 ]]; then
-    n_obj=`get_number_objects.py`
+    n_obj=`get_number_objects`
     nsh_step=`echo "$(($n_obj/$nsh_jobs))"`
   fi
 

From 1d25a8b0a1c8ad456547b875918260953f644575 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 17 Dec 2023 13:51:37 +0100
Subject: [PATCH 054/133] curl canfar local script updated

---
 scripts/sh/curl_canfar_local.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index e410c8d30..9481adc7c 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -23,5 +23,8 @@ ID=$3
 arg="$ID $NCORE exp"
 
 echo
-ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
+curl_cmd="curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d kcmd=$cmd\" --data-urlencode \"args=$arg\""
+echo $curl_cmd
+ID=`$curl_cmd`
+#ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
 echo $ID >> IDs.txt

From ca6b602b754bb772738cf3aa1e199875e8bb341e Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 19 Dec 2023 10:12:56 +0100
Subject: [PATCH 055/133] curl scripts updated

---
 scripts/sh/curl_canfar_local.sh         | 120 ++++++++++++++++++++----
 scripts/sh/curl_canfar_monitor_local.sh |   4 +-
 scripts/sh/init_run_exclusive_canfar.sh |   5 +-
 scripts/sh/job_curl_canfar_local.sh     |  16 ++++
 4 files changed, 126 insertions(+), 19 deletions(-)
 create mode 100644 scripts/sh/job_curl_canfar_local.sh

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index c0add5152..3ec61555d 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -3,32 +3,120 @@
 # Usage
 # ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.9 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID kind job
 
+# Command line arguments
+## Default values
 SSL=~/.ssl/cadcproxy.pem
-N_SMP=2
 SESSION=https://ws-uv.canfar.net/skaha/v0/session
-RESOURCES="ram=4&cores=$N_SMP"
 IMAGE=images.canfar.net/unions/shapepipe
 NAME=shapepipe
 
-# version of image on canfar, e.g. 0:7, 0:8
-version=$1
+job=-1
+ID=-1
+N_SMP=1
+kind=-1
+version=0.9
+cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
+dry_run=0
+
+# TODO psf
 
-# command on canfar, e.g. shapepipe/scripts/sh/init_run_exclusive_canfar.sh
-cmd=$2
+## Help string
+usage="Usage: $(basename "$0") -j JOB -e ID  -k KIND [OPTIONS]
+\n\nOptions:\n
+   -h\tthis message\n
+   -j, --job JOB\tRUnning JOB, bit-coded\n
+   -e, --exclusive ID
+    \timage ID\n
+   -p, --psf MODEL\n
+    \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n
+   -k, --kind KIND\n
+    \timage kind, allowed are 'tile' and 'exp'\n
+   -N, --N_SMP N_SMOp\n
+    \tnumber of jobs (SMP mode only), default from original config files\n
+   -V, --version\n
+    \tversion of docker image, default='$version'\n
+   -C, --command_remote\n
+    \tremote command to run on canfar, default='$cmd_remote'\n
+   -n, --dry_run LEVEL\n
+    \tdry run, from LEVEL=2 (no processing) to 0 (full run)\n
+"
 
-# Kind ("tile" or "exp")
-kind=$3
+## Help if no arguments
+if [ -z $1 ]; then
+        echo -ne $usage
+        exit 1
+fi
 
-# Job number
-job=$4
+## Parse command line
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -h)
+      echo -ne $usage
+      exit 0
+      ;;
+    -j|--job)
+      job="$2"
+      shift
+      ;;
+    -e|--exclusive)
+      ID="$2"
+      shift
+      ;;
+    -N|--N_SMP)
+      n_SMP="$2"
+      shift
+      ;;
+    -k|--kind)
+      kind="$2"
+      shift
+      ;;
+    -n|--dry_run)
+      dry_run="$2"
+      shift
+      ;;
+  esac
+  shift
+done
 
-# Image ID; has to be last argument to work with xargs
-ID=$5
+# Check options                                                                 
+if [ "$job" == "-1" ]; then                                                     
+  echo "No job indicated, use option -j"                                        
+  exit 2                                                                        
+fi                                                                              
+                                                                                
+if [ "$exclusive" == "-1" ]; then                                               
+  echo "No image ID indicated, use option -e"                                   
+  exit 3                                                                        
+fi                                                                              
+                                                                                
+if [ "kind" == "-1" ]; then                                                     
+  echo "No image kind indicated, use option -k"                                 
+  exit 4                                                                        
+fi
+
+if [ "$dry_run" != 0 ] && [ "$dry_run" != 1 ] && [ "$dry_run" != 2 ]; then
+  echo "Invalid dry_run option, allowed are 0, 1, and 2"
+  exit 5
+fi
 
 # command line arguments for remote script:
 # collect into string
-arg="-j $job -e $ID -n $N_SMP -k $kind"
 
-echo curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"
-ID=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd" --data-urlencode "args=$arg"`
-echo $ID >> IDs.txt
+if [ "$dry_run" == "1" ]; then
+  arg_dry_run="-n $dry_run"
+else
+  arg_dry_run=""
+fi
+
+RESOURCES="ram=4&cores=$N_SMP"
+arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run"
+
+
+if [ "$dry_run" == 2 ]; then
+  echo "Running command dry run:"
+  echo curl -E $SSL $SESSION?$RESOURCES -d \"image=$IMAGE:$version\" -d \"name=${NAME}\" -d \"cmd=$cmd_remote\" --data-urlencode \"args=$arg\"
+else
+  session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg"`
+  echo $session >> session_IDs.txt
+  echo "$session $ID" >> session_image_IDs.txt
+fi
diff --git a/scripts/sh/curl_canfar_monitor_local.sh b/scripts/sh/curl_canfar_monitor_local.sh
index 412e33c01..8260e583d 100755
--- a/scripts/sh/curl_canfar_monitor_local.sh
+++ b/scripts/sh/curl_canfar_monitor_local.sh
@@ -16,8 +16,8 @@ type=$1
 echo "type=$type"
 
 while [ 1 ]; do
-  ID=`tail -n 1 IDs.txt`
-  cmd="curl -E $SSL $SESSION/$ID?view=$type"
+  session_ID=`tail -n 1 session_IDs.txt`
+  cmd="curl -E $SSL $SESSION/$session_ID?view=$type"
   echo $cmd
   $cmd
 done
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 6b8903710..5ff13a8a3 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -9,6 +9,9 @@ ID=-1
 n_SMP=1
 kind=-1
 
+
+# TODO: psf
+
 ## Help string
 usage="Usage: $(basename "$0") -j JOB -e ID  -k KIND [OPTIONS]
 \n\nOptions:\n
@@ -73,7 +76,7 @@ if [ "kind" == "-1" ]; then
   exit 4
 fi
 
-echo "start init run exclusive canfar"
+echo "start init_run_exclusive_canfar"
 
 . /opt/conda/etc/profile.d/conda.sh
 
diff --git a/scripts/sh/job_curl_canfar_local.sh b/scripts/sh/job_curl_canfar_local.sh
new file mode 100644
index 000000000..a06d4eb9c
--- /dev/null
+++ b/scripts/sh/job_curl_canfar_local.sh
@@ -0,0 +1,16 @@
+# Usage:
+# Edit the file "all.txt"
+# screen
+# bash run_curl.sh kind job
+
+rm -f session_IDs.txt session_image_IDs.txt
+
+script_local=$HOME/astro/repositories/github/shapepipe/scripts/sh/curl_canfar_local.sh
+version="0.9"
+cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
+kind="$1"
+job="$2"
+
+echo $kind $job
+cat all.txt | xargs -n 1 -P 10 $script_local -v $version -c $cmd_remote -k $kind -j $job -e
+

From c643d0c1500a48cdaa2c4657fdb1b9bcd179e643 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 19 Dec 2023 09:21:50 +0000
Subject: [PATCH 056/133] run summary more OO

---
 scripts/python/summary_run.py           | 90 +++++++++++++++----------
 scripts/sh/init_run_exclusive_canfar.sh | 34 ++++++++--
 2 files changed, 83 insertions(+), 41 deletions(-)

diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index e8cc4f0d5..11a2725b7 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -3,6 +3,7 @@
 import sys
 import os
 import re
+import fnmatch
 
 import logging
 
@@ -336,12 +337,14 @@ def get_unique(self, names):
     def output_missing(
         self,
         module,
-        key_expected,
-        names_in_dir,
-        paths_in_dir,
-        n_mult,
+        idx,
         par_runtime=None,
     ):
+        key_expected = self._key_expected[idx]
+        names_in_dir = self._names_in_dir[idx]
+        paths_in_dir = self._paths_in_dir[idx]
+        n_mult = self._n_mult[idx]
+
         output_path = f"missing_job_{self._bit}_{module}.txt"
 
         list_expected = get_par_runtime(par_runtime, key_expected, kind="list")
@@ -428,11 +431,46 @@ def get_module_output_dir(self, full_path, module):
 
         return directory
 
+    def get_matches_final(self, directory, idx):
+
+        # Look over files
+        # os.path.whether exists is twice faster than try/except
+
+        if os.path.exists(directory):
+            pattern =  f"{self._pattern[idx]}*"
+            for entry2 in os.scandir(directory):
+                if (
+                    entry2.is_file()
+                    and (
+                        fnmatch.fnmatch(entry2.name, pattern)
+                    )
+                ):
+                    # Append matching files
+                    self._names_in_dir[idx].append(entry2.name)
+                    self._paths_in_dir[idx].append(
+                        os.path.join(directory, entry2.name)
+                    )
+
+        #if os.path.exists(directory):
+            #with os.scandir(directory) as entries2:
+                #files = [
+                    #entry2.name
+                    #for entry2 in entries2
+                    #if entry2.name.startswith(self._pattern[idx])
+                #]
+
+                ## Append matching files
+                #self._names_in_dir[idx].extend(files)
+                #self._paths_in_dir[idx].extend(
+                    #[os.path.join(directory, file)
+                    #for file in files]
+                #)
+
     def get_names_in_dir(self, iterable, module, idx):
 
-        # Initialise output file names
-        names_in_dir = []
-        paths_in_dir = []
+        # Initialise output file names and paths
+        self._names_in_dir[idx] = []
+        self._paths_in_dir[idx] = []
 
         # Loop over subdirs
         for jdx, subdir in enumerate(iterable):
@@ -465,24 +503,8 @@ def get_names_in_dir(self, iterable, module, idx):
                         full_path, module
                     )
 
-                    # Look over files
-                    # os.path.whether exists is twice faster than try/except
-                    if os.path.exists(directory):
-                        with os.scandir(directory) as entries2:
-                            files = [
-                                entry2.name
-                                for entry2 in entries2
-                                if entry2.name.startswith(self._pattern[idx])
-                            ]
-
-                            # Append matching files
-                            names_in_dir.extend(files)
-                            paths_in_dir.extend(
-                                [os.path.join(directory, file)
-                                for file in files]
-                            )
-
-        return names_in_dir, paths_in_dir
+                    # Find matching file names and paths
+                    self.get_matches_final(directory, idx)
 
     def update_subdirs(self, par_runtime):
         """Update Subdirs.
@@ -513,6 +535,7 @@ def check_numbers(self, par_runtime=None, indices=None):
         self.update_subdirs(par_runtime)
 
         # Initialise variables
+        self._names_in_dir = {}
         self._paths_in_dir = {}
         self._missing_IDs_job = []
         n_missing_job = 0
@@ -528,14 +551,12 @@ def check_numbers(self, par_runtime=None, indices=None):
                 iterable = tqdm(iterable, desc="subdirs", leave=True)
 
             # Get output file names and paths
-            names_in_dir, paths_in_dir = self.get_names_in_dir(
+            self.get_names_in_dir(
                 iterable,
                 module,
                 idx,
             )
 
-            self._paths_in_dir[idx] = paths_in_dir
-
             # If expected is string: Update parameter with runtime value
             # and set as integer
             if isinstance(self._key_expected[idx], str):
@@ -546,16 +567,18 @@ def check_numbers(self, par_runtime=None, indices=None):
                 n_expected_base = self._key_expected[idx]
 
             # Get some numbers
-            n_found = len(names_in_dir)
+            n_found = len(self._names_in_dir[idx])
             n_expected = n_expected_base * self._n_mult[idx]
             n_missing = n_expected - n_found
 
             n_missing_explained = 0
             if n_missing > 0:
+                # TODO: make check_special class function, deal with
+                # paths, names in dir
                 if False and module == "setools_runner":
                     (
-                        paths_in_dir,
-                        names_in_dir,
+                        self._paths_in_dir[idx],
+                        self._names_in_dir[idx],
                         n_missing_explained,
                     ) = check_special(module, paths_in_dir, names_in_dir)
 
@@ -575,10 +598,7 @@ def check_numbers(self, par_runtime=None, indices=None):
             if n_missing > 0:
                 missing_IDs = self.output_missing(
                     module,
-                    self._key_expected[idx],
-                    names_in_dir,
-                    paths_in_dir,
-                    self._n_mult[idx],
+                    idx,
                     par_runtime=par_runtime,
                 )
                 n_missing_job += n_missing
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index ff8c6d50e..500bab61c 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -9,6 +9,7 @@ ID=-1
 N_SMP=1
 kind=-1
 dry_run=0
+nsh_jobs=8
 
 ## Help string
 usage="Usage: $(basename "$0") -j JOB -e ID  -k KIND [OPTIONS]
@@ -130,12 +131,33 @@ if [ "$dry_run" == "0" ]; then
     ln -s $basedir/output/run_sp_combined_flag
   fi
 
-  # Remove unfinished ngmix dirs
-  if [ -e "run_sp_tile_ngmix_Ng1u/ngmix_runner" ]; then
-    echo "previous empty ngmix run found, removing"
-    n_out=`ls -rlt run_sp_tile_ngmix_Ng7u/ngmix_runner/output | wc -l`
-    if [ "$n_out" == "1" ]; then
-      rm -rf run_sp_tile_ngmix_Ng*u
+  # Indentify and remove unfinished ngmix dirs
+  min_n_out=2
+  for k in $(seq 1 $nsh_jobs); do
+    ngmix_run="run_sp_tile_ngmix_Ng${k}u/ngmix_runner"
+    if [ -e "$ngmix_run" ]; then
+      ngmix_out="$ngmix_run/output"
+      n_out=`ls -rlt $ngmix_out | wc -l`
+      if [ "$n_out" -lt "$min_n_out" ]; then
+          min_n_out=$n_out
+      fi
+      #echo $k $n_out $min_n_out
+    else
+      echo "ngmix separated run #$k not found"
+      min_n_out=0
+    fi
+  done
+  if [ "$min_n_out" -lt "2" ]; then
+    echo "At least one ngmix separated run no output files"
+    for k in $(seq 1 $nsh_jobs); do
+      cmd="rm -rf run_sp_tile_ngmix_Ng${k}u"
+      echo $cmd
+      `$cmd`
+    done
+  else
+    if [ "$job" == "128" ]; then
+      echo "ngmix found complete, all is well, exiting"
+      exit 0
     fi
   fi
 

From 53d9d35a53aaae74235e96666f4944fb6e5bedb2 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 21 Dec 2023 13:22:22 +0000
Subject: [PATCH 057/133] update_runs_log_file script: fixed bug when run dir
 is empty

---
 scripts/python/update_runs_log_file.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/python/update_runs_log_file.py b/scripts/python/update_runs_log_file.py
index b244c28ec..a08890117 100644
--- a/scripts/python/update_runs_log_file.py
+++ b/scripts/python/update_runs_log_file.py
@@ -42,7 +42,8 @@ def get_module_runs(subdirs):
     all_runs = {}
     for subdir in subdirs:
         runs = matching_subdirs(subdir, "_runner", tail=True)
-        all_runs[subdir] = runs
+        if len(runs) > 0:
+            all_runs[subdir] = runs
 
     return all_runs
 

From ce1179eea49af9b2fe53b2625c9809fe26ed52fd Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 21 Dec 2023 14:42:39 +0000
Subject: [PATCH 058/133] improved canfar job scripts; fixed some make cat bugs

---
 example/cfis/config_make_cat_psfex.ini        |   2 +-
 scripts/python/summary_run.py                 | 663 +-----------------
 scripts/sh/init_run_exclusive_canfar.sh       |  67 +-
 scripts/sh/job_curl_canfar_local.sh           |   3 +-
 .../modules/make_cat_package/make_cat.py      |   2 +-
 5 files changed, 51 insertions(+), 686 deletions(-)

diff --git a/example/cfis/config_make_cat_psfex.ini b/example/cfis/config_make_cat_psfex.ini
index fdf5414b5..a7407d990 100644
--- a/example/cfis/config_make_cat_psfex.ini
+++ b/example/cfis/config_make_cat_psfex.ini
@@ -55,7 +55,7 @@ TIMEOUT = 96:00:00
 [MAKE_CAT_RUNNER]
 
 # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
-INPUT_DIR = last:sextractor_runner_run_1, last:spread_model_runner, last:psfex_interp_runner, last:merge_sep_cats_runner
+INPUT_DIR = run_sp_tile_Sx:sextractor_runner, last:spread_model_runner, last:psfex_interp_runner, last:merge_sep_cats_runner
 
 # Input file pattern(s), list of strings with length matching number of expected input file types
 # Cannot contain wild cards
diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index 11a2725b7..462467e33 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -1,659 +1,14 @@
 #!/usr/bin/env python
 
 import sys
-import os
-import re
-import fnmatch
 
-import logging
-
-from collections import Counter
-
-from tqdm import tqdm
-
-
-def get_IDs_from_file(path):
-    """Get IDs From File.
-
-    Return IDs from text file. Removes letters and replaces
-    dots "." with dashes "-".
-
-    Parameters
-    ----------
-    path: str
-        input file path
-
-    Returns
-    --------
-    list
-        IDs
-
-    """
-    numbers = []
-    with open(path) as f_in:
-        for line in f_in:
-            entry = line.rstrip()
-            number = re.sub("[a-zA-Z]", "", entry)
-            numbers.append(number)
-
-    return numbers
-
-
-def replace_dot_dash(numbers):
-
-    results = [re.sub("\.", "-", number) for number in numbers]
-
-    return results
-
-
-def replace_dash_dot_if_tile(numbers):
-
-    pattern = re.compile(r"(\d{3})-(\d{3})")
-    results = [pattern.sub(r"\1.\2", number) for number in numbers]
-    
-    return results
-
-
-def replace_dot_dash(numbers):
-
-    results = [re.sub("\.", "-", number) for number in numbers]
-
-    return results
-
-
-
-def get_all_exposures(exp_number_file_list, verbose=False):
-    """Get All Exposures.
-
-    Return all exposure names from a list of text files.
-
-    Parameters
-    ----------
-    exp_number_list: list
-        input file names
-
-    """
-    exposures = set()
-    for idx, path in enumerate(exp_number_file_list):
-        exps = get_IDs_from_file(path)
-        exposures.update(exps)
-
-    return list(exposures)
-
-
-def get_all_shdus(exposures, n_CCD):
-    """Get All SHDUs.
-
-    Return all single-exposure single-HDU (CCD) IDs.
-
-    Parameters
-    ----------
-    exposures: list
-        exposure names
-    n_CCD: int
-        number of CCDs per exposure
-
-    Returns
-    --------
-    list
-        single-exposure single-HDU IDs
-
-    """
-    shdus = []
-    for exposure in exposures:
-        for idx_CCD in range(n_CCD):
-            shdus.append(f"{exposure}-{idx_CCD}")
-
-    return shdus
-
-
-def set_as_list(item=None, n=None, default=1):
-    """Set As List.
-
-    Return input as list.
-
-    Parameters
-    -----------
-    item: str, int, or list, optional
-        input item(s); default is None, in which
-        case the return is [1] * n
-    n: int, optional
-        number of list items to return, default is None,
-        in which case the number will be set to 1. If item and
-        n are not None, n has to be equal to len(item)
-    default: int, optional
-        value to return if item is not given;
-        default is 1
-
-    Raises
-    -------
-    IndexError
-        if n != len(item)
-
-    Returns
-    -------
-    list
-        input item(s) as list
-    """
-    my_n = n or 1
-
-    if not item:
-        result = [default] * my_n
-    elif not isinstance(item, list):
-        result = [item] * my_n
-    else:
-        result = item
-        if len(item) != my_n:
-            raise IndexError(f"item has length {len(item)} != {n}")
-
-    return result
-
-
-def check_special(module, paths_in_dir, names_in_dir):
-
-    if module == "setools_runner":
-        inds_special = []
-        for idx in range(len(paths_in_dir)):
-            base_path = paths_in_dir[idx].replace(names_in_dir[idx], "")
-
-            stats_dir = f"{base_path}/../stat"
-            stats_files = os.listdir(stats_dir)
-            if len(stats_files) != 1:
-                raise ValueError(
-                    f"Expected exactly one stats file in {stats_dir}, not"
-                    + f" {len(stats_files)}"
-                )
-
-            stats_path = os.path.join(stats_dir, stats_files[0])
-            with open(stats_path) as f_in:
-                lines = f_in.readlines()
-                for line in lines:
-                    entry = line.rstrip()
-                    m = re.search(line, "Nb stars = (\S*)")
-                    if m:
-                        value = int(m[2])
-                        if value == 0:
-                            inds_special.append(idx)
-                        else:
-                            print(f"b stars = {value}, not special")
-                        break
-
-        print(inds_special)
-        for idx in inds_special:
-            paths_in_dir.pop(idx)
-            names_in_dir.pop(idx)
-
-        return paths_in_dir, names_in_dir, len(inds_special)
-
-
-class job_data(object):
-    """Job Data.
-
-    Class to handle a job.
-
-    Parameters
-    ----------
-    bit: int
-        bit-coded job number
-    run_dir: str or list
-        run directory(ies)
-    modules: list
-        module names
-    key_expected: int or str
-        number of expected output files; if str: will be updated
-        with runtime value
-    n_mult: int or list, optional
-        multiplicity of output files, default `None`, in which
-        case it is set to 1
-    pattern: list, optional
-        if not None, file pattern to match; defafult is `None`
-    path_lef: str, optional
-        left (first) part of output directory, defaul is "./output"
-    output_subdirs: str, optional
-        output subdirectories if not `None`; default is `None`
-    path_right: str, optional
-        right (last) part of output subdir suffix if not `None`;
-        default is `None`
-    verbose: bool, optional
-        verbose output if True; default is False
-
-    """
-    def __init__(
-        self,
-        bit,
-        run_dir,
-        modules,
-        key_expected,
-        n_mult=None,
-        pattern=None,
-        path_left="./output",
-        output_subdirs=None,
-        path_right=None,
-        verbose=False,
-    ):
-        self._bit = bit
-        self._run_dir = set_as_list(item=run_dir, n=len(modules))
-        self._modules = modules
-        self._key_expected = set_as_list(item=key_expected, n=len(modules))
-        self._n_mult = set_as_list(item=n_mult, n=len(modules))
-        self._pattern = set_as_list(item=pattern, n=len(modules), default="")
-        self._path_left = path_left
-        self._output_subdirs = output_subdirs or [""]
-        self._path_right = set_as_list(
-            path_right, len(modules), default="."
-        )
-        self._verbose = verbose
-
-    def print_intro(self):
-        """Print Intro.
-
-        Print header line for job statistics.
-
-        """
-        logging.info(f" (Job {self._bit})")
-
-    @classmethod
-    def print_stats_header(self):
-        """Print Stats Header.
-
-        Print overall header information for stats output.
-
-        """
-        logging.info(
-            "module                          expected     found   miss_expl"
-            + " missing uniq_miss  fr_found"
-        )
-        logging.info("=" * 50)
-
-    @classmethod
-    def print_stats(
-        self,
-        module,
-        n_expected,
-        n_found,
-        n_missing_explained,
-        n_missing,
-        n_mult,
-    ):
-        """Print Stats.
-
-        Print output file statistics.
-
-        Parameters
-        ----------
-        module: str
-            module name
-        n_expected: int
-            number of expected files
-        n_found: int
-            number of found files
-        n_missing_explained: int
-            number of missing but explained files
-        n_missing: int
-            number of missing files
-        n_mult: int
-            multipicity
-
-        """
-        if n_expected > 0:
-            fraction_found = n_found / n_expected
-        else:
-            fraction_found = 1
-
-        n_missing_per_mult = n_missing / n_mult
-
-        logging.info(
-            f"{module:30s} {n_expected:9d} {n_found:9d}"
-            + f" {n_missing_explained:9d} {n_missing:9d}"
-            + f" {n_missing_per_mult:9.1f} {fraction_found:9.1%}"
-        )
-
-    @classmethod
-    def is_ID_in_str(self, ID, path):
-        if ID in path:
-            return True
-        #if re.sub("\.", "-", ID) in path:
-            #return True
-        #return False
-
-    @classmethod
-    def is_not_in_any(self, ID, list_str):
-        return not any(ID in string for string in list_str)
-
-    @classmethod
-    def get_unique(self, names):
-        n_all = len(names)
-        names_unique = list(set(names))
-        n_unique = len(names_unique)
-
-        if n_all != n_unique:
-            if True:  # self._verbose:
-                logging.warning(
-                    f"{n_all - n_unique} duplicates removed from {n_all} IDs"
-                )
-
-        return names_unique
-
-    def output_missing(
-        self,
-        module,
-        idx,
-        par_runtime=None,
-    ):
-        key_expected = self._key_expected[idx]
-        names_in_dir = self._names_in_dir[idx]
-        paths_in_dir = self._paths_in_dir[idx]
-        n_mult = self._n_mult[idx]
-
-        output_path = f"missing_job_{self._bit}_{module}.txt"
-
-        list_expected = get_par_runtime(par_runtime, key_expected, kind="list")
-        
-        # Count image IDs in names that were found earlier
-
-        ## Extract image IDs from names
-        IDs = []
-        pattern = re.compile(r"(?:\d{3}-\d{3}|\d{7}-\d+)")
-        for name, path in zip(names_in_dir, paths_in_dir):
-            match = pattern.search(name)
-            if match:
-                IDs.append(match.group())
-            else:
-                raise ValueError(f"No ID found in {name}")
-
-
-        ## Count occurences
-        ID_counts = Counter(IDs)
-
-        ## Add to missing if ocurence less than n_mult
-        missing_IDs = []
-        for ID in list_expected:
-            if ID_counts[ID] < n_mult:
-                missing_IDs.append(ID)
-
-        n_all = len(missing_IDs)
-        missing_IDs_unique = self.get_unique(missing_IDs)
-        n_unique = len(missing_IDs_unique)
-
-        if n_unique > 0:
-            IDs_dot = replace_dash_dot_if_tile(missing_IDs_unique)
-            with open(output_path, "w") as f_out:
-                for ID in IDs_dot:
-                    print(ID, file=f_out)
-
-        return missing_IDs_unique
-
-    def output_missing_job(self):
-        output_path = f"missing_job_{self._bit}_all.txt"
-
-        missing_IDs_all = set(self._missing_IDs_job)
-
-        if len(missing_IDs_all) > 0:
-            with open(output_path, "w") as f_out:
-                for ID in missing_IDs_all:
-                    print(ID, file=f_out)
-        else:
-            logging.warning("no missing IDs in output_missing_job")
-
-    @classmethod
-    def get_last_full_path(self, base_and_subdir, matches):
-        """Get Last Full Path
-
-        Return full path of last file in list.
-
-        """
-        # Sort according to creation time
-        matches_sorted = sorted(
-            matches,
-            key=lambda entry: entry.name,
-        )
-
-        # Get most recent one
-        last = matches_sorted[-1]
-
-        # Get full path
-        full_path = os.path.join(base_and_subdir, last.name)
-
-        return full_path
-
-    @classmethod
-    def get_module_output_dir(self, full_path, module):
-        """Get Module Output Dir.
-
-        Return output directory name for given module.
-
-        """
-        directory = f"{full_path}/{module}/output"
-
-        # Some modules have special requirements 
-        if module == "setools_runner":
-            directory = f"{directory}/rand_split"
-
-        return directory
-
-    def get_matches_final(self, directory, idx):
-
-        # Look over files
-        # os.path.whether exists is twice faster than try/except
-
-        if os.path.exists(directory):
-            pattern =  f"{self._pattern[idx]}*"
-            for entry2 in os.scandir(directory):
-                if (
-                    entry2.is_file()
-                    and (
-                        fnmatch.fnmatch(entry2.name, pattern)
-                    )
-                ):
-                    # Append matching files
-                    self._names_in_dir[idx].append(entry2.name)
-                    self._paths_in_dir[idx].append(
-                        os.path.join(directory, entry2.name)
-                    )
-
-        #if os.path.exists(directory):
-            #with os.scandir(directory) as entries2:
-                #files = [
-                    #entry2.name
-                    #for entry2 in entries2
-                    #if entry2.name.startswith(self._pattern[idx])
-                #]
-
-                ## Append matching files
-                #self._names_in_dir[idx].extend(files)
-                #self._paths_in_dir[idx].extend(
-                    #[os.path.join(directory, file)
-                    #for file in files]
-                #)
-
-    def get_names_in_dir(self, iterable, module, idx):
-
-        # Initialise output file names and paths
-        self._names_in_dir[idx] = []
-        self._paths_in_dir[idx] = []
-
-        # Loop over subdirs
-        for jdx, subdir in enumerate(iterable):
-            base_and_subdir = (
-                f"{self._path_left}/{subdir}/"
-                + f"{self._path_right[idx]}"
-            )
-            if os.path.isdir(base_and_subdir):
-
-                matches = []
-
-                # Loop over entries (files and dirs)
-                with os.scandir(base_and_subdir) as entries:
-                    for entry in entries:
-
-                        # Append directory name if matches module
-                        if entry.name.startswith(self._run_dir[idx]):
-                            matches.append(entry)
-
-                    # This entry does not match module -> next
-                    if not matches:
-                        continue
-
-                    full_path = self.get_last_full_path(
-                        base_and_subdir, matches
-                    )
-
-                    # Get module output directory
-                    directory = self.get_module_output_dir(
-                        full_path, module
-                    )
-
-                    # Find matching file names and paths
-                    self.get_matches_final(directory, idx)
-
-    def update_subdirs(self, par_runtime):
-        """Update Subdirs.
-
-        Update subdir names with runtime information if required.
-
-        """
-        if not isinstance(self._output_subdirs, list):
-            self._output_subdirs = get_par_runtime(
-                par_runtime, self._output_subdirs, kind="list"
-            )
-
-    def check_numbers(self, par_runtime=None, indices=None):
-        """Check Numbers.
-
-        Check output file numbers and IDs.
-
-        Parameters
-        ----------
-        par_runtime : dict, optional
-            runtime parameter. default is None
-        indices: list, optional
-            if not None (default), only check modules corresponding
-            to indices
-
-        """
-        # Update subdirs if not already set as list
-        self.update_subdirs(par_runtime)
-
-        # Initialise variables
-        self._names_in_dir = {}
-        self._paths_in_dir = {}
-        self._missing_IDs_job = []
-        n_missing_job = 0
-
-        # Loop over modules
-        for idx, module in enumerate(self._modules):
-            if indices is not None and idx not in indices:
-                continue
-
-            # Look over subdirs
-            iterable = self._output_subdirs
-            if len(iterable) > 1 and self._verbose:
-                iterable = tqdm(iterable, desc="subdirs", leave=True)
-
-            # Get output file names and paths
-            self.get_names_in_dir(
-                iterable,
-                module,
-                idx,
-            )
-
-            # If expected is string: Update parameter with runtime value
-            # and set as integer
-            if isinstance(self._key_expected[idx], str):
-                n_expected_base = get_par_runtime(
-                    par_runtime, self._key_expected[idx], kind="n"
-                )
-            else:
-                n_expected_base = self._key_expected[idx]
-
-            # Get some numbers
-            n_found = len(self._names_in_dir[idx])
-            n_expected = n_expected_base * self._n_mult[idx]
-            n_missing = n_expected - n_found
-
-            n_missing_explained = 0
-            if n_missing > 0:
-                # TODO: make check_special class function, deal with
-                # paths, names in dir
-                if False and module == "setools_runner":
-                    (
-                        self._paths_in_dir[idx],
-                        self._names_in_dir[idx],
-                        n_missing_explained,
-                    ) = check_special(module, paths_in_dir, names_in_dir)
-
-                n_missing = n_missing - n_missing_explained
-
-            # Print statistics
-            self.print_stats(
-                module,
-                n_expected,
-                n_found,
-                n_missing_explained,
-                n_missing,
-                self._n_mult[idx],
-            )
-
-            # Write missing IDs for module to file
-            if n_missing > 0:
-                missing_IDs = self.output_missing(
-                    module,
-                    idx,
-                    par_runtime=par_runtime,
-                )
-                n_missing_job += n_missing
-                self._missing_IDs_job.extend(missing_IDs)
-
-        # Write missing IDs for entire job to file
-        if n_missing_job > 0:
-            self.output_missing_job()
-
-
-def get_par_runtime(par_runtime, key, kind="n"):
-    """Get Par RunTime.
-
-    Return runtime parameter value.
-
-    Parameters
-    ----------
-    par_runtime: dict
-        runtime parameter
-    key: str
-        key
-
-    """
-    combined_key = f"{kind}_{key}"
-    return par_runtime[combined_key]
-
-
-def print_par_runtime(par_runtime, verbose=True):
-    # Print runtime parameter values
-    if verbose:
-        logging.info("")
-        logging.info("===========")
-        logging.info("par_runtime")
-        logging.info("-----------")
-        for key, value in par_runtime.items():
-            if not key.startswith("list"):
-                logging.info(f"{key:30s} {value:6d}")
-            else:
-                logging.info(f"{key:29s} [{len(value):6d}]")
-        logging.info("===========")
-        logging.info("")
+from shapepipe.utilities.summary import *
 
 
 def main(argv=None):
-    # Set default parameters
-    # p_def = params_default()
-
-    # Command line options
-    # options, args = parse_options(p_def)
-
-    # if check_options(options) is False:
-    # return 1
 
-    # param = update_param(p_def, options)
 
+    # Setting
     verbose = True
     log_file_name = "summary_log.txt"
     handlers = [logging.FileHandler(log_file_name), logging.StreamHandler()]
@@ -776,13 +131,19 @@ def main(argv=None):
         verbose=verbose,
     )
 
+    n_sh = 8
+    run_dirs = [f"run_sp_tile_ngmix_Ng{idx+1}u" for idx in range(n_sh)]
+    output_path_missing_IDs = [
+        f"missing_job_128_ngmix_runner_{idx+1}.txt" for idx in range(n_sh)
+    ]
     jobs["128"] = job_data(
         "128",
-        "run_sp_tile_ngmix_Ng1u",
-        ["ngmix_runner"],
+        run_dirs,
+        ["ngmix_runner"] * 8,
         "tile_IDs",
         path_left=f"{main_dir}/tile_runs",
         output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
+        output_path_missing_IDs=output_path_missing_IDs,
         verbose=verbose,
     )
 
@@ -807,8 +168,8 @@ def main(argv=None):
 
     print_par_runtime(par_runtime, verbose=verbose)
 
-    for key in ["2", "4", "8", "16", "32", "64", "128"]:
-    #for key in ["128"]:
+    #for key in ["2", "4", "8", "16", "32", "64", "128"]:
+    for key in ["128"]:
         job = jobs[key]
         job.print_intro()
         job.check_numbers(par_runtime=par_runtime)
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 9461a0b72..2dbcf41a5 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -78,7 +78,7 @@ if [ "$exclusive" == "-1" ]; then
   exit 3
 fi
 
-if [ "kind" == "-1" ]; then
+if [ "$kind" == "-1" ]; then
   echo "No image kind indicated, use option -k"
   exit 4
 fi
@@ -99,6 +99,7 @@ cd $basedir
 
 if [ "$dry_run" == "0" ]; then
 
+  # Update links to exposure run directories
   if [ "$kind" == "tile" ]; then
     rm -rf tile_runs/$ID/output/run_exp_SxSePsf*
     link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs
@@ -127,42 +128,48 @@ if [ "$dry_run" == "0" ]; then
   #rm run_sp_exp_SpMh*
   #rm  run_sp_MaMa_*
 
+  # Update links to global run directories (GiFeGie, Uz, Ma?, combined_flag?)
   for dir in $basedir/output/run_sp_*; do
-  ln -sf $dir
+    ln -sf $dir
   done
   if [ ! -e run_sp_combined_flag ]; then
     ln -s $basedir/output/run_sp_combined_flag
   fi
 
-  # Indentify and remove unfinished ngmix dirs
-  min_n_out=2
-  for k in $(seq 1 $nsh_jobs); do
-    ngmix_run="run_sp_tile_ngmix_Ng${k}u/ngmix_runner"
-    if [ -e "$ngmix_run" ]; then
-      ngmix_out="$ngmix_run/output"
-      n_out=`ls -rlt $ngmix_out | wc -l`
-      if [ "$n_out" -lt "$min_n_out" ]; then
-          min_n_out=$n_out
-      fi
-      #echo $k $n_out $min_n_out
-    else
-      echo "ngmix separated run #$k not found"
-      min_n_out=0
-    fi
-  done
-  if [ "$min_n_out" -lt "2" ]; then
-    echo "At least one ngmix separated run no output files"
+  (( do_job= $job & 128 ))
+  if [[ $do_job != 0 ]]; then
+
+    # Indentify and remove unfinished ngmix dirs
+    min_n_out=2
     for k in $(seq 1 $nsh_jobs); do
-      cmd="rm -rf run_sp_tile_ngmix_Ng${k}u"
-      echo $cmd
-      `$cmd`
+      ngmix_run="run_sp_tile_ngmix_Ng${k}u/ngmix_runner"
+      if [ -e "$ngmix_run" ]; then
+        ngmix_out="$ngmix_run/output"
+        n_out=`ls -rlt $ngmix_out | wc -l`
+        if [ "$n_out" -lt "$min_n_out" ]; then
+            min_n_out=$n_out
+        fi
+        #echo $k $n_out $min_n_out
+      else
+        echo "ngmix separated run #$k not found"
+        min_n_out=0
+      fi
     done
-  else
-    if [ "$job" == "128" ]; then
-      echo "ngmix found complete, all is well, exiting"
-      exit 0
+    if [ "$min_n_out" -lt "2" ]; then
+      echo "At least one ngmix separated run no output files"
+      for k in $(seq 1 $nsh_jobs); do
+        cmd="rm -rf run_sp_tile_ngmix_Ng${k}u"
+        echo $cmd
+        `$cmd`
+      done
+    else
+      if [ "$job" == "128" ]; then
+        echo "ngmix found complete, all is well, exiting"
+        exit 0
+      fi
     fi
-  fi
+
+  fi 
 
   cd ..
   update_runs_log_file.py
@@ -172,10 +179,6 @@ if [ "$dry_run" == "0" ]; then
 
 fi
 
-#export SP_RUN=.
-#export SP_CONFIG=$HOME/shapepipe/example/cfis
-#shapepipe_run -c $SP_CONFIG/config_exp_Pi.ini -e $ID
-
 echo -n "environment: "
 echo $CONDA_PREFIX
 
diff --git a/scripts/sh/job_curl_canfar_local.sh b/scripts/sh/job_curl_canfar_local.sh
index a06d4eb9c..e8a16513f 100644
--- a/scripts/sh/job_curl_canfar_local.sh
+++ b/scripts/sh/job_curl_canfar_local.sh
@@ -5,7 +5,8 @@
 
 rm -f session_IDs.txt session_image_IDs.txt
 
-script_local=$HOME/astro/repositories/github/shapepipe/scripts/sh/curl_canfar_local.sh
+#script_local=$HOME/astro/repositories/github/shapepipe/scripts/sh/curl_canfar_local.sh
+script_local=./curl_canfar_local.sh
 version="0.9"
 cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
 kind="$1"
diff --git a/shapepipe/modules/make_cat_package/make_cat.py b/shapepipe/modules/make_cat_package/make_cat.py
index 6bbe26648..4e30e3552 100644
--- a/shapepipe/modules/make_cat_package/make_cat.py
+++ b/shapepipe/modules/make_cat_package/make_cat.py
@@ -16,7 +16,7 @@
 from sqlitedict import SqliteDict
 
 from shapepipe.pipeline import file_io
-from shapepipe.utitities import galaxy
+from shapepipe.utilities import galaxy
 
 
 def prepare_final_cat_file(output_path, file_number_string):

From a005119ad5503a6fca3ce28f7e5b703e414f14c5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 21 Dec 2023 15:44:44 +0100
Subject: [PATCH 059/133] added curl to Dockerimage

---
 Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile b/Dockerfile
index 6cd1923e7..4cfd891d8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,6 +27,7 @@ RUN apt-get update --allow-releaseinfo-change && \
     apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \
     apt-get install vim -y && \
     apt-get install locate -y && \
+    apt-get install curl -y && \
     apt-get clean
 
 RUN apt-get install acl -y && \

From 22ee0f3a53a688df7aee7116ba2e7c8c64c40c6e Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 21 Dec 2023 15:49:57 +0100
Subject: [PATCH 060/133] Remove temp hack from mask

---
 shapepipe/modules/mask_package/mask.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/shapepipe/modules/mask_package/mask.py b/shapepipe/modules/mask_package/mask.py
index 230f2e897..b8b81a3cc 100644
--- a/shapepipe/modules/mask_package/mask.py
+++ b/shapepipe/modules/mask_package/mask.py
@@ -312,15 +312,6 @@ def make_mask(self):
             f'{self._img_prefix}'
             + f'{self._outname_base}{self._img_number}.fits'
         )
-        if (
-            os.path.exists(f"output/run_sp_MaMa_2023-11-21_06-11-21/mask_runner_run_1/output/{output_file_name}")
-            or os.path.exists(f"output/run_sp_MaMa_2023-11-21_06-11-21/mask_runner_run_2/output/{output_file_name}")
-        ):
-            print("MKDEBUG skipping ", output_file_name)
-            return None, None
-        else:
-            print("MKDEBUG processing ", output_file_name)
-
 
         if self._config['MD']['make']:
             self.missing_data()

From b682e7c2b74775489b4929f0982da2fb8cbf759d Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 21 Dec 2023 15:51:13 +0100
Subject: [PATCH 061/133] removed unused code from mask

---
 shapepipe/modules/mask_package/mask.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/shapepipe/modules/mask_package/mask.py b/shapepipe/modules/mask_package/mask.py
index b8b81a3cc..e34eb9583 100644
--- a/shapepipe/modules/mask_package/mask.py
+++ b/shapepipe/modules/mask_package/mask.py
@@ -308,11 +308,6 @@ def make_mask(self):
         Main function to create the mask.
 
         """
-        output_file_name = (
-            f'{self._img_prefix}'
-            + f'{self._outname_base}{self._img_number}.fits'
-        )
-
         if self._config['MD']['make']:
             self.missing_data()
 

From 7fc3645751dc05ccb4ab43ec1d7399f82577acb2 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sat, 23 Dec 2023 11:01:57 +0000
Subject: [PATCH 062/133] curl scripts updated

---
 scripts/python/link_to_exp_for_tile.py  |   2 +-
 scripts/python/summary_run.py           |   6 +-
 scripts/sh/curl_canfar_local.sh         |   2 +-
 scripts/sh/curl_canfar_monitor_local.sh |   8 +-
 scripts/sh/init_run_exclusive_canfar.sh | 175 ++++++++++++++----------
 scripts/sh/job_sp_canfar.bash           |  16 ++-
 6 files changed, 122 insertions(+), 87 deletions(-)

diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index f6e01021b..9d19003d8 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -327,7 +327,7 @@ def main(argv=None):
     patterns = ["run_sp_exp_SxSePsf", "run_sp_exp_Pi"]
     for pattern in patterns:
         paths, number = get_paths(exp_base_dir, exp_shdu_IDs, pattern)
-        print(number)
+        #print(number)
 
         create_links_paths(tile_base_dir, tile_ID, paths, verbose=verbose)
 
diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index 462467e33..df44a7be7 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -24,7 +24,7 @@ def main(argv=None):
     list_tile_IDs_dot = get_IDs_from_file(tile_ID_path)
 
     # tile IDs with dashes
-    list_tile_IDs = replace_dot_dash(list_tile_IDs_dot)
+    list_tile_IDs = job_data.replace_dot_dash(list_tile_IDs_dot)
     n_tile_IDs = len(list_tile_IDs)
     n_CCD = 40
 
@@ -168,8 +168,8 @@ def main(argv=None):
 
     print_par_runtime(par_runtime, verbose=verbose)
 
-    #for key in ["2", "4", "8", "16", "32", "64", "128"]:
-    for key in ["128"]:
+    for key in ["2", "4", "8", "16", "32", "64", "128"]:
+    #for key in ["128"]:
         job = jobs[key]
         job.print_intro()
         job.check_numbers(par_runtime=par_runtime)
diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index 3ec61555d..bacf09576 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#!/usr/bin/env bash
 
 # Usage
 # ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.9 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID kind job
diff --git a/scripts/sh/curl_canfar_monitor_local.sh b/scripts/sh/curl_canfar_monitor_local.sh
index 8260e583d..4ba0bf875 100755
--- a/scripts/sh/curl_canfar_monitor_local.sh
+++ b/scripts/sh/curl_canfar_monitor_local.sh
@@ -1,10 +1,4 @@
-#!/usr/bin/bash
-
-# --false-start --tcp-fastopen  faster
-
-# --http2   slower
-# --tlsv1.2 slower?
-# --insecure ?
+#!/usr/bin/env bash
 
 # -H "Accept-Encoding: gzip"    faster?
 
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 2dbcf41a5..3f1559c4b 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -10,6 +10,7 @@ N_SMP=1
 kind=-1
 dry_run=0
 nsh_jobs=8
+VERBOSE=1
 
 
 # TODO: psf
@@ -83,6 +84,44 @@ if [ "$kind" == "-1" ]; then
   exit 4
 fi
 
+# Functions
+
+## Print string, executes command, and prints return value.
+function command () {
+   cmd=$1
+   dry_run=$2
+
+   RED='\033[0;31m'
+   GREEN='\033[0;32m'
+   NC='\033[0m' # No Color
+   # Color escape characters show up in log files
+   #RED=''
+   #GREEN=''
+   #NC=''
+
+   if [ $VERBOSE == 1 ]; then
+        echo "running '$cmd' (dry run=$dry_run)"
+   fi
+   if [ "$dry_run" == "0" ]; then
+        $cmd
+        res=$?
+
+        if [ $VERBOSE == 1 ]; then
+            if [ $res == 0 ]; then
+              echo -e "${GREEN}success, return value = $res${NC}"
+            else
+              echo -e "${RED}error, return value = $res${NC}"
+              if [ $STOP == 1 ]; then
+                  echo "${RED}exiting  $(basename "$0")', error in command '$cmd'${NC}"
+                  exit $res
+              else
+                  echo "${RED}continuing '$(basename "$0")', error in command '$cmd'${NC}"
+              fi
+            fi
+        fi
+   fi
+}
+
 echo "start init_run_exclusive_canfar"
 
 if [ "$dry_run" == 1 ]; then
@@ -97,100 +136,88 @@ basedir=$HOME/cosmostat/P3_v2/psfex
 cd $basedir
 
 
-if [ "$dry_run" == "0" ]; then
-
-  # Update links to exposure run directories
-  if [ "$kind" == "tile" ]; then
-    rm -rf tile_runs/$ID/output/run_exp_SxSePsf*
-    link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs
-  fi
+# Update links to exposure run directories
+if [ "$kind" == "tile" ]; then
+  command "rm -rf tile_runs/$ID/output/run_exp_SxSePsf*" $dry_run
+  command "link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs" $dry_run
+fi
 
-  cd ${kind}_runs
 
-  if [ ! -d "$ID" ]; then
-    mkdir $ID
-  fi
+cd ${kind}_runs
 
-  cd $ID
-  pwd
+if [ ! -d "$ID" ]; then
+  command "mkdir $ID" $dry_run
+fi
 
-  if [ ! -d "output" ]; then
-    mkdir output
-  fi
+cd $ID
+pwd
 
-  cd output
+if [ ! -d "output" ]; then
+  command "mkdir output" $dry_run
+fi
 
-  if [ ! -f log_exp_headers.sqlite ]; then
-    ln -s $basedir/output/log_exp_headers.sqlite
-  fi
+cd output
 
-  # Remove potentially obsolete link
-  #rm run_sp_exp_SpMh*
-  #rm  run_sp_MaMa_*
+if [ ! -f log_exp_headers.sqlite ]; then
+  command "ln -s $basedir/output/log_exp_headers.sqlite" $dry_run
+fi
 
-  # Update links to global run directories (GiFeGie, Uz, Ma?, combined_flag?)
-  for dir in $basedir/output/run_sp_*; do
-    ln -sf $dir
-  done
-  if [ ! -e run_sp_combined_flag ]; then
-    ln -s $basedir/output/run_sp_combined_flag
-  fi
+# Remove potentially obsolete link
+#rm run_sp_exp_SpMh*
+#rm  run_sp_MaMa_*
 
-  (( do_job= $job & 128 ))
-  if [[ $do_job != 0 ]]; then
+# Update links to global run directories (GiFeGie, Uz, Ma?, combined_flag?)
+for dir in $basedir/output/run_sp_*; do
+  command "ln -sf $dir" $dry_run
+done
+if [ ! -e run_sp_combined_flag ]; then
+  command "ln -s $basedir/output/run_sp_combined_flag" $dry_run
+fi
 
-    # Indentify and remove unfinished ngmix dirs
-    min_n_out=2
-    for k in $(seq 1 $nsh_jobs); do
-      ngmix_run="run_sp_tile_ngmix_Ng${k}u/ngmix_runner"
-      if [ -e "$ngmix_run" ]; then
-        ngmix_out="$ngmix_run/output"
-        n_out=`ls -rlt $ngmix_out | wc -l`
-        if [ "$n_out" -lt "$min_n_out" ]; then
-            min_n_out=$n_out
-        fi
-        #echo $k $n_out $min_n_out
-      else
-        echo "ngmix separated run #$k not found"
-        min_n_out=0
+(( do_job= $job & 128 ))
+#if [[ $do_job != 0 ]]; then
+if [ 0 == 1 ]; then
+
+  # Indentify and remove unfinished ngmix dirs
+  min_n_out=2
+  for k in $(seq 1 $nsh_jobs); do
+    ngmix_run="run_sp_tile_ngmix_Ng${k}u/ngmix_runner"
+    if [ -e "$ngmix_run" ]; then
+      ngmix_out="$ngmix_run/output"
+      n_out=`ls -rlt $ngmix_out | wc -l`
+      if [ "$n_out" -lt "$min_n_out" ]; then
+          min_n_out=$n_out
       fi
-    done
-    if [ "$min_n_out" -lt "2" ]; then
-      echo "At least one ngmix separated run no output files"
-      for k in $(seq 1 $nsh_jobs); do
-        cmd="rm -rf run_sp_tile_ngmix_Ng${k}u"
-        echo $cmd
-        `$cmd`
-      done
+      #echo $k $n_out $min_n_out
     else
-      if [ "$job" == "128" ]; then
-        echo "ngmix found complete, all is well, exiting"
-        exit 0
-      fi
+      echo "ngmix separated run #$k not found"
+      min_n_out=0
     fi
+  done
+  if [ "$min_n_out" -lt "2" ]; then
+    echo "At least one ngmix separated run no output files"
+    for k in $(seq 1 $nsh_jobs); do
+      command "rm -rf run_sp_tile_ngmix_Ng${k}u" $dry_run
+    done
+  else
+    if [ "$job" == "128" ]; then
+      echo "ngmix found complete, all is well, exiting"
+      exit 0
+    fi
+  fi
 
-  fi 
-
-  cd ..
-  update_runs_log_file.py
+fi 
 
-  echo -n "pwd: "
-  pwd
+cd ..
+command update_runs_log_file.py $dry_run
 
-fi
+echo -n "pwd: "
+pwd
 
 echo -n "environment: "
 echo $CONDA_PREFIX
 
-cmd="job_sp_canfar.bash -p psfex -j $job -e $ID --n_smp $N_SMP"
-echo -n "Running commnd '$cmd' ..."
-
-if [ "$dry_run" == 1 ]; then
-  echo " dry run"
-else
-  echo
-  $cmd
-fi
+command "job_sp_canfar.bash -p psfex -j $job -e $ID --n_smp $N_SMP" $dry_run
 
 cd $basedir
 
diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
index 97280f374..17695f41d 100755
--- a/scripts/sh/job_sp_canfar.bash
+++ b/scripts/sh/job_sp_canfar.bash
@@ -453,7 +453,21 @@ if [[ $do_job != 0 ]]; then
   ### Shapes, run $nsh_jobs parallel processes
   VERBOSE=0
   for k in $(seq 1 $nsh_jobs); do
-      command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" "Run shapepipe (tile: ngmix+galsim $k)" &
+
+      # if output dir for subrun exists but no output: re-run
+      ngmix_run=$OUTPUT/"run_sp_tile_ngmix_Ng${k}u/ngmix_runner"
+      if [ -e "$ngmix_run" ]; then
+        ngmix_out="$ngmix_run/output"
+        n_out=`ls -rlt $ngmix_out | wc -l`
+        if [ "$n_out" -lt 2 ]; then
+          command "rm -rf $OUTPUT/run_sp_tile_ngmix_Ng${k}u" "Re-running existing empty ngmix subrun $k"
+          command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" "Run shapepipe (tile: ngmix $k)" &
+        else
+          echo "Skipping existing non-empty ngmix subrun $k"
+        fi
+      else
+        command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" "Run shapepipe (tile: ngmix $k)" &
+      fi
   done
   wait
   VERBOSE=1

From 15a4b7bf79c9a3f365741f0aeb10d1fe1d70ac47 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 09:24:05 +0000
Subject: [PATCH 063/133] combine psf validation files: preles now with
 prepare_tiles_for_final.bash

---
 example/cfis/config_exp_Pi.ini          |   2 +-
 scripts/sh/curl_canfar_local.sh         |   2 +-
 scripts/sh/init_run_exclusive_canfar.sh |  34 +++--
 scripts/sh/job_curl_canfar_local.sh     |   8 +-
 scripts/sh/prepare_star_cat.bash        |  92 -------------
 scripts/sh/prepare_tiles_for_final.bash | 171 +++++++++++++++---------
 6 files changed, 133 insertions(+), 176 deletions(-)
 delete mode 100755 scripts/sh/prepare_star_cat.bash

diff --git a/example/cfis/config_exp_Pi.ini b/example/cfis/config_exp_Pi.ini
index ae510058e..12056341b 100644
--- a/example/cfis/config_exp_Pi.ini
+++ b/example/cfis/config_exp_Pi.ini
@@ -46,7 +46,7 @@ OUTPUT_DIR = $SP_RUN/output
 [JOB]
 
 # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
-SMP_BATCH_SIZE = 2
+SMP_BATCH_SIZE = 1
 
 # Timeout value (optional), default is None, i.e. no timeout limit applied
 TIMEOUT = 96:00:00
diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index bacf09576..f17f16d31 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -14,7 +14,7 @@ job=-1
 ID=-1
 N_SMP=1
 kind=-1
-version=0.9
+version=1.0
 cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
 dry_run=0
 
diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 3f1559c4b..2d1e381d0 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -158,24 +158,29 @@ fi
 
 cd output
 
-if [ ! -f log_exp_headers.sqlite ]; then
-  command "ln -s $basedir/output/log_exp_headers.sqlite" $dry_run
-fi
+if [ 0 == 1 ]; then
 
-# Remove potentially obsolete link
-#rm run_sp_exp_SpMh*
-#rm  run_sp_MaMa_*
+  if [ ! -f log_exp_headers.sqlite ]; then
+    command "ln -s $basedir/output/log_exp_headers.sqlite" $dry_run
+  fi
+
+  # Remove potentially obsolete link
+  #rm run_sp_exp_SpMh*
+  #rm  run_sp_MaMa_*
+
+  # Update links to global run directories (GiFeGie, Uz, Ma?, combined_flag?)
+  for dir in $basedir/output/run_sp_*; do
+    command "ln -sf $dir" $dry_run
+  done
+  if [ ! -e run_sp_combined_flag ]; then
+    command "ln -s $basedir/output/run_sp_combined_flag" $dry_run
+  fi
 
-# Update links to global run directories (GiFeGie, Uz, Ma?, combined_flag?)
-for dir in $basedir/output/run_sp_*; do
-  command "ln -sf $dir" $dry_run
-done
-if [ ! -e run_sp_combined_flag ]; then
-  command "ln -s $basedir/output/run_sp_combined_flag" $dry_run
 fi
 
 (( do_job= $job & 128 ))
 #if [[ $do_job != 0 ]]; then
+# The following is now dealt with in job_sh_canar.bash
 if [ 0 == 1 ]; then
 
   # Indentify and remove unfinished ngmix dirs
@@ -217,7 +222,10 @@ pwd
 echo -n "environment: "
 echo $CONDA_PREFIX
 
-command "job_sp_canfar.bash -p psfex -j $job -e $ID --n_smp $N_SMP" $dry_run
+#command "job_sp_canfar.bash -p psfex -j $job -e $ID --n_smp $N_SMP" $dry_run
+
+export SP_RUN=.
+command "shapepipe_run -c $HOME/shapepipe/example/cfis/config_exp_Pi.ini" $dry_run
 
 cd $basedir
 
diff --git a/scripts/sh/job_curl_canfar_local.sh b/scripts/sh/job_curl_canfar_local.sh
index e8a16513f..600a15964 100644
--- a/scripts/sh/job_curl_canfar_local.sh
+++ b/scripts/sh/job_curl_canfar_local.sh
@@ -5,13 +5,13 @@
 
 rm -f session_IDs.txt session_image_IDs.txt
 
-#script_local=$HOME/astro/repositories/github/shapepipe/scripts/sh/curl_canfar_local.sh
-script_local=./curl_canfar_local.sh
-version="0.9"
+script_local=curl_canfar_local.sh
+version="1.0"
 cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
+N_SMP=1
 kind="$1"
 job="$2"
 
 echo $kind $job
-cat all.txt | xargs -n 1 -P 10 $script_local -v $version -c $cmd_remote -k $kind -j $job -e
+cat all.txt | xargs -n 1 -P 1 $script_local -v $version -c $cmd_remote -N $N_SMP -k $kind -j $job -e
 
diff --git a/scripts/sh/prepare_star_cat.bash b/scripts/sh/prepare_star_cat.bash
deleted file mode 100755
index ff8cb167f..000000000
--- a/scripts/sh/prepare_star_cat.bash
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env bash
-
-# Name: prepare_star_cat.bash
-# Description: Create directory and links to all PSF or star catalogue files
-#              from previous ShapePipe runs.
-# Author: Martin Kilbinger <martin.kilbinger@cea.fr>
-
-
-# Command line arguments
-
-## Default values
-psf='mccd'
-
-## Help string
-usage="Usage: $(basename "$0") [OPTIONS]
-\n\nOptions:\n
-   -h\tthis message\n
-   -p, --psf MODEL\n
-    \tPSF model, one in ['psfex'|'mccd'|'setools'], default='$psf'\n
-"
-
-## Parse command line
-while [ $# -gt 0 ]; do
-  case "$1" in
-    -h)
-      echo -ne $usage
-      exit 0
-      ;;
-    -p|--psf)
-      psf="$2"
-      shift
-      ;;
-    *)
-      echo -ne usage
-      exit 1
-      ;;
-  esac
-  shift
-done
-
-
-## Path variables
-if [ "$psf" == "psfex" ] || [ "$psf" == "mccd" ]; then
-  psfval_file_base="validation_psf"
-  dir_individual="psf_validation_ind"
-else
-  psfval_file_base="mask/star_selection"
-  dir_individual="star_all_ind"
-fi
-
-pwd=`pwd`
-
-
-## Functions
-function link_s () {
-    target=$1
-    link_name=$2
- 
-    if [ -L "$link_name" ]; then
-	      let "n_skipped+=1"
-    else
-        ln -s $target $link_name
-	      let "n_created+=1"
-    fi
-
-    return $n
-}
-
-
-# Create output dirs
-if [ ! -d "$dir_individual" ]; then
-    mkdir -p $dir_individual
-fi
-
-if [ "$psf" == "psfex" ]; then
-  runner="psfex_interp_runner"
-elif [ "$psf" == "mccd" ]; then
-  runner="mccd_fit_val_runner"
-else
-  runner="setools_runner"
-fi
-
-# Find all psf validation files and create links.
-# Assumes untar_results.sh has been run before.
-n_skipped=0
-n_created=0
-FILES=output/*/${runner}/output/${psfval_file_base}*
-for val in ${FILES[@]}; do
-    base=`basename $val`
-    link_s "$pwd/$val" "$dir_individual/$base"
-done
-echo " Created $n_created links, skipped $n_skipped files"
diff --git a/scripts/sh/prepare_tiles_for_final.bash b/scripts/sh/prepare_tiles_for_final.bash
index cf56d5895..8aff01e00 100755
--- a/scripts/sh/prepare_tiles_for_final.bash
+++ b/scripts/sh/prepare_tiles_for_final.bash
@@ -1,23 +1,25 @@
 #!/usr/bin/env bash
 
 # Name: prepare_tiles_for_final.bash
-# Description: Create shapepipe run directory with
-#              links to all `final_cat` fits files
+# Description: Create new shapepipe run directory with
+#              links to source files from combined existing runs
 # Author: Martin Kilbinger <martin.kilbinger@cea.fr>
-# Date: 06/2020
-# Version: 0.1
+
 
 # Command line arguments
 
 ## Default values
-cat='final'  
+cat='final'
+psf="mccd"
 
 ## Help string
 usage="Usage: $(basename "$0") [OPTIONS]
 \n\nOptions:\n
    -h\tthis message\n
+   -p, --psf MODEL\n
+    \tPSF model, one in ['psfex'|'mccd'|'setools'], default='$psf'\n
    -c, --cat TYPE\n
-    \tCatalogue type, one in ['final'|'flag'|'image'], default='$cat'\n
+    \tCatalogue type, one in ['final'|'flag'|'psf'|'image'], default='$cat'\n
 "
 
 ## Parse command line
@@ -31,6 +33,10 @@ while [ $# -gt 0 ]; do
       cat="$2"
       shift
       ;;
+    -p|--psf)
+      psf="$2"
+      shift
+      ;;
     *)
       echo -ne $usage
       exit 1
@@ -40,92 +46,127 @@ while [ $# -gt 0 ]; do
 done
 
 
+## Check options
+if [ "$cat" != "final" ] \
+  && [ "$cat" != "flag" ] \
+  && [ "$cat" != "psf" ] \
+  && [ "$cat" != "image" ]; then
+  echo "cat (option -c) needs to be 'final', 'flag', 'psf', or 'image'"
+  exit 2
+fi
+
+## Check options
+if [ "$psf" != "psfex" ] \
+  && [ "$psf" != "mccd" ] \
+  && [ "$psf" != "setools" ]; then
+  echo "PSF (option -p) needs to be 'psfex' or 'mccd'"
+  exit 2
+fi
+
+
 ## Functions
 function link_s () {
     target=$1
     link_name=$2
 
-    if [ -e "$link_name" ]; then
+    if [ -L "$link_name" ]; then
         echo "link with name $link_name already exists, skipping..."
+        let "n_skipped+=1"
     else
         echo "create link $target <- $link_name"
         ln -s $target $link_name
+        let "n_created+=1"
     fi
 }
 
-## Check options
-if [ "$cat" != "final" ] && [ "$cat" != "flag" ] && [ "$cat" != "image" ]; then
-  echo "cat (option -c) needs to be 'final', 'flag', or 'image'"
-  exit 2
-fi
 
+# Start program
 
-### Start ###
+n_skipped=0
+n_created=0
 
 pwd=`pwd`
 out_base="output"
 
+# Set paths:
+## run_out: target output new run directory
+## run_in: source input run base directory
+## module: source input module runner sub-directory
+## pattern: source file pattern
+
 if [ "$cat" == "final" ]; then
-  run_dir="run_sp_combined"
-  INPUT="$pwd/$out_base/run_sp_Mc_*"
+
+  run_out="run_sp_combined"
+  run_in="$pwd/$out_base/run_sp_Mc_*"
+  module="make_catalog_runner"
+  pattern="final_cat-*"
+
 elif [ "$cat" == "flag" ]; then
-  run_dir="run_sp_combined_flag"
-  INPUT="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
+
+  run_out="run_sp_combined_flag"
+  run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
+  module="mask_runner_run_1"
+  pattenr="pipeline_flag-*"
+
+elif [ "$cat" == "image" ]; then
+
+  run_out="run_sp_combined_image"
+  run_in="$pwd/$out_base/run_sp_Git_*"
+  module="get_images_runner"
+  pattern="CFIS_image-*"
+
+elif [ "$cat" == "psf" ]; then
+
+  run_out="run_sp_combined_psf"
+
+  #MKDEBUG TODO: add option
+  # v1
+  #run_in="$pwf/$out_base/run_sp_exp_Pi_*"
+  # v2
+  run_in="exp_runs/*/$out_base/"
+
+  pattern="validation_psf-*"
+  if [ "$psf" == "psfex" ]; then
+    module="psfex_interp_runner"
+  elif [ "$psf" == "setools" ]; then
+    module="setools_runner"
+  else
+    module="mccd_interp_runner"
+  fi
+
 else
-  run_dir="run_sp_combined_image"
-  INPUT="$pwd/$out_base/run_sp_Git_*"
+
+  echo "Invalid catalogue type $cat"
+  exit 2
+
 fi
 
-log_path="$pwd/$out_base/log_run_sp.txt"
-OUTPUT="$pwd/$out_base/$run_dir"
+OUTPUT="$pwd/$out_base/$run_out"
 mkdir -p $OUTPUT
 
-# Directories and file patterns to create/link
-if [ "$cat" == "final" ]; then
-  DIRS=(
-	  "make_catalog_runner"
-  )
-  PATTERNS=(
-	  "final_cat-*"
-  )
-elif [ "$cat" == "flag" ]; then
-  DIRS=(
-<<<<<<< HEAD
-	  "mask_runner_run_1"
-=======
-	  "mask_runner_run_2"
->>>>>>> bd82f95ae96043c2de6ac577dae311c74a03e133
-  )
-  PATTERNS=(
-	  "pipeline_flag-*"
-  )
-else
-  DIRS=(
-    "get_images_runner"
-  )
-  PATTERNS=(
-	  "CFIS_image-*"
-  )
-fi
 
 # Create links
-for n in "${!PATTERNS[@]}"; do
-    pattern=${PATTERNS[$n]}
-    dir=$OUTPUT/${DIRS[$n]}/output
-    echo $n $pattern $dir
-    mkdir -p $dir
-    FILES=(`find $INPUT -name "$pattern"`)
-    n_files=${#FILES[@]}
-    i=0
-    for file in ${FILES[@]}; do
-	    target=$file
-	    link_name=$dir/`basename $file`
-	    link_s $target $link_name
-        ((i=i+1))
-    done
-    echo " $n_files target files, $i links created/skipped"
+
+## target directory
+outdir=$OUTPUT/$module/output
+mkdir -p $outdir
+
+## identify source files
+FILES=(`find $run_in -name "$pattern"`)
+n_files=${#FILES[@]}
+
+## Look over source files
+i=0
+for file in ${FILES[@]}; do
+
+ target=$file
+ link_name=$outdir/`basename $file`
+ link_s $target $link_name
+ ((i=i+1))
+
 done
 
+echo " $n_files target files, $i links created/skipped"
+
 # Update log file
-modules=`echo ${DIRS[@]} | tr ' ' ,`
-echo "./$out_base/$run_dir $modules" >> $log_path
+update_runs_log_file.py

From 0d56e23b1e0cab77b6a5ed66cc7693d307e51d16 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:35:20 +0100
Subject: [PATCH 064/133] Update post_processing.md

post processing doc: linked to (new) vos_retrieve file
---
 docs/source/post_processing.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/source/post_processing.md b/docs/source/post_processing.md
index ebd123889..017afa15f 100644
--- a/docs/source/post_processing.md
+++ b/docs/source/post_processing.md
@@ -3,10 +3,13 @@
 This page shows all required steps of post-processing the results from one or
 more `ShapePipe` runs. Post-processing combines various individual `ShapePipe`
 output files, and creates joint results, for example combining individual tile
-catalogues in a large sky area. The output of post-processing is a joint _shape
+catalogues into a large sky area. The output of post-processing is a joint _shape
 catalogue_, containing all required information to create a calibrated shear
 catalogue via _metacalibration_), a joint star catalogue, and PSF diagnostic plots.
 
+If main ShapePipe processing happened at the old canfar VM system (e.g. CFIS v0 and v1), go
+[here|vos_retrieve.md] for details how to retrieve the ShapePipe output files.
+
 Some of the following steps pertain specifically to runs carried out on [canfar](https://www.canfar.net/en),
 but most are general.
 

From fde5982c147585bededb4d1600330c4a71c3c3c6 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 14:40:42 +0000
Subject: [PATCH 065/133] prepare tiles script loop tests

---
 scripts/sh/prepare_tiles_for_final.bash | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scripts/sh/prepare_tiles_for_final.bash b/scripts/sh/prepare_tiles_for_final.bash
index 8aff01e00..aa2b755c2 100755
--- a/scripts/sh/prepare_tiles_for_final.bash
+++ b/scripts/sh/prepare_tiles_for_final.bash
@@ -123,7 +123,7 @@ elif [ "$cat" == "psf" ]; then
   # v1
   #run_in="$pwf/$out_base/run_sp_exp_Pi_*"
   # v2
-  run_in="exp_runs/*/$out_base/"
+  run_in="$pwd/exp_runs/*/$out_base/"
 
   pattern="validation_psf-*"
   if [ "$psf" == "psfex" ]; then
@@ -152,12 +152,15 @@ outdir=$OUTPUT/$module/output
 mkdir -p $outdir
 
 ## identify source files
-FILES=(`find $run_in -name "$pattern"`)
-n_files=${#FILES[@]}
+#FILES=(`find $run_in -type f -name "$pattern" -print0 | xargs -0 echo`)
+#n_files=${#FILES[@]}
 
 ## Look over source files
 i=0
-for file in ${FILES[@]}; do
+#for file in ${FILES[@]}; do
+
+find $run_in -type f -name "$pattern" | while IFS= read -r file; do
+
 
  target=$file
  link_name=$outdir/`basename $file`
@@ -166,7 +169,8 @@ for file in ${FILES[@]}; do
 
 done
 
-echo " $n_files target files, $i links created/skipped"
+#echo " $n_files target files, $i links created/skipped"
+echo " $i total, "n_skipped skipped, "n_created links created"
 
 # Update log file
 update_runs_log_file.py

From ac0a3399cceec843f9cde3c869793c82bd0d7f3f Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:42:00 +0100
Subject: [PATCH 066/133] curl canfar local script minor change

---
 scripts/sh/job_curl_canfar_local.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/sh/job_curl_canfar_local.sh b/scripts/sh/job_curl_canfar_local.sh
index e8a16513f..7e4f0206d 100644
--- a/scripts/sh/job_curl_canfar_local.sh
+++ b/scripts/sh/job_curl_canfar_local.sh
@@ -6,7 +6,7 @@
 rm -f session_IDs.txt session_image_IDs.txt
 
 #script_local=$HOME/astro/repositories/github/shapepipe/scripts/sh/curl_canfar_local.sh
-script_local=./curl_canfar_local.sh
+script_local=curl_canfar_local.sh
 version="0.9"
 cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
 kind="$1"

From 4048bf17a602fee9d30b45a9a3df7dd1e93b5b75 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:54:27 +0100
Subject: [PATCH 067/133] added vos doc md file

---
 docs/source/vos_retrieve.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 docs/source/vos_retrieve.md

diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md
new file mode 100644
index 000000000..74c590b3a
--- /dev/null
+++ b/docs/source/vos_retrieve.md
@@ -0,0 +1,14 @@
+## Retrieve files from VOspace
+
+This page describes how ShapePipe output files can be retrieved via the Virtual Observatory Space
+on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete.
+
+1. Retrieve ShapePipe result files 
+
+  For a local run on the same machine as for post-processing, nothing needs to be done. In some cases, the run was carried out on a remote machine or cluster, and the resulting ShapePipe output files need to be retrieved.
+
+  In the specific case of canfar_avail_results.py, this is done as follows.
+
+    A. Check availability of results
+
+      A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check which tiles have finished running, and whose results have been uploaded, use

From 0a3f80e94333ddbbc57deee10d09b0c58948d8f0 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:55:12 +0100
Subject: [PATCH 068/133] Update post_processing.md

---
 docs/source/post_processing.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/post_processing.md b/docs/source/post_processing.md
index 017afa15f..555ffdf85 100644
--- a/docs/source/post_processing.md
+++ b/docs/source/post_processing.md
@@ -8,7 +8,7 @@ catalogue_, containing all required information to create a calibrated shear
 catalogue via _metacalibration_), a joint star catalogue, and PSF diagnostic plots.
 
 If main ShapePipe processing happened at the old canfar VM system (e.g. CFIS v0 and v1), go
-[here|vos_retrieve.md] for details how to retrieve the ShapePipe output files.
+[here](vos_retrieve.md) for details how to retrieve the ShapePipe output files.
 
 Some of the following steps pertain specifically to runs carried out on [canfar](https://www.canfar.net/en),
 but most are general.

From d81bdf2de69034c45450d5b57b2fa87e2b7b5508 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:55:45 +0100
Subject: [PATCH 069/133] Update vos_retrieve.md

---
 docs/source/vos_retrieve.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md
index 74c590b3a..9c82dbd44 100644
--- a/docs/source/vos_retrieve.md
+++ b/docs/source/vos_retrieve.md
@@ -9,6 +9,6 @@ on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete
 
   In the specific case of canfar_avail_results.py, this is done as follows.
 
-    A. Check availability of results
+  A. Check availability of results
 
-      A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check which tiles have finished running, and whose results have been uploaded, use
+    A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check which tiles have finished running, and whose results have been uploaded, use

From 672001da1cad9eb274494e2779420189c53085cf Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:56:31 +0100
Subject: [PATCH 070/133] Update vos_retrieve.md

---
 docs/source/vos_retrieve.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md
index 9c82dbd44..1527e482f 100644
--- a/docs/source/vos_retrieve.md
+++ b/docs/source/vos_retrieve.md
@@ -5,10 +5,11 @@ on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete
 
 1. Retrieve ShapePipe result files 
 
-  For a local run on the same machine as for post-processing, nothing needs to be done. In some cases, the run was carried out on a remote machine or cluster, and the resulting ShapePipe output files need to be retrieved.
+   For a local run on the same machine as for post-processing, nothing needs to be done. In some cases, the run was carried out on a remote machine or cluster, and the resulting ShapePipe output files  
+  need to be retrieved.
 
-  In the specific case of canfar_avail_results.py, this is done as follows.
+   In the specific case of canfar_avail_results.py, this is done as follows.
 
-  A. Check availability of results
+   A. Check availability of results
 
-    A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check which tiles have finished running, and whose results have been uploaded, use
+      A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check which tiles have finished running, and whose results have been uploaded, use

From cc96a0c468e167d50398ce8c5aa8d98ded34312f Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:56:49 +0100
Subject: [PATCH 071/133] Update vos_retrieve.md

---
 docs/source/vos_retrieve.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md
index 1527e482f..7beb79611 100644
--- a/docs/source/vos_retrieve.md
+++ b/docs/source/vos_retrieve.md
@@ -10,6 +10,7 @@ on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete
 
    In the specific case of canfar_avail_results.py, this is done as follows.
 
-   A. Check availability of results
+   A. Check availability of results  
 
-      A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check which tiles have finished running, and whose results have been uploaded, use
+      A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check 
+      which tiles have finished running, and whose results have been uploaded, use

From 87bb2b7b440809d2e912d66ec0cdaeefc95c50ba Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:57:11 +0100
Subject: [PATCH 072/133] Update vos_retrieve.md

---
 docs/source/vos_retrieve.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md
index 7beb79611..ff0a4ef8e 100644
--- a/docs/source/vos_retrieve.md
+++ b/docs/source/vos_retrieve.md
@@ -10,7 +10,7 @@ on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete
 
    In the specific case of canfar_avail_results.py, this is done as follows.
 
-   A. Check availability of results  
+   1. Check availability of results  
 
       A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check 
       which tiles have finished running, and whose results have been uploaded, use

From 76c08a607c3f19c21782d585c39ad431a8514297 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:58:57 +0100
Subject: [PATCH 073/133] Update vos_retrieve.md

copied vos sections
---
 docs/source/vos_retrieve.md | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md
index ff0a4ef8e..c0374cd19 100644
--- a/docs/source/vos_retrieve.md
+++ b/docs/source/vos_retrieve.md
@@ -13,4 +13,37 @@ on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete
    1. Check availability of results  
 
       A canfar job can submit a large number of tiles, whose processing time can vary a lot. We assume that the submitted tile ID list is available locally via the ascii file tile_numbers.txt. To check 
-      which tiles have finished running, and whose results have been uploaded, use
+      which tiles have finished running, and whose results have been uploaded, use 
+      ```bash
+      canfar_avail_results -i tile_numbers.txt -v -p PSF --input_path INPUT_PATH
+      ```
+      where PSF is one in [`psfex`|`mccd`], and INPUT_PATH the input path on vos, default `vos:cfis/cosmostat/kilbinger/results`.
+      See `-h` for all options.
+
+   2. Download results
+
+      All results files will be downloaded with
+      ```bash
+      canfar_download_results -i tile_numbers.txt -v -p PSF --input_vos INPUT_VOS
+      ```
+      Use the same options as for same as for `canfar_avail_results`.
+      
+      This command can be run in the same directory at subsequent times, to complete an ongoing run: Only newer files will be downloaded
+      from the `vos` directory. This also assures that partially downloaded or corrupt files will be replaced.
+
+      Checking the `vos` directorty can be slow for large patches.
+      To only download files that are not yet present locally (in `.`), first write the missing ones to an ascii file, using again the
+      script `canfar_avail_results`, but this time with `.` as input path:
+      ```bash
+      canfar_avail_results -i tile_numbers.txt --input_path . -p PSF -v -o missing.txt
+      '''
+      Then, download only the missing files with
+      ```bash
+      canfar_download_results -i missing.txt --input_vos cosmostat/kilbinger/results_mccd_oc2 -p mccd -v
+      ```
+
+   3. Un-tar results
+     ```bash
+      untar_results -p PSF
+      ```
+      On success, `ShapePipe` output `fits` and `log` files will be now in various subdirs of the `output` directory.

From faa0836e811af6f70c8525075ccc7f8c3bd373a7 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:59:14 +0100
Subject: [PATCH 074/133] Update vos_retrieve.md

---
 docs/source/vos_retrieve.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md
index c0374cd19..2d7183813 100644
--- a/docs/source/vos_retrieve.md
+++ b/docs/source/vos_retrieve.md
@@ -36,7 +36,7 @@ on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete
       script `canfar_avail_results`, but this time with `.` as input path:
       ```bash
       canfar_avail_results -i tile_numbers.txt --input_path . -p PSF -v -o missing.txt
-      '''
+      ```
       Then, download only the missing files with
       ```bash
       canfar_download_results -i missing.txt --input_vos cosmostat/kilbinger/results_mccd_oc2 -p mccd -v

From 85f6d91d11b5b52b54b44ca44823df865c740cbb Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 15:59:49 +0100
Subject: [PATCH 075/133] Update vos_retrieve.md

---
 docs/source/vos_retrieve.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/source/vos_retrieve.md b/docs/source/vos_retrieve.md
index 2d7183813..3606752c8 100644
--- a/docs/source/vos_retrieve.md
+++ b/docs/source/vos_retrieve.md
@@ -47,3 +47,5 @@ on canfar. This system was used for the CFIS v0 and v1 runs, and is now obsolete
       untar_results -p PSF
       ```
       On success, `ShapePipe` output `fits` and `log` files will be now in various subdirs of the `output` directory.
+
+At this step all required `ShapePipe` resulting output files are available in the current working directory.

From 4f816359e6c6ea47069a2d0ca9821e62de06d859 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 16:46:04 +0000
Subject: [PATCH 076/133] combine runs script renamed

---
 scripts/python/summary_run.py                 |  14 +-
 ...tiles_for_final.bash => combine_runs.bash} |  29 +-
 scripts/sh/post_proc_sp.bash                  |   4 +-
 shapepipe/utilities/summary.py                | 672 ++++++++++++++++++
 4 files changed, 701 insertions(+), 18 deletions(-)
 rename scripts/sh/{prepare_tiles_for_final.bash => combine_runs.bash} (88%)
 create mode 100755 shapepipe/utilities/summary.py

diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index df44a7be7..bfd926194 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -147,6 +147,16 @@ def main(argv=None):
         verbose=verbose,
     )
 
+
+    jobs["1024"] = job_data(
+        "1024",
+        "run_sp_combined_psf",
+        ["psfex_interp_runner"],
+        "shdus",
+        path_left=f"{main_dir}/output",
+        verbose=verbose
+    )
+
     job_data.print_stats_header()
 
     for key in "1":
@@ -168,8 +178,8 @@ def main(argv=None):
 
     print_par_runtime(par_runtime, verbose=verbose)
 
-    for key in ["2", "4", "8", "16", "32", "64", "128"]:
-    #for key in ["128"]:
+    #for key in ["2", "4", "8", "16", "32", "64", "128"]:
+    for key in ["1024"]:
         job = jobs[key]
         job.print_intro()
         job.check_numbers(par_runtime=par_runtime)
diff --git a/scripts/sh/prepare_tiles_for_final.bash b/scripts/sh/combine_runs.bash
similarity index 88%
rename from scripts/sh/prepare_tiles_for_final.bash
rename to scripts/sh/combine_runs.bash
index aa2b755c2..b26ab39b5 100755
--- a/scripts/sh/prepare_tiles_for_final.bash
+++ b/scripts/sh/combine_runs.bash
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Name: prepare_tiles_for_final.bash
+# Name: combine_runs.bash
 # Description: Create new shapepipe run directory with
 #              links to source files from combined existing runs
 # Author: Martin Kilbinger <martin.kilbinger@cea.fr>
@@ -94,31 +94,28 @@ out_base="output"
 ## module: source input module runner sub-directory
 ## pattern: source file pattern
 
+run_out="run_sp_combined_$cat"
+
 if [ "$cat" == "final" ]; then
 
-  run_out="run_sp_combined"
   run_in="$pwd/$out_base/run_sp_Mc_*"
   module="make_catalog_runner"
   pattern="final_cat-*"
 
 elif [ "$cat" == "flag" ]; then
 
-  run_out="run_sp_combined_flag"
   run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
   module="mask_runner_run_1"
   pattenr="pipeline_flag-*"
 
 elif [ "$cat" == "image" ]; then
 
-  run_out="run_sp_combined_image"
   run_in="$pwd/$out_base/run_sp_Git_*"
   module="get_images_runner"
   pattern="CFIS_image-*"
 
 elif [ "$cat" == "psf" ]; then
 
-  run_out="run_sp_combined_psf"
-
   #MKDEBUG TODO: add option
   # v1
   #run_in="$pwf/$out_base/run_sp_exp_Pi_*"
@@ -141,6 +138,7 @@ else
 
 fi
 
+
 OUTPUT="$pwd/$out_base/$run_out"
 mkdir -p $OUTPUT
 
@@ -152,20 +150,23 @@ outdir=$OUTPUT/$module/output
 mkdir -p $outdir
 
 ## identify source files
+
+# The following can result in an "Argument list too long" error
 #FILES=(`find $run_in -type f -name "$pattern" -print0 | xargs -0 echo`)
-#n_files=${#FILES[@]}
 
-## Look over source files
 i=0
-#for file in ${FILES[@]}; do
+for dir in $run_in; do
+  FILES=(`find $dir -type f -name "$pattern" -print0 | xargs -0 echo`)
 
-find $run_in -type f -name "$pattern" | while IFS= read -r file; do
+  ## Look over source files
+  for file in ${FILES[@]}; do
 
+  target=$file
+  link_name=$outdir/`basename $file`
+  link_s $target $link_name
+  ((i=i+1))
 
- target=$file
- link_name=$outdir/`basename $file`
- link_s $target $link_name
- ((i=i+1))
+  done
 
 done
 
diff --git a/scripts/sh/post_proc_sp.bash b/scripts/sh/post_proc_sp.bash
index 72c620e68..930d5b79d 100755
--- a/scripts/sh/post_proc_sp.bash
+++ b/scripts/sh/post_proc_sp.bash
@@ -70,7 +70,7 @@ SP_CONFIG=$SP_BASE/example/cfis
 # PSF
 
 ## Collect all psfinterp results 
-prepare_star_cat -p $psf
+combine_runs -p $psf -t psf
 
 ## Merge all psfinterp results and compute PSF residuals
 shapepipe_run -c $SP_CONFIG/config_MsPl_$psf.ini 
@@ -79,7 +79,7 @@ shapepipe_run -c $SP_CONFIG/config_MsPl_$psf.ini
 # Galaxies
 
 ## Prepare output directory with links to all 'final_cat' result files
-prepare_tiles_for_final
+combine_runs
 
 ## Merge final output files to single mother catalog
 input_final=output/run_sp_combined/make_catalog_runner/output
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
new file mode 100755
index 000000000..ac69c86a1
--- /dev/null
+++ b/shapepipe/utilities/summary.py
@@ -0,0 +1,672 @@
+"""SUMMARY
+
+Author: Martin Kilbinger <martin.kilbinger@cea.fr>
+
+"""
+
+import sys
+import os
+import re
+import fnmatch
+
+import logging
+
+from collections import Counter
+
+from tqdm import tqdm
+
+
+def get_IDs_from_file(path):
+    """Get IDs From File.
+
+    Return IDs from text file. Removes letters and replaces
+    dots "." with dashes "-".
+
+    Parameters
+    ----------
+    path: str
+        input file path
+
+    Returns
+    --------
+    list
+        IDs
+
+    """
+    numbers = []
+    with open(path) as f_in:
+        for line in f_in:
+            entry = line.rstrip()
+            number = re.sub("[a-zA-Z]", "", entry)
+            numbers.append(number)
+
+    return numbers
+
+
+
+def get_all_exposures(exp_number_file_list, verbose=False):
+    """Get All Exposures.
+
+    Return all exposure names from a list of text files.
+
+    Parameters
+    ----------
+    exp_number_list: list
+        input file names
+
+    """
+    exposures = set()
+    for idx, path in enumerate(exp_number_file_list):
+        exps = get_IDs_from_file(path)
+        exposures.update(exps)
+
+    return list(exposures)
+
+
+def get_all_shdus(exposures, n_CCD):
+    """Get All SHDUs.
+
+    Return all single-exposure single-HDU (CCD) IDs.
+
+    Parameters
+    ----------
+    exposures: list
+        exposure names
+    n_CCD: int
+        number of CCDs per exposure
+
+    Returns
+    --------
+    list
+        single-exposure single-HDU IDs
+
+    """
+    shdus = []
+    for exposure in exposures:
+        for idx_CCD in range(n_CCD):
+            shdus.append(f"{exposure}-{idx_CCD}")
+
+    return shdus
+
+
+def set_as_list(item=None, n=None, default=1):
+    """Set As List.
+
+    Return input as list.
+
+    Parameters
+    -----------
+    item: str, int, or list, optional
+        input item(s); default is None, in which
+        case the return is [1] * n
+    n: int, optional
+        number of list items to return, default is None,
+        in which case the number will be set to 1. If item and
+        n are not None, n has to be equal to len(item)
+    default: int, optional
+        value to return if item is not given;
+        default is 1
+
+    Raises
+    -------
+    IndexError
+        if n != len(item)
+
+    Returns
+    -------
+    list
+        input item(s) as list
+    """
+    my_n = n or 1
+
+    if not item:
+        result = [default] * my_n
+    elif not isinstance(item, list):
+        result = [item] * my_n
+    else:
+        result = item
+        if len(item) != my_n:
+            raise IndexError(f"item has length {len(item)} != {n}")
+
+    return result
+
+
+def check_special(module, paths_in_dir, names_in_dir):
+
+    if module == "setools_runner":
+        inds_special = []
+        for idx in range(len(paths_in_dir)):
+            base_path = paths_in_dir[idx].replace(names_in_dir[idx], "")
+
+            stats_dir = f"{base_path}/../stat"
+            stats_files = os.listdir(stats_dir)
+            if len(stats_files) != 1:
+                raise ValueError(
+                    f"Expected exactly one stats file in {stats_dir}, not"
+                    + f" {len(stats_files)}"
+                )
+
+            stats_path = os.path.join(stats_dir, stats_files[0])
+            with open(stats_path) as f_in:
+                lines = f_in.readlines()
+                for line in lines:
+                    entry = line.rstrip()
+                    m = re.search(line, "Nb stars = (\S*)")
+                    if m:
+                        value = int(m[2])
+                        if value == 0:
+                            inds_special.append(idx)
+                        else:
+                            print(f"b stars = {value}, not special")
+                        break
+
+        print(inds_special)
+        for idx in inds_special:
+            paths_in_dir.pop(idx)
+            names_in_dir.pop(idx)
+
+        return paths_in_dir, names_in_dir, len(inds_special)
+
+
+class job_data(object):
+    """Job Data.
+
+    Class to handle a job.
+
+    Parameters
+    ----------
+    bit: int
+        bit-coded job number
+    run_dir: str or list
+        run directory(ies)
+    modules: list
+        module names
+    key_expected: int or str
+        number of expected output files; if str: will be updated
+        with runtime value
+    n_mult: int or list, optional
+        multiplicity of output files, default `None`, in which
+        case it is set to 1
+    pattern: list, optional
+        if not None, file pattern to match; defafult is `None`
+    path_lef: str, optional
+        left (first) part of output directory, defaul is "./output"
+    output_subdirs: str, optional
+        output subdirectories if not `None`; default is `None`
+    path_right: str, optional
+        right (last) part of output subdir suffix if not `None`;
+        default is `None`
+    output_path_missing_IDs: list, optional
+        output path of missing ID, if `None` (default) will be
+        given by job bit and module.
+    verbose: bool, optional
+        verbose output if True; default is False
+
+    """
+    def __init__(
+        self,
+        bit,
+        run_dir,
+        modules,
+        key_expected,
+        n_mult=None,
+        pattern=None,
+        path_left="./output",
+        output_subdirs=None,
+        path_right=None,
+        output_path_missing_IDs=None,
+        verbose=False,
+    ):
+        self._bit = bit
+        self._run_dir = set_as_list(item=run_dir, n=len(modules))
+        self._modules = modules
+        self._key_expected = set_as_list(item=key_expected, n=len(modules))
+        self._n_mult = set_as_list(item=n_mult, n=len(modules))
+        self._pattern = set_as_list(item=pattern, n=len(modules), default="")
+        self._path_left = path_left
+        self._output_subdirs = output_subdirs or [""]
+        self._path_right = set_as_list(
+            path_right, len(modules), default="."
+        )
+        self._output_path_missing_IDs=output_path_missing_IDs
+        self._verbose = verbose
+
+    def print_intro(self):
+        """Print Intro.
+
+        Print header line for job statistics.
+
+        """
+        logging.info(f" (Job {self._bit})")
+
+    @classmethod
+    def print_stats_header(self):
+        """Print Stats Header.
+
+        Print overall header information for stats output.
+
+        """
+        logging.info(
+            "module                          expected     found   miss_expl"
+            + " missing uniq_miss  fr_found"
+        )
+        logging.info("=" * 100)
+
+    @classmethod
+    def print_stats(
+        self,
+        module,
+        n_expected,
+        n_found,
+        n_missing_explained,
+        n_missing,
+        n_mult,
+    ):
+        """Print Stats.
+
+        Print output file statistics.
+
+        Parameters
+        ----------
+        module: str
+            module name
+        n_expected: int
+            number of expected files
+        n_found: int
+            number of found files
+        n_missing_explained: int
+            number of missing but explained files
+        n_missing: int
+            number of missing files
+        n_mult: int
+            multipicity
+
+        """
+        if n_expected > 0:
+            fraction_found = n_found / n_expected
+        else:
+            fraction_found = 1
+
+        n_missing_per_mult = n_missing / n_mult
+
+        logging.info(
+            f"{module:30s} {n_expected:9d} {n_found:9d}"
+            + f" {n_missing_explained:9d} {n_missing:9d}"
+            + f" {n_missing_per_mult:9.1f} {fraction_found:9.1%}"
+        )
+
+    @classmethod
+    def is_ID_in_str(self, ID, path):
+        if ID in path:
+            return True
+        #if re.sub("\.", "-", ID) in path:
+            #return True
+        #return False
+
+    @classmethod
+    def is_not_in_any(self, ID, list_str):
+        return not any(ID in string for string in list_str)
+
+    @classmethod
+    def replace_dot_dash(self, numbers):
+
+        results = [re.sub("\.", "-", number) for number in numbers]
+
+        return results
+
+    @classmethod
+    def replace_dash_dot_if_tile(self, numbers):
+
+        pattern = re.compile(r"(\d{3})-(\d{3})")
+        results = [pattern.sub(r"\1.\2", number) for number in numbers]
+    
+        return results
+
+    @classmethod
+    def get_unique(self, names):
+        n_all = len(names)
+        names_unique = list(set(names))
+        n_unique = len(names_unique)
+
+        if n_all != n_unique:
+            if True:  # self._verbose:
+                logging.warning(
+                    f"{n_all - n_unique} duplicates removed from {n_all} IDs"
+                )
+
+        return names_unique
+
+    @classmethod
+    def write_IDs_to_file(self, output_path, IDs):
+        """Write IDs to file.
+
+        Write list if image IDs to text file.
+
+        Parameters
+        ----------
+        output_path: str
+            output file path
+        IDs: list
+            image IDs
+
+        """
+        IDs_dot = self.replace_dash_dot_if_tile(IDs)
+        with open(output_path, "w") as f_out:
+            for ID in IDs_dot:
+                print(ID, file=f_out)
+
+    def output_missing(
+        self,
+        module,
+        idx,
+        par_runtime=None,
+    ):
+        """Output Missing.
+
+        Writes IDs of missing images to disk.
+
+        """
+        key_expected = self._key_expected[idx]
+        names_in_dir = self._names_in_dir[idx]
+        paths_in_dir = self._paths_in_dir[idx]
+        n_mult = self._n_mult[idx]
+
+        list_expected = get_par_runtime(par_runtime, key_expected, kind="list")
+        
+        # Count image IDs in names that were found earlier
+
+        ## Extract image IDs from names
+        IDs = []
+        pattern = re.compile(r"(?:\d{3}-\d{3}|\d{7}-\d+)")
+        for name, path in zip(names_in_dir, paths_in_dir):
+            match = pattern.search(name)
+            if match:
+                IDs.append(match.group())
+            else:
+                raise ValueError(f"No ID found in {name}")
+
+
+        ## Count occurences
+        ID_counts = Counter(IDs)
+
+        ## Add to missing if ocurence less than n_mult
+        missing_IDs = []
+        for ID in list_expected:
+            if ID_counts[ID] < n_mult:
+                missing_IDs.append(ID)
+
+        n_all = len(missing_IDs)
+        missing_IDs_unique = self.get_unique(missing_IDs)
+        n_unique = len(missing_IDs_unique)
+
+        if n_unique > 0:
+            if not self._output_path_missing_IDs:
+                output_path = f"missing_job_{self._bit}_{module}.txt"
+            else:
+                output_path = self._output_path_missing_IDs[idx]
+
+        self.write_IDs_to_file(output_path, missing_IDs_unique)
+
+        return missing_IDs_unique
+
+    def output_missing_job(self):
+        output_path = f"missing_job_{self._bit}_all.txt"
+
+        missing_IDs_all = set(self._missing_IDs_job)
+
+        if len(missing_IDs_all) > 0:
+            self.write_IDs_to_file(output_path, missing_IDs_all)
+        else:
+            logging.warning("no missing IDs in output_missing_job")
+
+    @classmethod
+    def get_last_full_path(self, base_and_subdir, matches):
+        """Get Last Full Path
+
+        Return full path of last file in list.
+
+        """
+        # Sort according to creation time
+        matches_sorted = sorted(
+            matches,
+            key=lambda entry: entry.name,
+        )
+
+        # Get most recent one
+        last = matches_sorted[-1]
+
+        # Get full path
+        full_path = os.path.join(base_and_subdir, last.name)
+
+        return full_path
+
+    @classmethod
+    def get_module_output_dir(self, full_path, module):
+        """Get Module Output Dir.
+
+        Return output directory name for given module.
+
+        """
+        directory = f"{full_path}/{module}/output"
+
+        # Some modules have special requirements 
+        if module == "setools_runner":
+            directory = f"{directory}/rand_split"
+
+        return directory
+
+    def get_matches_final(self, directory, idx):
+
+        # Look over files
+        # os.path.whether exists is twice faster than try/except
+
+        if os.path.exists(directory):
+            pattern =  f"{self._pattern[idx]}*"
+            for entry2 in os.scandir(directory):
+                if (
+                    entry2.is_file()
+                    and (
+                        fnmatch.fnmatch(entry2.name, pattern)
+                    )
+                ):
+                    # Append matching files
+                    self._names_in_dir[idx].append(entry2.name)
+                    self._paths_in_dir[idx].append(
+                        os.path.join(directory, entry2.name)
+                    )
+
+        #if os.path.exists(directory):
+            #with os.scandir(directory) as entries2:
+                #files = [
+                    #entry2.name
+                    #for entry2 in entries2
+                    #if entry2.name.startswith(self._pattern[idx])
+                #]
+
+                ## Append matching files
+                #self._names_in_dir[idx].extend(files)
+                #self._paths_in_dir[idx].extend(
+                    #[os.path.join(directory, file)
+                    #for file in files]
+                #)
+
+    def get_names_in_dir(self, iterable, module, idx):
+
+        # Initialise output file names and paths
+        self._names_in_dir[idx] = []
+        self._paths_in_dir[idx] = []
+
+        # Loop over subdirs
+        for jdx, subdir in enumerate(iterable):
+            base_and_subdir = (
+                f"{self._path_left}/{subdir}/"
+                + f"{self._path_right[idx]}"
+            )
+            print("MKDEBUG ", base_and_subdir)
+            if os.path.isdir(base_and_subdir):
+
+                matches = []
+
+                # Loop over entries (files and dirs)
+                with os.scandir(base_and_subdir) as entries:
+                    for entry in entries:
+
+                        print("MKDEBUG ", entry, self._run_dir[idx])
+                        # Append directory name if matches module
+                        if entry.name.startswith(self._run_dir[idx]):
+                            print("MKDEBUG matches")
+                            matches.append(entry)
+
+                    # This entry does not match module -> next
+                    if not matches:
+                        continue
+
+                    full_path = self.get_last_full_path(
+                        base_and_subdir, matches
+                    )
+
+                    # Get module output directory
+                    directory = self.get_module_output_dir(
+                        full_path, module
+                    )
+                    print("MKDEBUG ", directory)
+
+                    # Find matching file names and paths
+                    self.get_matches_final(directory, idx)
+            else:
+                print(f"Directory base_and_subdir {base_and_subdir} not found")
+
+    def update_subdirs(self, par_runtime):
+        """Update Subdirs.
+
+        Update subdir names with runtime information if required.
+
+        """
+        if not isinstance(self._output_subdirs, list):
+            self._output_subdirs = get_par_runtime(
+                par_runtime, self._output_subdirs, kind="list"
+            )
+
+    def check_numbers(self, par_runtime=None, indices=None):
+        """Check Numbers.
+
+        Check output file numbers and IDs.
+
+        Parameters
+        ----------
+        par_runtime : dict, optional
+            runtime parameter. default is None
+        indices: list, optional
+            if not None (default), only check modules corresponding
+            to indices
+
+        """
+        # Update subdirs if not already set as list
+        self.update_subdirs(par_runtime)
+
+        # Initialise variables
+        self._names_in_dir = {}
+        self._paths_in_dir = {}
+        self._missing_IDs_job = []
+        n_missing_job = 0
+
+        # Loop over modules
+        for idx, module in enumerate(self._modules):
+            if indices is not None and idx not in indices:
+                continue
+
+            # Look over subdirs
+            iterable = self._output_subdirs
+            if len(iterable) > 1 and self._verbose:
+                iterable = tqdm(iterable, desc="subdirs", leave=False)
+
+            # Get output file names and paths
+            self.get_names_in_dir(
+                iterable,
+                module,
+                idx,
+            )
+
+            # If expected is string: Update parameter with runtime value
+            # and set as integer
+            if isinstance(self._key_expected[idx], str):
+                n_expected_base = get_par_runtime(
+                    par_runtime, self._key_expected[idx], kind="n"
+                )
+            else:
+                n_expected_base = self._key_expected[idx]
+
+            # Get some numbers
+            n_found = len(self._names_in_dir[idx])
+            n_expected = n_expected_base * self._n_mult[idx]
+            n_missing = n_expected - n_found
+
+            n_missing_explained = 0
+            if n_missing > 0:
+                # TODO: make check_special class function, deal with
+                # paths, names in dir
+                if False and module == "setools_runner":
+                    (
+                        self._paths_in_dir[idx],
+                        self._names_in_dir[idx],
+                        n_missing_explained,
+                    ) = check_special(module, paths_in_dir, names_in_dir)
+
+                n_missing = n_missing - n_missing_explained
+
+            # Print statistics
+            self.print_stats(
+                module,
+                n_expected,
+                n_found,
+                n_missing_explained,
+                n_missing,
+                self._n_mult[idx],
+            )
+
+            # Write missing IDs for module to file
+            if n_missing > 0:
+                missing_IDs = self.output_missing(
+                    module,
+                    idx,
+                    par_runtime=par_runtime,
+                )
+                n_missing_job += n_missing
+                self._missing_IDs_job.extend(missing_IDs)
+
+        # Write missing IDs for entire job to file
+        if n_missing_job > 0:
+            self.output_missing_job()
+
+
+def get_par_runtime(par_runtime, key, kind="n"):
+    """Get Par RunTime.
+
+    Return runtime parameter value.
+
+    Parameters
+    ----------
+    par_runtime: dict
+        runtime parameter
+    key: str
+        key
+
+    """
+    combined_key = f"{kind}_{key}"
+    return par_runtime[combined_key]
+
+
+def print_par_runtime(par_runtime, verbose=True):
+    # Print runtime parameter values
+    if verbose:
+        logging.info("")
+        logging.info("===========")
+        logging.info("par_runtime")
+        logging.info("-----------")
+        for key, value in par_runtime.items():
+            if not key.startswith("list"):
+                logging.info(f"{key:30s} {value:6d}")
+            else:
+                logging.info(f"{key:29s} [{len(value):6d}]")
+        logging.info("===========")
+        logging.info("")

From d4cb47ce14ed9f8fcdd38af42cf4f68488f22560 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 25 Dec 2023 18:02:41 +0100
Subject: [PATCH 077/133] Update post_processing.md

cleaned up psf post proc
---
 docs/source/post_processing.md | 77 ++++++----------------------------
 1 file changed, 13 insertions(+), 64 deletions(-)

diff --git a/docs/source/post_processing.md b/docs/source/post_processing.md
index 555ffdf85..8d9922dbb 100644
--- a/docs/source/post_processing.md
+++ b/docs/source/post_processing.md
@@ -10,59 +10,7 @@ catalogue via _metacalibration_), a joint star catalogue, and PSF diagnostic plo
 If main ShapePipe processing happened at the old canfar VM system (e.g. CFIS v0 and v1), go
 [here](vos_retrieve.md) for details how to retrieve the ShapePipe output files.
 
-Some of the following steps pertain specifically to runs carried out on [canfar](https://www.canfar.net/en),
-but most are general.
-
-1. Retrieve `ShapePipe` result files
-
-   For a local run on the same machine as for post-processing, nothing needs to be done.
-   In some cases, the run was carried out on a remote machine or cluster, and the resulting `ShapePipe`
-   output files need to be retrieved.
-   
-   In the specific case of canfar_avail_results.py, this is done as follows.
-   
-   A. Check availability of results
-
-      A `canfar` job can submit a large number of tiles, whose processing time can vary a lot.
-      We assume that the submitted tile ID list is available locally via the ascii file `tile_numbers.txt`. 
-      To check which tiles have finished running, and whose results have been uploaded, use
-      ```bash
-      canfar_avail_results -i tile_numbers.txt -v -p PSF --input_path INPUT_PATH
-      ```
-      where PSF is one in [`psfex`|`mccd`], and INPUT_PATH the input path on vos, default `vos:cfis/cosmostat/kilbinger/results`.
-      See `-h` for all options.
-
-   B. Download results
-
-      All results files will be downloaded with
-      ```bash
-      canfar_download_results -i tile_numbers.txt -v -p PSF --input_vos INPUT_VOS
-      ```
-      Use the same options as for same as for `canfar_avail_results`.
-      
-      This command can be run in the same directory at subsequent times, to complete an ongoing run: Only newer files will be downloaded
-      from the `vos` directory. This also assures that partially downloaded or corrupt files will be replaced.
-
-      Checking the `vos` directorty can be slow for large patches.
-      To only download files that are not yet present locally (in `.`), first write the missing ones to an ascii file, using again the
-      script `canfar_avail_results`, but this time with `.` as input path:
-      ```bash
-      canfar_avail_results -i tile_numbers.txt --input_path . -p PSF -v -o missing.txt
-      '''
-      Then, download only the missing files with
-      ```bash
-      canfar_download_results -i missing.txt --input_vos cosmostat/kilbinger/results_mccd_oc2 -p mccd -v
-      ```
-
-   C. Un-tar results
-     ```bash
-      untar_results -p PSF
-      ```
-      On success, `ShapePipe` output `fits` and `log` files will be now in various subdirs of the `output` directory.
-
-At this step all required `ShapePipe` resulting output files are available in the current working directory.
-
-2. Optional: Split output in sub-samples
+1. Optional: Split output into sub-samples
 
    An optional intermediate step is to create directories for sub-samples, for example one directory
    for each patch on the sky. This will create symbolic links to the results `.tgz` files downloaded in
@@ -73,33 +21,34 @@ At this step all required `ShapePipe` resulting output files are available in th
     ```
     The following steps will then be done in the directory `tiles_W3`.
 
-3. Run PSF diagnostics, create merged catalogue
+2. Run PSF diagnostics, create merged catalogue
 
    Type
    ```bash
    post_proc_sp -p PSF
    ```
-   to automatically perform a number of post-processing steps. Chose the PSF model with the option
+   to automatically perform a number of post-processing steps. Choose the PSF model with the option
    `-p psfex|mccd`. In detail, these are (and can also be done individually
    by hand):
    
-   A. Analyse psf validation files
+   1. Analyse psf validation files
    
       ```bash
-      prepare_star_cat -p PSF
+      combine_runs -t psf -p PSF
       ```
       with options as for `post_proc_sp`.
-      This script identifies all psf validation files (from all processed tiles downloaded to `pwd`), creates symbolic links,
-      merges the catalogues, and creates plots of PSF ellipticity, size, and residuals over the focal plane.
+      This script creates a new combined psf run in the ShapePipe `output` directory, by identifying all psf validation files
+      and creating symbolic links. The run log file is updated.
 
-   B. Create plots of the PSF and their residuals in the focal plane, as a diagnostic of the overall PSF model.
-     As a scale-dependend test, which propagates directly to the shear correlation function, the rho statistics are computed,
-     see {cite:p}`rowe:10` and {cite:p}`jarvis:16`,
+   3. Merge individual psf validation files into one catalogue. Create plots of the PSF and their residuals in the focal plane,
+      as a diagnostic of the overall PSF model.
+      As a scale-dependend test, which propagates directly to the shear correlation function, the rho statistics are computed,
+      see {cite:p}`rowe:10` and {cite:p}`jarvis:16`,
       ```bash
       shapepipe_run -c /path/to/shapepipe/example/cfis/config_MsPl_PSF.ini
       ``` 
 
-   C. Prepare output directory
+   4. Prepare output directory
    
       Create links to all 'final_cat' result files with 
       ```bash
@@ -108,7 +57,7 @@ At this step all required `ShapePipe` resulting output files are available in th
       The corresponding output directory that is created is `output/run_sp_combined/make_catalog_runner/output`.
       On success, it contains links to all `final_cat` output catalogues
 
-   D. Merge final output files
+   5. Merge final output files
    
       Create a single main shape catalog:
       ```bash

From 2ffba3b486afac805100caf98fa61f35461c2b47 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 26 Dec 2023 10:45:25 +0000
Subject: [PATCH 078/133] P3 proceesing to final cat

---
 example/cfis/config_MsPl_psfex.ini |  4 +-
 example/cfis/config_Ms_psfex.ini   | 68 ++++++++++++++++++++++++++++++
 scripts/sh/combine_runs.bash       |  8 +++-
 scripts/sh/job_sp_canfar.bash      | 11 ++---
 shapepipe/utilities/summary.py     |  4 --
 5 files changed, 79 insertions(+), 16 deletions(-)
 create mode 100644 example/cfis/config_Ms_psfex.ini

diff --git a/example/cfis/config_MsPl_psfex.ini b/example/cfis/config_MsPl_psfex.ini
index 7fcb3ba20..d21fdd126 100644
--- a/example/cfis/config_MsPl_psfex.ini
+++ b/example/cfis/config_MsPl_psfex.ini
@@ -35,7 +35,7 @@ LOG_NAME = log_sp
 RUN_LOG_NAME = log_run_sp
 
 # Input directory, containing input files, single string or list of names
-INPUT_DIR = $SP_RUN/psf_validation_ind
+INPUT_DIR = $SP_RUN/output
 
 # Output directory
 OUTPUT_DIR = $SP_RUN/output
@@ -54,7 +54,7 @@ TIMEOUT = 96:00:00
 ## Module options
 [MERGE_STARCAT_RUNNER]
 
-INPUT_DIR = psf_validation_ind
+INPUT_DIR = last:psfex_interp_runner
 
 PSF_MODEL = psfex
 
diff --git a/example/cfis/config_Ms_psfex.ini b/example/cfis/config_Ms_psfex.ini
new file mode 100644
index 000000000..5a105e773
--- /dev/null
+++ b/example/cfis/config_Ms_psfex.ini
@@ -0,0 +1,68 @@
+# ShapePipe configuration file for post-processing.
+# merge star cat.
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_Ms
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = merge_starcat_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN/output
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 4
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+[MERGE_STARCAT_RUNNER]
+
+INPUT_DIR = last:psfex_interp_runner
+
+PSF_MODEL = psfex
+
+NUMBERING_SCHEME = -0000000-0
+
+# Input file pattern(s), list of strings with length matching number of expected input file types
+# Cannot contain wild cards
+FILE_PATTERN = validation_psf
+
+# FILE_EXT (optional) list of string extensions to identify input files
+FILE_EXT = .fits
diff --git a/scripts/sh/combine_runs.bash b/scripts/sh/combine_runs.bash
index b26ab39b5..89e844c63 100755
--- a/scripts/sh/combine_runs.bash
+++ b/scripts/sh/combine_runs.bash
@@ -98,7 +98,11 @@ run_out="run_sp_combined_$cat"
 
 if [ "$cat" == "final" ]; then
 
-  run_in="$pwd/$out_base/run_sp_Mc_*"
+  # v1
+  #run_in="$pwd/$out_base/run_sp_Mc_*"
+  # v2
+  run_in="$pwd/tile_runs/*/$out_base/run_sp_Mc_*"
+
   module="make_catalog_runner"
   pattern="final_cat-*"
 
@@ -120,7 +124,7 @@ elif [ "$cat" == "psf" ]; then
   # v1
   #run_in="$pwf/$out_base/run_sp_exp_Pi_*"
   # v2
-  run_in="$pwd/exp_runs/*/$out_base/"
+  run_in="$pwd/exp_runs/*/$out_base/run_sp_exp_Pi_*"
 
   pattern="validation_psf-*"
   if [ "$psf" == "psfex" ]; then
diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
index 17695f41d..744e00161 100755
--- a/scripts/sh/job_sp_canfar.bash
+++ b/scripts/sh/job_sp_canfar.bash
@@ -161,10 +161,10 @@ export SP_RUN=`pwd`
 export SP_CONFIG=$SP_RUN/cfis
 export SP_CONFIG_MOD=$SP_RUN/cfis_mod
 
-## Other variables
+# To find ssl certificate
+export VM_HOME=$HOME
 
-# Input tile numbers ASCII file
-export TILE_NUMBERS_PATH=tile_numbers.txt
+## Other variables
 
 # Output
 OUTPUT=$SP_RUN/output
@@ -326,11 +326,6 @@ mkdir -p $SP_CONFIG_MOD
 
 # Processing
 
-## Check for input tile list
-#if [ ! -e $TILE_NUMBERS_PATH ]; then
-  #echo "Tile numbers file $TILE_NUMBERS_PATH not found, exiting"
-#fi
-
 ### Retrieve config files
 if [[ $config_dir == *"vos:"* ]]; then
   command_sp "$VCP $config_dir ." "Retrieve shapepipe config files"
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index ac69c86a1..fc160c972 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -502,7 +502,6 @@ def get_names_in_dir(self, iterable, module, idx):
                 f"{self._path_left}/{subdir}/"
                 + f"{self._path_right[idx]}"
             )
-            print("MKDEBUG ", base_and_subdir)
             if os.path.isdir(base_and_subdir):
 
                 matches = []
@@ -511,10 +510,8 @@ def get_names_in_dir(self, iterable, module, idx):
                 with os.scandir(base_and_subdir) as entries:
                     for entry in entries:
 
-                        print("MKDEBUG ", entry, self._run_dir[idx])
                         # Append directory name if matches module
                         if entry.name.startswith(self._run_dir[idx]):
-                            print("MKDEBUG matches")
                             matches.append(entry)
 
                     # This entry does not match module -> next
@@ -529,7 +526,6 @@ def get_names_in_dir(self, iterable, module, idx):
                     directory = self.get_module_output_dir(
                         full_path, module
                     )
-                    print("MKDEBUG ", directory)
 
                     # Find matching file names and paths
                     self.get_matches_final(directory, idx)

From 9f81502236391cb3f257b665c4ef9cbed3427240 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sat, 6 Jan 2024 10:35:29 +0000
Subject: [PATCH 079/133] config files updated

---
 example/cfis/config_GitFeGie_symlink.ini |  2 +
 example/cfis/config_exp_Ma_onthefly.ini  | 76 ++++++++++++++++++++++++
 example/cfis/config_exp_psfex.ini        | 10 ++--
 example/cfis/config_tile_Ma_onthefly.ini |  4 +-
 example/cfis/config_tile_Sx.ini          |  6 +-
 5 files changed, 86 insertions(+), 12 deletions(-)
 create mode 100644 example/cfis/config_exp_Ma_onthefly.ini

diff --git a/example/cfis/config_GitFeGie_symlink.ini b/example/cfis/config_GitFeGie_symlink.ini
index 22e834533..971df44d4 100644
--- a/example/cfis/config_GitFeGie_symlink.ini
+++ b/example/cfis/config_GitFeGie_symlink.ini
@@ -80,6 +80,8 @@ INPUT_NUMBERING = \d{3}\.\d{3}
 # Output file pattern without number
 OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight-
 
+CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output
+
 # Copy/download method, one in 'vos', 'symlink'
 RETRIEVE = symlink
 
diff --git a/example/cfis/config_exp_Ma_onthefly.ini b/example/cfis/config_exp_Ma_onthefly.ini
new file mode 100644
index 000000000..5f629500d
--- /dev/null
+++ b/example/cfis/config_exp_Ma_onthefly.ini
@@ -0,0 +1,76 @@
+# ShapePipe configuration file for masking of exposures
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_exp_Ma
+
+# Add date and time to RUN_NAME, optional, default: False
+; RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = mask_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = .
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 16
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+### Mask exposures
+[MASK_RUNNER]
+
+# Parent module
+INPUT_DIR = last:split_exp_runner
+
+# Update numbering convention, accounting for HDU number of
+# single-exposure single-HDU files
+NUMBERING_SCHEME = -0000000-0
+
+# Path of mask config file
+MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask
+
+# External mask file flag, use if True, otherwise ignore
+USE_EXT_FLAG = True
+
+# External star catalogue flag, use external cat if True,
+# obtain from online catalogue if False
+USE_EXT_STAR = False
+
+# File name suffix for the output flag files (optional)
+PREFIX = pipeline
diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini
index 11f375456..52ca81f24 100644
--- a/example/cfis/config_exp_psfex.ini
+++ b/example/cfis/config_exp_psfex.ini
@@ -9,8 +9,8 @@
 VERBOSE = True
 
 # Name of run (optional) default: shapepipe_run
-#RUN_NAME = run_sp_exp_SxSePsfPi
-RUN_NAME = run_sp_exp_SxSePsf
+RUN_NAME = run_sp_exp_SxSePsfPi
+#RUN_NAME = run_sp_exp_SxSePsf
 
 # Add date and time to RUN_NAME, optional, default: True
 ; RUN_DATETIME = False
@@ -20,8 +20,7 @@ RUN_NAME = run_sp_exp_SxSePsf
 [EXECUTION]
 
 # Module name, single string or comma-separated list of valid module runner names
-MODULE = sextractor_runner, setools_runner, psfex_runner
-#psfex_interp_runner
+MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner
          
 
 # Run mode, SMP or MPI
@@ -59,8 +58,7 @@ TIMEOUT = 96:00:00
 [SEXTRACTOR_RUNNER]
 
 # Input from two modules
-INPUT_DIR = last:split_exp_runner, last:mask_runner_run_2
-#INPUT_MODULE = split_exp_runner, mask_runner_run_2
+INPUT_DIR = last:split_exp_runner, run_sp_exp_Ma:mask_runner
 
 # Read pipeline flag files created by mask module
 FILE_PATTERN = image, weight, pipeline_flag
diff --git a/example/cfis/config_tile_Ma_onthefly.ini b/example/cfis/config_tile_Ma_onthefly.ini
index 6f3723a0f..1b1fbbbea 100644
--- a/example/cfis/config_tile_Ma_onthefly.ini
+++ b/example/cfis/config_tile_Ma_onthefly.ini
@@ -34,7 +34,7 @@ LOG_NAME = log_sp
 RUN_LOG_NAME = log_run_sp
 
 # Input directory, containing input files, single string or list of names
-INPUT_DIR = .
+INPUT_DIR = $SP_RUN/output
 
 # Output directory
 OUTPUT_DIR = $SP_RUN/output
@@ -53,7 +53,7 @@ TIMEOUT = 96:00:00
 ## Module options
 
 ### Mask tiles
-[MASK_RUNNER_RUN]
+[MASK_RUNNER]
 
 # Input directory, containing input files, single string or list of names
 INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner
diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini
index 09eb921c2..1c418245d 100644
--- a/example/cfis/config_tile_Sx.ini
+++ b/example/cfis/config_tile_Sx.ini
@@ -1,4 +1,4 @@
-# ShapePipe configuration file for tile detection/
+# ShapePipe configuration file for tile detection
 
 
 ## Default ShapePipe options
@@ -55,9 +55,7 @@ TIMEOUT = 96:00:00
 
 [SEXTRACTOR_RUNNER]
 
-INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1
-
-INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1
+INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, run_sp_tile_Ma:mask_runner
 
 FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag
 

From 6eb11cfa2080ce49f5ce524d9be4cc4fc868fd3a Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 15 Jan 2024 11:00:58 +0000
Subject: [PATCH 080/133] PSFEx interp runner: allowing all: for
 ME_DOT_PSF_PDIRS

---
 example/cfis/config_tile_PiViSmVi_canfar.ini  |  2 +-
 .../psfex_interp_package/psfex_interp.py      | 51 +++++++++++--------
 shapepipe/modules/psfex_interp_runner.py      | 17 +++++--
 3 files changed, 45 insertions(+), 25 deletions(-)

diff --git a/example/cfis/config_tile_PiViSmVi_canfar.ini b/example/cfis/config_tile_PiViSmVi_canfar.ini
index 71bdc699e..51900b525 100644
--- a/example/cfis/config_tile_PiViSmVi_canfar.ini
+++ b/example/cfis/config_tile_PiViSmVi_canfar.ini
@@ -87,7 +87,7 @@ CHI2_THRESH = 2
 
 # Multi-epoch mode parameters
 
-ME_DOT_PSF_DIR = psfex_runner
+ME_DOT_PSF_DIR = all:psfex_runner
 
 # Input psf file pattern
 ME_DOT_PSF_PATTERN = star_split_ratio_80
diff --git a/shapepipe/modules/psfex_interp_package/psfex_interp.py b/shapepipe/modules/psfex_interp_package/psfex_interp.py
index ca7d6bea3..ea3db1de7 100644
--- a/shapepipe/modules/psfex_interp_package/psfex_interp.py
+++ b/shapepipe/modules/psfex_interp_package/psfex_interp.py
@@ -2,7 +2,7 @@
 
 This module computes the PSFs from a PSFEx model at several galaxy positions.
 
-:Authors: Morgan Schmitz and Axel Guinot
+:Authors: Morgan Schmitz, Axel Guinot, Martin Kilbinger
 
 """
 
@@ -115,7 +115,7 @@ class PSFExInterpolator(object):
     Parameters
     ----------
     dotpsf_path : str
-        Path to PSFEx output file
+        Path to PSFEx output file; can be `None` in multi-epoch mode
     galcat_path : str
         Path to SExtractor-like galaxy catalogue
     output_path : str
@@ -151,18 +151,19 @@ def __init__(
     ):
 
         # Path to PSFEx output file
+        self._dotpsf_path = dotpsf_path
         if (
-            isinstance(dotpsf_path, type(None))
-            or os.path.isfile(dotpsf_path)
+            not isinstance(dotpsf_path, type(None))
+            and not os.path.isfile(dotpsf_path)
         ):
-            self._dotpsf_path = dotpsf_path
-        else:
             raise ValueError(f'Cound not find file {dotpsf_path}.')
+
         # Path to catalogue containing galaxy positions
         if os.path.isfile(galcat_path):
             self._galcat_path = galcat_path
         else:
             raise ValueError(f'Cound not find file {galcat_path}.')
+
         # Path to output file to be written
         self._output_path = output_path + '/galaxy_psf'
         # Path to output file to be written for validation
@@ -512,25 +513,28 @@ def _write_output_validation(self, star_dict, psfex_cat_dict):
 
         output.save_as_fits(data, sex_cat_path=self._galcat_path)
 
-    def process_me(self, dot_psf_dir, dot_psf_pattern, f_wcs_path):
+    def process_me(self, dot_psf_dirs, dot_psf_pattern, f_wcs_path):
         """Process Multi-Epoch.
 
-        Process the multi-epoch.
+        Process multi-epoc PSF interpolation.
 
         Parameters
         ----------
-        dot_psf_dir : str
-            Path to the directory containing the ``.psf`` files
+        dot_psf_dirs : list
+            Paths to the directory containing the ``.psf`` files
         dot_psf_pattern : str
             Common pattern of the ``.psf`` files
         f_wcs_path : str
             Path to the log file containing the WCS for each CCDs
 
         """
-        if os.path.exists(dot_psf_dir):
-            self._dot_psf_dir = dot_psf_dir
-        else:
-            raise ValueError(f'Cound not find directory {dot_psf_dir}.')
+        if not any(
+             os.path.exists(dot_psf_dir)
+             for dot_psf_dir in dot_psf_dirs
+        ):
+            raise ValueError('Cound not find any dot psf directory.')
+
+        self._dot_psf_dirs = dot_psf_dirs
 
         self._dot_psf_pattern = dot_psf_pattern
 
@@ -569,7 +573,6 @@ def _interpolate_me(self):
         all_id = np.copy(cat.get_data()['NUMBER'])
         key_ne = 'N_EPOCH'
         if key_ne not in cat.get_data().dtype.names:
-            print("MKDEBUG ", cat.get_data())
             raise KeyError(
                 f'Key {key_ne} not found in input galaxy catalogue'
                 + f'{self._galcat_path}, needed for'
@@ -595,10 +598,18 @@ def _interpolate_me(self):
             for ccd in ccd_list:
                 if ccd == -1:
                     continue
-                dot_psf_path = (
-                    f'{self._dot_psf_dir}/{self._dot_psf_pattern}-{exp_name}'
-                    + f'-{ccd}.psf'
-                )
+                found = False
+                for dot_psf_dir in self._dot_psf_dirs:
+                    dot_psf_path = (
+                        f'{dot_psf_dir}/{self._dot_psf_pattern}-{exp_name}'
+                        + f'-{ccd}.psf'
+                    )
+                    if os.path.exists(dot_psf_path):
+                        found = True
+                        break
+                if not found:
+                    raise ValueError("No dot psf file found")
+
                 ind_obj = np.where(cat.get_data(hdu_index)['CCD_N'] == ccd)[0]
                 obj_id = all_id[ind_obj]
                 gal_pos = np.array(
@@ -641,7 +652,7 @@ def _interpolate_me(self):
                     and self.interp_PSFs == FILE_NOT_FOUND
                 ):
                     self._w_log.info(
-                        f'Psf model file {self._dotpsf_path} not found. '
+                        f'Psf model file {dot_psf_path} not found. '
                         + 'Object inside this ccd will lose an epoch.'
                     )
                     continue
diff --git a/shapepipe/modules/psfex_interp_runner.py b/shapepipe/modules/psfex_interp_runner.py
index f18b543fb..e9c0cca54 100644
--- a/shapepipe/modules/psfex_interp_runner.py
+++ b/shapepipe/modules/psfex_interp_runner.py
@@ -2,14 +2,14 @@
 
 Module runner for ``psfex_interp``.
 
-:Author: Axel Guinot
+:Author: Axel Guinot, Martin Kilbinger
 
 """
 
 from shapepipe.modules.module_decorator import module_runner
 from shapepipe.modules.psfex_interp_package import psfex_interp
 
-from shapepipe.pipeline.run_log import get_last_dir
+from shapepipe.pipeline.run_log import get_last_dir, get_all_dirs
 
 
 @module_runner(
@@ -66,7 +66,16 @@ def psfex_interp_runner(
             module_config_sec,
             'ME_DOT_PSF_DIR',
         )
-        dot_psf_dir = get_last_dir(run_dirs['run_log'], module)
+        module_name = module.split(":")[-1]
+        if "last" in module:
+            dot_psf_dirs = [get_last_dir(run_dirs['run_log'], module_name)]
+        elif "all" in module:
+            dot_psf_dirs = get_all_dirs(run_dirs["run_log"], module_name)
+        else:
+            raise ValueError(
+                "Expected qualifier 'last:' or 'all' before module"
+               + f" '{module}' in config entry 'ME_DOT_PSF_DIR'")
+
         dot_psf_pattern = config.get(
             module_config_sec,
             'ME_DOT_PSF_PATTERN',
@@ -90,7 +99,7 @@ def psfex_interp_runner(
         )
 
         # Process inputs multi-epoch
-        psfex_interp_inst.process_me(dot_psf_dir, dot_psf_pattern, f_wcs_path)
+        psfex_interp_inst.process_me(dot_psf_dirs, dot_psf_pattern, f_wcs_path)
 
     # Run in VALIDATION mode
     elif mode == 'VALIDATION':

From 485da581e940fe79605c1616bb913ef6aa93f130 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 15 Jan 2024 12:33:20 +0000
Subject: [PATCH 081/133] init run exc script: added -d option; updates

---
 scripts/sh/init_run_exclusive_canfar.sh | 100 +++++++++---------------
 1 file changed, 38 insertions(+), 62 deletions(-)

diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 2d1e381d0..05ca2a848 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -10,6 +10,7 @@ N_SMP=1
 kind=-1
 dry_run=0
 nsh_jobs=8
+dir=`pwd`
 VERBOSE=1
 
 
@@ -28,6 +29,8 @@ usage="Usage: $(basename "$0") -j JOB -e ID  -k KIND [OPTIONS]
     \timage kind, allowed are 'tile' and 'exp'\n
    -N, --N_SMP N_SMOp\n
     \tnumber of jobs (SMP mode only), default from original config files\n
+   -d, --directory\n
+    \trun directory, default is pwd ($dir)\n
    -n, --dry_run\n
     \tdry run, no actuall processing\n
 "
@@ -61,6 +64,10 @@ while [ $# -gt 0 ]; do
       kind="$2"                                                                
       shift                                                                     
       ;;                                                                        
+    -d|--directory)
+      dir="$2"
+      shift
+      ;;
     -n|--dry_run)
       dry_run=1
       ;;
@@ -132,14 +139,11 @@ fi
 
 conda activate shapepipe
 
-basedir=$HOME/cosmostat/P3_v2/psfex
-cd $basedir
-
+cd $dir
+echo $pwd
 
-# Update links to exposure run directories
-if [ "$kind" == "tile" ]; then
-  command "rm -rf tile_runs/$ID/output/run_exp_SxSePsf*" $dry_run
-  command "link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs" $dry_run
+if [ ! -d ${kind}_runs ]; then
+  command "mkdir ${kind}_runs" $dry_run
 fi
 
 
@@ -158,62 +162,38 @@ fi
 
 cd output
 
-if [ 0 == 1 ]; then
-
-  if [ ! -f log_exp_headers.sqlite ]; then
-    command "ln -s $basedir/output/log_exp_headers.sqlite" $dry_run
-  fi
-
-  # Remove potentially obsolete link
-  #rm run_sp_exp_SpMh*
-  #rm  run_sp_MaMa_*
+if [ ! -f log_exp_headers.sqlite ]; then
+  command "ln -s $dir/output/log_exp_headers.sqlite" $dry_run
+fi
 
-  # Update links to global run directories (GiFeGie, Uz, Ma?, combined_flag?)
-  for dir in $basedir/output/run_sp_*; do
-    command "ln -sf $dir" $dry_run
-  done
-  if [ ! -e run_sp_combined_flag ]; then
-    command "ln -s $basedir/output/run_sp_combined_flag" $dry_run
-  fi
 
-fi
+# Update links to global run directories (GiFeGie, Uz, Ma?, combined_flag?)
+for dir in $dir/output/run_sp_*; do
+  command "ln -sf $dir" $dry_run
+done
 
-(( do_job= $job & 128 ))
-#if [[ $do_job != 0 ]]; then
-# The following is now dealt with in job_sh_canar.bash
-if [ 0 == 1 ]; then
-
-  # Indentify and remove unfinished ngmix dirs
-  min_n_out=2
-  for k in $(seq 1 $nsh_jobs); do
-    ngmix_run="run_sp_tile_ngmix_Ng${k}u/ngmix_runner"
-    if [ -e "$ngmix_run" ]; then
-      ngmix_out="$ngmix_run/output"
-      n_out=`ls -rlt $ngmix_out | wc -l`
-      if [ "$n_out" -lt "$min_n_out" ]; then
-          min_n_out=$n_out
-      fi
-      #echo $k $n_out $min_n_out
-    else
-      echo "ngmix separated run #$k not found"
-      min_n_out=0
-    fi
-  done
-  if [ "$min_n_out" -lt "2" ]; then
-    echo "At least one ngmix separated run no output files"
-    for k in $(seq 1 $nsh_jobs); do
-      command "rm -rf run_sp_tile_ngmix_Ng${k}u" $dry_run
-    done
-  else
-    if [ "$job" == "128" ]; then
-      echo "ngmix found complete, all is well, exiting"
-      exit 0
+# Update links to exposure run directories, which were created in job 32
+(( do_job= $job & 64 ))
+if [[ $do_job != 0 ]]; then
+  if [ "$kind" == "tile" ]; then
+    cd ../../..
+    #command "rm -rf tile_runs/$ID/output/run_exp_SxSePsf*" $dry_run
+    command "link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs" $dry_run
+    cd ${kind}_runs/$ID/output
+
+    # Remove duplicate job-32 runs (tile detection)
+    n_16=`ls -rt1d run_sp_tile_Sx_* | wc -l`
+    if [ "$n_16" != "1" ]; then
+      n_remove="$(($n_16-1))"
+      echo "removing $n_remove duplicate old job-32 runs"
+      rm -rf `ls -rt1d run_sp_tile_Sx_* | head -$n_remove`
     fi
   fi
-
-fi 
+fi
 
 cd ..
+
+# Update log file
 command update_runs_log_file.py $dry_run
 
 echo -n "pwd: "
@@ -222,12 +202,8 @@ pwd
 echo -n "environment: "
 echo $CONDA_PREFIX
 
-#command "job_sp_canfar.bash -p psfex -j $job -e $ID --n_smp $N_SMP" $dry_run
+command "job_sp_canfar.bash -p psfex -j $job -e $ID --n_smp $N_SMP" $dry_run
 
-export SP_RUN=.
-command "shapepipe_run -c $HOME/shapepipe/example/cfis/config_exp_Pi.ini" $dry_run
-
-cd $basedir
+cd $dir
 
 echo "end init run tile canfar"
-

From ef3a2c92df2879cba9ffa3ea6b091eb1b5ccb5da Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 15 Jan 2024 12:33:56 +0000
Subject: [PATCH 082/133] job sp canfar script: remove old vos upload code

---
 scripts/sh/job_sp_canfar.bash | 58 -----------------------------------
 1 file changed, 58 deletions(-)

diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
index 744e00161..91f67619b 100755
--- a/scripts/sh/job_sp_canfar.bash
+++ b/scripts/sh/job_sp_canfar.bash
@@ -38,7 +38,6 @@ usage="Usage: $(basename "$0") [OPTIONS] [TILE_ID]
    \t  64: galaxy selection on tiles (offline)\n
    \t 128: shapes and morphology (offline)\n
    \t 256: paste catalogues (offline)\n
-   \t 512: upload results (online)\n
    -c, --config_dir DIR\n
    \t config file directory, default='$config_dir'\n
    -p, --psf MODEL\n
@@ -485,60 +484,3 @@ if [[ $do_job != 0 ]]; then
   command_sp "shapepipe_run -c $SP_CONFIG/config_make_cat_$psf.ini" "Run shapepipe (tile: create final cat $psf)" "$VERBOSE" "$ID"
 
 fi
-
-## Upload results (online)
-(( do_job= $job & 512 ))
-if [[ $do_job != 0 ]]; then
-
-  ### module and pipeline log files
-  upload_logs "$ID" "$VERBOSE"
-
-  ### Final shape catalog
-  ### pipeline_flags are the tile masks, for random cats
-  ### SETools masks (selection), stats and plots
-  ### ${psf}_interp_exp for diagnostics, validation with leakage,
-  ### validation with residuals, rho stats
-
-  NAMES=(
-    "final_cat"
-    "pipeline_flag"
-    "setools_mask"
-    "setools_stat"
-    "setools_plot"
-  )
-  DIRS=(
-    "*/make_cat_runner/output"
-    "*/mask_runner_run_1/output"
-    "*/setools_runner/output/mask"
-    "*/setools_runner/output/stat"
-    "*/setools_runner/output/plot"
-  )
-  PATTERNS=(
-    "final_cat-*"
-    "pipeline_flag-???-???*"
-    "*"
-    "*"
-    "*"
-  )
-
-  # PSF validation
-  pattern="validation_psf-*"
-  if [ "$psf" == "psfex" ]; then
-    name="psfex_interp_exp"
-    dir="*/psfex_interp_runner/output"
-  else
-    name="mccd_fit_val_runner"
-    dir="*/mccd_fit_val_runner/output"
-  fi
-  upl=$output_rel/$dir/$pattern
-  upload "$name" "$ID" "$VERBOSE" "${upl[@]}"
-
-  for n in "${!NAMES[@]}"; do
-      name=${NAMES[$n]}
-      dir=${DIRS[$n]}
-      pattern=${PATTERNS[$n]}
-      upl=$output_rel/$dir/$pattern
-      upload "$name" "$ID" "$VERBOSE" "${upl[@]}"
-  done
-
-fi

From a20cb4f8214f6bc4088744c12c30d881b009ceb5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 15 Jan 2024 12:35:40 +0000
Subject: [PATCH 083/133] summary create library and param files

---
 scripts/python/summary_params_pre_v2.py | 223 ++++++++++++++++++++++++
 scripts/python/summary_run.py           | 174 ++----------------
 shapepipe/utilities/summary.py          |  39 ++++-
 3 files changed, 273 insertions(+), 163 deletions(-)
 create mode 100644 scripts/python/summary_params_pre_v2.py

diff --git a/scripts/python/summary_params_pre_v2.py b/scripts/python/summary_params_pre_v2.py
new file mode 100644
index 000000000..ec310b237
--- /dev/null
+++ b/scripts/python/summary_params_pre_v2.py
@@ -0,0 +1,223 @@
+# Parameters for summary run
+
+import os
+from shapepipe.utilities.summary import *
+
+def init_par_runtime(list_tile_IDs):
+    
+    # Numbers updated at runtime 
+    par_runtime = {}
+
+    par_runtime["n_tile_IDs"] = len(list_tile_IDs)
+    par_runtime["list_tile_IDs"] = list_tile_IDs
+
+    return par_runtime
+
+
+def update_par_runtime_after_find_exp(par_runtime, all_exposures):
+    
+    n_CCD = 40
+    
+    # Single-HDU single exposure images
+    par_runtime["n_shdus"] = get_par_runtime(par_runtime, "exposures") * n_CCD
+    par_runtime["list_shdus"] = get_all_shdus(all_exposures, n_CCD)
+
+    ## For split_exposure_runner, the output is image, weight,flag per single-HDU image
+    ## and a header per exposure.
+    par_runtime["n_3*n_shdus+n_exposures"] = (
+        3 * get_par_runtime(par_runtime, "shdus")
+        + get_par_runtime(par_runtime, "exposures")
+    )
+    
+    return par_runtime
+
+
+def set_jobs_v2_pre_v2(patch, retrieve, verbose):
+    """ Return information about shapepipe jobs
+    
+    """
+    print(f"Set job info for patch {patch}")
+
+    # Main input and output directory
+    main_dir = f"{os.environ['HOME']}/cosmostat/v2/pre_v2/psfex/{patch}"
+
+    # Logging
+    path = f"{main_dir}/summary"
+    if not os.path.isdir(path):
+        os.mkdir(path)
+    log_file_name = f"{path}/summary_log.txt"
+    handlers = [logging.FileHandler(log_file_name), logging.StreamHandler()]
+    logging.basicConfig(
+        level=logging.INFO, format="%(message)s", handlers=handlers
+    )
+
+    logging.info(f"Checking main directory = {main_dir}")
+
+    # Number of links created for retrieved images
+    if retrieve == "vos":
+        n_link = 2
+    else:
+        n_link = 1
+
+    # Tile IDs
+    tile_ID_path = f"{main_dir}/tile_numbers.txt"
+
+    ## Tile IDs with dots
+    list_tile_IDs_dot = get_IDs_from_file(tile_ID_path)
+
+    jobs = {}
+
+#        "run_sp_GitFeGie_",
+    # Set the first job (retrieve images)
+    jobs["1"] = job_data(
+        1,
+         "run_sp_Git",
+       [
+            "get_images_runner_run_1",
+            "find_exposures_runner",
+            "get_images_runner_run_2",
+        ],
+        ["tile_IDs", "tile_IDs", "exposures"],
+        pattern=["CFIS_", "", ""],
+        n_mult=[1 * n_link, 1, 3],
+        path_left=f"{main_dir}/output",
+        verbose=verbose,
+    )
+
+    jobs["2"] = job_data(
+        2,
+        ["run_sp_Uz", "run_sp_exp_SpMh", "run_sp_exp_SpMh_2023-12"],
+        ["uncompress_fits_runner", "merge_headers_runner", "split_exp_runner"],
+        ["tile_IDs", 0, "3*n_shdus+n_exposures"],
+        n_mult=[1, 1, 1],
+        path_left=f"{main_dir}/output",
+        verbose=verbose,
+    )
+
+    if patch == "P3":
+        run_dir_mask_tiles = "run_sp_combined_flag"
+        run_dir_maks_exp = run_dir_tiles
+        mask_module_tiles = "mask_runner_run_1"
+        mask_module_exp = "mask_runner_run_2"
+    else:
+        run_dir_mask_tiles = "run_sp_tile_Ma"
+        run_dir_mask_exp = "run_sp_exp_Ma"
+        mask_module_tiles = "mask_runner"
+        mask_module_exp = "mask_runner"
+
+    jobs["4"] = job_data(
+        4,
+        run_dir_mask_tiles,
+        [mask_module_tiles],
+        ["tile_IDs"],
+        path_left=f"{main_dir}/output",
+        verbose=verbose,
+    )
+
+    jobs["8"] = job_data(
+        8,
+        run_dir_mask_exp,
+        [mask_module_exp],
+        ["shdus"],
+        path_left=f"{main_dir}/output",
+        verbose=verbose,
+    )
+
+    jobs["16"] = job_data(
+        16,
+        "run_sp_tile_Sx",
+        ["sextractor_runner"],
+        ["tile_IDs"],
+        n_mult=2,
+        path_left=f"{main_dir}/tile_runs",
+        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
+        verbose=verbose,
+    )
+
+    # TODO 1 setools_runner output/rand_split
+    # TODO 2 add back Pi
+    jobs["32"] = job_data(
+        32,
+        [
+            "run_sp_exp_SxSePsf",
+            "run_sp_exp_SxSePsf",
+            "run_sp_exp_SxSePsf",
+        ],  # "run_sp_exp_Pi"],
+        [
+            "sextractor_runner",
+            "setools_runner",
+            "psfex_runner",
+        ],  # "psfex_interp_runner"],
+        "shdus",
+        n_mult=[2, 2, 2],  # 1],
+        path_left=f"{main_dir}/exp_runs",
+        output_subdirs="shdus",
+        path_right="output",
+        verbose=verbose,
+    )
+
+    # For P3
+    jobs["33"] = job_data(
+        33,
+        "run_sp_exp_Pi",
+        ["psfex_interp_runner"],
+        "shdus",
+        path_left=f"{main_dir}/exp_runs",
+        output_subdirs="shdus",
+        path_right="output",
+        verbose=verbose,
+    )
+
+    jobs["64"] = job_data(
+        "64",
+        "run_sp_tile_PsViSmVi",
+        [
+            "psfex_interp_runner",
+            "vignetmaker_runner_run_1",
+            "spread_model_runner",
+            "vignetmaker_runner_run_2",
+        ],
+        "tile_IDs",
+        n_mult=[1, 1, 1, 4],
+        path_left=f"{main_dir}/tile_runs",
+        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
+        verbose=verbose,
+    )
+
+    n_sh = 8
+    run_dirs = [f"run_sp_tile_ngmix_Ng{idx+1}u" for idx in range(n_sh)]
+    output_path_missing_IDs = [
+        f"missing_job_128_ngmix_runner_{idx+1}.txt" for idx in range(n_sh)
+    ]
+    jobs["128"] = job_data(
+        "128",
+        run_dirs,
+        ["ngmix_runner"] * 8,
+        "tile_IDs",
+        path_left=f"{main_dir}/tile_runs",
+        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
+        output_path_missing_IDs=output_path_missing_IDs,
+        verbose=verbose,
+    )
+
+    jobs["256"] = job_data(
+        "256",
+        ["run_sp_Ms", "run_sp_Mc"],
+        ["merge_sep_cats_runner", "make_cat_runner"],
+        "tile_IDs",
+        path_left=f"{main_dir}/tile_runs",
+        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
+        verbose=verbose,
+    )
+
+    # Post-processing
+    jobs["1024"] = job_data(
+        "1024",
+        "run_sp_combined_psf",
+        ["psfex_interp_runner"],
+        "shdus",
+        path_left=f"{main_dir}/output",
+        verbose=verbose,
+    )
+    
+    return jobs, list_tile_IDs_dot
diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index bfd926194..bb6e25d05 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -1,162 +1,27 @@
 #!/usr/bin/env python
 
 import sys
+import os
 
 from shapepipe.utilities.summary import *
 
 
 def main(argv=None):
 
+    patch = argv[1]
 
-    # Setting
-    verbose = True
-    log_file_name = "summary_log.txt"
-    handlers = [logging.FileHandler(log_file_name), logging.StreamHandler()]
-    logging.basicConfig(
-        level=logging.INFO, format="%(message)s", handlers=handlers
-    )
-
-    main_dir = "."
     retrieve = "vos"
-    tile_ID_path = f"{main_dir}/tile_numbers.txt"
-
-    # tile IDs with dots
-    list_tile_IDs_dot = get_IDs_from_file(tile_ID_path)
+    verbose = True
+    
+    import summary_params_pre_v2 as params
 
-    # tile IDs with dashes
+    jobs, list_tile_IDs_dot = params.set_jobs_v2_pre_v2(patch, retrieve, verbose)
+    
     list_tile_IDs = job_data.replace_dot_dash(list_tile_IDs_dot)
-    n_tile_IDs = len(list_tile_IDs)
-    n_CCD = 40
-
-    par_runtime = {}
-
-    par_runtime["n_tile_IDs"] = n_tile_IDs
-    par_runtime["list_tile_IDs"] = list_tile_IDs
-
-    jobs = {}
-
-    if retrieve == "vos":
-        n_link = 2
-    else:
-        n_link = 1
-
-    jobs["1"] = job_data(
-        1,
-        "run_sp_GitFeGie_",
-        [
-            "get_images_runner_run_1",
-            "find_exposures_runner",
-            "get_images_runner_run_2",
-        ],
-        ["tile_IDs", "tile_IDs", "exposures"],
-        pattern=["CFIS_", "", ""],
-        n_mult=[1 * n_link, 1, 3],
-        path_left=f"{main_dir}/output",
-        verbose=verbose,
-    )
-
-    jobs["2"] = job_data(
-        2,
-        ["run_sp_Uz", "run_sp_exp_SpMh", "run_sp_exp_SpMh_2023-12"],
-        ["uncompress_fits_runner", "merge_headers_runner", "split_exp_runner"],
-        ["tile_IDs", 0, "3*n_shdus+n_exposures"],
-        n_mult=[1, 1, 1],
-        path_left=f"{main_dir}/output",
-        verbose=verbose,
-    )
-
-    jobs["4"] = job_data(
-        4,
-        "run_sp_combined_flag",
-        ["mask_runner_run_1"],
-        ["tile_IDs"],
-        path_left=f"{main_dir}/output",
-        verbose=verbose,
-    )
-
-    jobs["8"] = job_data(
-        8,
-        "run_sp_combined_flag",
-        ["mask_runner_run_2"],
-        ["shdus"],
-        path_left=f"{main_dir}/output",
-        verbose=verbose,
-    )
-
-    jobs["16"] = job_data(
-        16,
-        "run_sp_tile_Sx",
-        ["sextractor_runner"],
-        ["tile_IDs"],
-        n_mult=2,
-        path_left=f"{main_dir}/tile_runs",
-        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
-        verbose=verbose,
-    )
-
-    # TODO setools_runner output/rand_split
-    jobs["32"] = job_data(
-        32,
-        [
-            "run_sp_exp_SxSePsf",
-            "run_sp_exp_SxSePsf",
-            "run_sp_exp_SxSePsf",
-        ],  # "run_sp_exp_Pi"],
-        [
-            "sextractor_runner",
-            "setools_runner",
-            "psfex_runner",
-        ],  # "psfex_interp_runner"],
-        "shdus",
-        n_mult=[2, 2, 2],  # 1],
-        path_left=f"{main_dir}/exp_runs",
-        output_subdirs="shdus",
-        path_right="output",
-        verbose=verbose,
-    )
-
-    jobs["64"] = job_data(
-        "64",
-        "run_sp_tile_PsViSmVi",
-        [
-            "psfex_interp_runner",
-            "vignetmaker_runner_run_1",
-            "spread_model_runner",
-            "vignetmaker_runner_run_2",
-        ],
-        "tile_IDs",
-        n_mult=[1, 1, 1, 4],
-        path_left=f"{main_dir}/tile_runs",
-        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
-        verbose=verbose,
-    )
-
-    n_sh = 8
-    run_dirs = [f"run_sp_tile_ngmix_Ng{idx+1}u" for idx in range(n_sh)]
-    output_path_missing_IDs = [
-        f"missing_job_128_ngmix_runner_{idx+1}.txt" for idx in range(n_sh)
-    ]
-    jobs["128"] = job_data(
-        "128",
-        run_dirs,
-        ["ngmix_runner"] * 8,
-        "tile_IDs",
-        path_left=f"{main_dir}/tile_runs",
-        output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
-        output_path_missing_IDs=output_path_missing_IDs,
-        verbose=verbose,
-    )
-
-
-    jobs["1024"] = job_data(
-        "1024",
-        "run_sp_combined_psf",
-        ["psfex_interp_runner"],
-        "shdus",
-        path_left=f"{main_dir}/output",
-        verbose=verbose
-    )
-
+    
+    # Numbers updated at runtime
+    par_runtime = params.init_par_runtime(list_tile_IDs)
+    
     job_data.print_stats_header()
 
     for key in "1":
@@ -166,20 +31,19 @@ def main(argv=None):
 
         all_exposures = get_all_exposures(job._paths_in_dir[1], verbose=True)
         par_runtime["n_exposures"] = len(all_exposures)
+        par_runtime["list_exposures"] = all_exposures
 
         job.check_numbers(par_runtime, indices=[2])
 
-    # Update runtime parameter
-    par_runtime["n_shdus"] = get_par_runtime(par_runtime, "exposures") * n_CCD
-    par_runtime["n_3*n_shdus+n_exposures"] = 3 * get_par_runtime(
-        par_runtime, "shdus"
-    ) + get_par_runtime(par_runtime, "exposures")
-    par_runtime["list_shdus"] = get_all_shdus(all_exposures, n_CCD)
-
+    par_runtime = params.update_par_runtime_after_find_exp(par_runtime, all_exposures)
+    
     print_par_runtime(par_runtime, verbose=verbose)
 
-    #for key in ["2", "4", "8", "16", "32", "64", "128"]:
-    for key in ["1024"]:
+    # Get all keys after "1"
+    keys = sorted(jobs.keys(), key=int)
+    _ = keys.pop(0)
+
+    for key in keys:
         job = jobs[key]
         job.print_intro()
         job.check_numbers(par_runtime=par_runtime)
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index fc160c972..c39e943c7 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -189,8 +189,10 @@ class job_data(object):
         case it is set to 1
     pattern: list, optional
         if not None, file pattern to match; defafult is `None`
-    path_lef: str, optional
-        left (first) part of output directory, defaul is "./output"
+    path_main: str, optional
+        main (left-most) part of output directory, default is "."       
+    path_left: str, optional
+        left (first) part of output directory, default is "./output"
     output_subdirs: str, optional
         output subdirectories if not `None`; default is `None`
     path_right: str, optional
@@ -211,7 +213,8 @@ def __init__(
         key_expected,
         n_mult=None,
         pattern=None,
-        path_left="./output",
+        path_main=".",
+        path_left="output",
         output_subdirs=None,
         path_right=None,
         output_path_missing_IDs=None,
@@ -223,6 +226,7 @@ def __init__(
         self._key_expected = set_as_list(item=key_expected, n=len(modules))
         self._n_mult = set_as_list(item=n_mult, n=len(modules))
         self._pattern = set_as_list(item=pattern, n=len(modules), default="")
+        self._path_main = path_main
         self._path_left = path_left
         self._output_subdirs = output_subdirs or [""]
         self._path_right = set_as_list(
@@ -377,7 +381,9 @@ def output_missing(
 
         ## Extract image IDs from names
         IDs = []
-        pattern = re.compile(r"(?:\d{3}-\d{3}|\d{7}-\d+)")
+        pattern = re.compile(
+            r"(?:\d{3}-\d{3}|\d{7}-\d+|\d{7})"
+        )
         for name, path in zip(names_in_dir, paths_in_dir):
             match = pattern.search(name)
             if match:
@@ -401,7 +407,9 @@ def output_missing(
 
         if n_unique > 0:
             if not self._output_path_missing_IDs:
-                output_path = f"missing_job_{self._bit}_{module}.txt"
+                output_path = (
+                    f"{self._path_main}/summary/missing_job_{self._bit}_{module}.txt"
+                )
             else:
                 output_path = self._output_path_missing_IDs[idx]
 
@@ -499,7 +507,8 @@ def get_names_in_dir(self, iterable, module, idx):
         # Loop over subdirs
         for jdx, subdir in enumerate(iterable):
             base_and_subdir = (
-                f"{self._path_left}/{subdir}/"
+                f"{self._path_main}/"
+                + f"{self._path_left}/{subdir}/"
                 + f"{self._path_right[idx]}"
             )
             if os.path.isdir(base_and_subdir):
@@ -530,7 +539,8 @@ def get_names_in_dir(self, iterable, module, idx):
                     # Find matching file names and paths
                     self.get_matches_final(directory, idx)
             else:
-                print(f"Directory base_and_subdir {base_and_subdir} not found")
+                #print(f"Directory base_and_subdir {base_and_subdir} not found")
+                pass
 
     def update_subdirs(self, par_runtime):
         """Update Subdirs.
@@ -571,11 +581,17 @@ def check_numbers(self, par_runtime=None, indices=None):
             if indices is not None and idx not in indices:
                 continue
 
+            if self._verbose:
+                print(f"** module {module}")
+
             # Look over subdirs
             iterable = self._output_subdirs
             if len(iterable) > 1 and self._verbose:
                 iterable = tqdm(iterable, desc="subdirs", leave=False)
 
+            if self._verbose:
+                print(f"*** subdirs {self._output_subdirs}")
+
             # Get output file names and paths
             self.get_names_in_dir(
                 iterable,
@@ -649,6 +665,13 @@ def get_par_runtime(par_runtime, key, kind="n"):
 
     """
     combined_key = f"{kind}_{key}"
+    if (
+        combined_key == "list_3*n_shdus+n_exposures"
+        and combined_key not in par_runtime
+    ):
+        print("{combined_key} not set, TBD")
+        return []
+
     return par_runtime[combined_key]
 
 
@@ -663,6 +686,6 @@ def print_par_runtime(par_runtime, verbose=True):
             if not key.startswith("list"):
                 logging.info(f"{key:30s} {value:6d}")
             else:
-                logging.info(f"{key:29s} [{len(value):6d}]")
+                logging.info(f"{key:29s} {len(value):6d} entries")
         logging.info("===========")
         logging.info("")

From 29b463b220b1ba932be6fa138ff86cb867170bef Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 15 Jan 2024 12:37:01 +0000
Subject: [PATCH 084/133] minor changes

---
 scripts/python/link_to_exp_for_tile.py | 3 +--
 scripts/sh/job_curl_canfar_local.sh    | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index 9d19003d8..fc8ca9a69 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -324,10 +324,9 @@ def main(argv=None):
     exp_IDs = get_exp_IDs(tile_base_dir, tile_ID, verbose=verbose)
     exp_shdu_IDs = get_exp_single_HDU_IDs(exp_IDs, n_CPU)
 
-    patterns = ["run_sp_exp_SxSePsf", "run_sp_exp_Pi"]
+    patterns = ["run_sp_exp_SxSePsf"] #, "run_sp_exp_Pi"]
     for pattern in patterns:
         paths, number = get_paths(exp_base_dir, exp_shdu_IDs, pattern)
-        #print(number)
 
         create_links_paths(tile_base_dir, tile_ID, paths, verbose=verbose)
 
diff --git a/scripts/sh/job_curl_canfar_local.sh b/scripts/sh/job_curl_canfar_local.sh
index 600a15964..665f6ea79 100644
--- a/scripts/sh/job_curl_canfar_local.sh
+++ b/scripts/sh/job_curl_canfar_local.sh
@@ -5,7 +5,7 @@
 
 rm -f session_IDs.txt session_image_IDs.txt
 
-script_local=curl_canfar_local.sh
+script_local=~/shapepipe/scripts/sh/curl_canfar_local.sh
 version="1.0"
 cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
 N_SMP=1

From cb67e8f32ff4ef8877103c531b0f7902dd5f478e Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Mon, 15 Jan 2024 16:12:35 +0000
Subject: [PATCH 085/133] curl local script: now working with -e ID and -f
 file_IDs, in dry and real run

---
 scripts/sh/curl_canfar_local.sh | 67 ++++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 13 deletions(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index f17f16d31..4b85d7cef 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -1,17 +1,18 @@
 #!/usr/bin/env bash
 
-# Usage
-# ~/astro/repositories/github/shapepipe/scripts/sh/curl_canfar.sh 0.9 shapepipe/scripts/sh/init_run_exlusive_canfar.sh ID kind job
-
-# Command line arguments
-## Default values
+# Global variables
 SSL=~/.ssl/cadcproxy.pem
 SESSION=https://ws-uv.canfar.net/skaha/v0/session
 IMAGE=images.canfar.net/unions/shapepipe
 NAME=shapepipe
 
+
+# Command line arguments
+
+## Default values
 job=-1
 ID=-1
+file_IDs=-1
 N_SMP=1
 kind=-1
 version=1.0
@@ -21,12 +22,14 @@ dry_run=0
 # TODO psf
 
 ## Help string
-usage="Usage: $(basename "$0") -j JOB -e ID  -k KIND [OPTIONS]
+usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS]
 \n\nOptions:\n
    -h\tthis message\n
    -j, --job JOB\tRUnning JOB, bit-coded\n
    -e, --exclusive ID
     \timage ID\n
+   -f, --file_IDs path
+    \tfile containing IDs\n
    -p, --psf MODEL\n
     \tPSF model, one in ['psfex'|'mccd'], default='$psf'\n
    -k, --kind KIND\n
@@ -62,6 +65,10 @@ while [ $# -gt 0 ]; do
       ID="$2"
       shift
       ;;
+    -f|--file_IDs)
+      file_IDs="$2"
+      shift
+      ;;
     -N|--N_SMP)
       n_SMP="$2"
       shift
@@ -84,8 +91,8 @@ if [ "$job" == "-1" ]; then
   exit 2                                                                        
 fi                                                                              
                                                                                 
-if [ "$exclusive" == "-1" ]; then                                               
-  echo "No image ID indicated, use option -e"                                   
+if [ "$ID" == "-1" ] && [ "$file_IDs" == "-1" ]; then                                               
+  echo "No image ID(s) indicated, use option -e ID or -f file_IDs"                                   
   exit 3                                                                        
 fi                                                                              
                                                                                 
@@ -109,14 +116,48 @@ else
 fi
 
 RESOURCES="ram=4&cores=$N_SMP"
-arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run"
+
+# TODO: dir as command line argument to this script
+dir=`pwd`
+arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
 
 
 if [ "$dry_run" == 2 ]; then
+
   echo "Running command dry run:"
-  echo curl -E $SSL $SESSION?$RESOURCES -d \"image=$IMAGE:$version\" -d \"name=${NAME}\" -d \"cmd=$cmd_remote\" --data-urlencode \"args=$arg\"
+
+  if [ "$ID" == "-1" ]; then
+
+    for ID in `cat $file_IDs`; do
+      arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+      echo curl -E $SSL $SESSION?$RESOURCES -d \"image=$IMAGE:$version\" -d \"name=${NAME}\" -d \"cmd=$cmd_remote\" --data-urlencode \"args=$arg\"
+    done
+
+  else
+
+    arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+    echo curl -E $SSL $SESSION?$RESOURCES -d \"image=$IMAGE:$version\" -d \"name=${NAME}\" -d \"cmd=$cmd_remote\" --data-urlencode \"args=$arg\"
+
+  fi
+
 else
-  session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg"`
-  echo $session >> session_IDs.txt
-  echo "$session $ID" >> session_image_IDs.txt
+
+  if [ "$ID" == "-1" ]; then
+
+    for ID in `cat $file_IDs`; do
+      arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+      session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg"`
+      echo $session >> session_IDs.txt
+      echo "$session $ID" >> session_image_IDs.txt
+    done
+
+  else
+
+    arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+    session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg"`
+    echo $session >> session_IDs.txt
+    echo "$session $ID" >> session_image_IDs.txt
+
+  fi
+
 fi

From f0acd65e274ef646c4e258ba710cf3eb63acc7a4 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 16 Jan 2024 15:56:22 +0000
Subject: [PATCH 086/133] summary: fixed main path, verbose

---
 scripts/python/summary_params_pre_v2.py    | 47 +++++++++++++---------
 scripts/python/summary_run.py              |  2 +-
 scripts/sh/init_run_sp_exclusive_canfar.sh | 25 ------------
 shapepipe/utilities/summary.py             | 14 +++++--
 4 files changed, 40 insertions(+), 48 deletions(-)
 delete mode 100755 scripts/sh/init_run_sp_exclusive_canfar.sh

diff --git a/scripts/python/summary_params_pre_v2.py b/scripts/python/summary_params_pre_v2.py
index ec310b237..635dc15da 100644
--- a/scripts/python/summary_params_pre_v2.py
+++ b/scripts/python/summary_params_pre_v2.py
@@ -39,10 +39,10 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
     print(f"Set job info for patch {patch}")
 
     # Main input and output directory
-    main_dir = f"{os.environ['HOME']}/cosmostat/v2/pre_v2/psfex/{patch}"
+    path_main = f"{os.environ['HOME']}/cosmostat/v2/pre_v2/psfex/{patch}"
 
     # Logging
-    path = f"{main_dir}/summary"
+    path = f"{path_main}/summary"
     if not os.path.isdir(path):
         os.mkdir(path)
     log_file_name = f"{path}/summary_log.txt"
@@ -51,7 +51,7 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         level=logging.INFO, format="%(message)s", handlers=handlers
     )
 
-    logging.info(f"Checking main directory = {main_dir}")
+    logging.info(f"Checking main directory = {path_main}")
 
     # Number of links created for retrieved images
     if retrieve == "vos":
@@ -60,7 +60,7 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         n_link = 1
 
     # Tile IDs
-    tile_ID_path = f"{main_dir}/tile_numbers.txt"
+    tile_ID_path = f"{path_main}/tile_numbers.txt"
 
     ## Tile IDs with dots
     list_tile_IDs_dot = get_IDs_from_file(tile_ID_path)
@@ -80,7 +80,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         ["tile_IDs", "tile_IDs", "exposures"],
         pattern=["CFIS_", "", ""],
         n_mult=[1 * n_link, 1, 3],
-        path_left=f"{main_dir}/output",
+        path_main=path_main,
+        path_left="output",
         verbose=verbose,
     )
 
@@ -90,13 +91,14 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         ["uncompress_fits_runner", "merge_headers_runner", "split_exp_runner"],
         ["tile_IDs", 0, "3*n_shdus+n_exposures"],
         n_mult=[1, 1, 1],
-        path_left=f"{main_dir}/output",
+        path_main=path_main,
+        path_left="output",
         verbose=verbose,
     )
 
-    if patch == "P3":
+    if patch == "PX":
         run_dir_mask_tiles = "run_sp_combined_flag"
-        run_dir_maks_exp = run_dir_tiles
+        run_dir_mask_exp = run_dir_mask_tiles
         mask_module_tiles = "mask_runner_run_1"
         mask_module_exp = "mask_runner_run_2"
     else:
@@ -110,7 +112,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         run_dir_mask_tiles,
         [mask_module_tiles],
         ["tile_IDs"],
-        path_left=f"{main_dir}/output",
+        path_main=path_main,
+        path_left="output",
         verbose=verbose,
     )
 
@@ -119,7 +122,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         run_dir_mask_exp,
         [mask_module_exp],
         ["shdus"],
-        path_left=f"{main_dir}/output",
+        path_main=path_main,
+        path_left="output",
         verbose=verbose,
     )
 
@@ -129,7 +133,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         ["sextractor_runner"],
         ["tile_IDs"],
         n_mult=2,
-        path_left=f"{main_dir}/tile_runs",
+        path_main=path_main,
+        path_left="tile_runs",
         output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
         verbose=verbose,
     )
@@ -150,7 +155,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         ],  # "psfex_interp_runner"],
         "shdus",
         n_mult=[2, 2, 2],  # 1],
-        path_left=f"{main_dir}/exp_runs",
+        path_main=path_main,
+        path_left="exp_runs",
         output_subdirs="shdus",
         path_right="output",
         verbose=verbose,
@@ -162,7 +168,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         "run_sp_exp_Pi",
         ["psfex_interp_runner"],
         "shdus",
-        path_left=f"{main_dir}/exp_runs",
+        path_main=path_main,
+        path_left="exp_runs",
         output_subdirs="shdus",
         path_right="output",
         verbose=verbose,
@@ -179,7 +186,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         ],
         "tile_IDs",
         n_mult=[1, 1, 1, 4],
-        path_left=f"{main_dir}/tile_runs",
+        path_main=path_main,
+        path_left="tile_runs",
         output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
         verbose=verbose,
     )
@@ -187,14 +195,15 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
     n_sh = 8
     run_dirs = [f"run_sp_tile_ngmix_Ng{idx+1}u" for idx in range(n_sh)]
     output_path_missing_IDs = [
-        f"missing_job_128_ngmix_runner_{idx+1}.txt" for idx in range(n_sh)
+        f"{path_main}/summary/missing_job_128_ngmix_runner_{idx+1}.txt" for idx in range(n_sh)
     ]
     jobs["128"] = job_data(
         "128",
         run_dirs,
         ["ngmix_runner"] * 8,
         "tile_IDs",
-        path_left=f"{main_dir}/tile_runs",
+        path_main=path_main,
+        path_left="tile_runs",
         output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
         output_path_missing_IDs=output_path_missing_IDs,
         verbose=verbose,
@@ -205,7 +214,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         ["run_sp_Ms", "run_sp_Mc"],
         ["merge_sep_cats_runner", "make_cat_runner"],
         "tile_IDs",
-        path_left=f"{main_dir}/tile_runs",
+        path_main=path_main,
+        path_left="tile_runs",
         output_subdirs=[f"{tile_ID}/output" for tile_ID in list_tile_IDs_dot],
         verbose=verbose,
     )
@@ -216,7 +226,8 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         "run_sp_combined_psf",
         ["psfex_interp_runner"],
         "shdus",
-        path_left=f"{main_dir}/output",
+        path_main=path_main,
+        path_left="output",
         verbose=verbose,
     )
     
diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index bb6e25d05..973d74bd4 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -11,7 +11,7 @@ def main(argv=None):
     patch = argv[1]
 
     retrieve = "vos"
-    verbose = True
+    verbose = False
     
     import summary_params_pre_v2 as params
 
diff --git a/scripts/sh/init_run_sp_exclusive_canfar.sh b/scripts/sh/init_run_sp_exclusive_canfar.sh
deleted file mode 100755
index 3d8e25c79..000000000
--- a/scripts/sh/init_run_sp_exclusive_canfar.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-echo "start init run sp exclusive canfar"
-
-. /opt/conda/etc/profile.d/conda.sh
-
-conda activate shapepipe
-
-basedir=$HOME/cosmostat/P3_v2/psfex
-cd $basedir
-
-
-ID=$1
-n_SMP=$2
-typ=$3
-echo "ID=$ID n_SMP=$n_SMP type=$typ"
-
-export SP_RUN=.
-export SP_CONFIG=$HOME/shapepipe/example/cfis
-shapepipe_run -c $SP_CONFIG/config_exp_Pi.ini -e $ID
-
-cd $basedir
-
-echo "end init run sp exclusive canfar"
-
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index c39e943c7..e594e7e8f 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -160,7 +160,7 @@ def check_special(module, paths_in_dir, names_in_dir):
                             print(f"b stars = {value}, not special")
                         break
 
-        print(inds_special)
+        #print(inds_special)
         for idx in inds_special:
             paths_in_dir.pop(idx)
             names_in_dir.pop(idx)
@@ -418,7 +418,7 @@ def output_missing(
         return missing_IDs_unique
 
     def output_missing_job(self):
-        output_path = f"missing_job_{self._bit}_all.txt"
+        output_path = f"{self._path_main}/summary/missing_job_{self._bit}_all.txt"
 
         missing_IDs_all = set(self._missing_IDs_job)
 
@@ -511,6 +511,9 @@ def get_names_in_dir(self, iterable, module, idx):
                 + f"{self._path_left}/{subdir}/"
                 + f"{self._path_right[idx]}"
             )
+            if self._verbose:
+                print(f"**** base_and_subdir {base_and_subdir}")
+
             if os.path.isdir(base_and_subdir):
 
                 matches = []
@@ -527,6 +530,9 @@ def get_names_in_dir(self, iterable, module, idx):
                     if not matches:
                         continue
 
+                    if self._verbose:
+                        print("Matching entries: ", matches)
+
                     full_path = self.get_last_full_path(
                         base_and_subdir, matches
                     )
@@ -539,8 +545,8 @@ def get_names_in_dir(self, iterable, module, idx):
                     # Find matching file names and paths
                     self.get_matches_final(directory, idx)
             else:
-                #print(f"Directory base_and_subdir {base_and_subdir} not found")
-                pass
+                if self._verbose:
+                    print(f"Directory {base_and_subdir} not found")
 
     def update_subdirs(self, par_runtime):
         """Update Subdirs.

From 1bc602b99aa9ab703ea9ac166b3599ba9edb089c Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 16 Jan 2024 16:09:14 +0000
Subject: [PATCH 087/133] Started canfar howto

---
 docs/source/canfar.md | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 docs/source/canfar.md

diff --git a/docs/source/canfar.md b/docs/source/canfar.md
new file mode 100644
index 000000000..991d53ea7
--- /dev/null
+++ b/docs/source/canfar.md
@@ -0,0 +1,31 @@
+
+Steps from testing to parallel running:
+
+1. In ID directory, run shapepipe with config file
+
+2. In ID directory, run job
+
+   job_sp_canfar -j JOB
+
+3. In base directory, run single image
+
+  First in dry mode
+
+  init_run_exclusive_canfar.sh -j 64 -e 271.281 -p psfex -k tile -n
+
+  Then for real, remove option -n
+
+4. In base dir, run single image with curl
+
+    First in dry-mode=2 (showing curl command)
+    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile -n 2
+
+    First in dry-mode=1 (using curl command)
+    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile -n 1
+
+    Then real run
+    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile
+
+5. In base dir, run collection of images
+
+    curl_canfar_local.sh -j 64 -f tile_numbers.txt -p psfex -k tile

From d6ff43722360503103f63a1f923b24ae1611494c Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 16 Jan 2024 16:09:37 +0000
Subject: [PATCH 088/133] Added summary run notebook

---
 scripts/jupyter/summary_run.ipynb | 368 ++++++++++++++++++++++++++++++
 1 file changed, 368 insertions(+)
 create mode 100644 scripts/jupyter/summary_run.ipynb

diff --git a/scripts/jupyter/summary_run.ipynb b/scripts/jupyter/summary_run.ipynb
new file mode 100644
index 000000000..26983d292
--- /dev/null
+++ b/scripts/jupyter/summary_run.ipynb
@@ -0,0 +1,368 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "130112a4-f2ca-4d26-b884-d8b054676f9c",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "c5e463a7-448f-4548-ab81-e660cfe57045",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/arc/home/kilbinger/shapepipe/scripts/jupyter\n",
+      "Defaulting to user installation because normal site-packages is not writeable\n",
+      "Processing /arc/home/kilbinger/shapepipe\n",
+      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25hRequirement already satisfied: joblib>=0.13 in /opt/conda/lib/python3.7/site-packages (from shapepipe==1.0.1) (1.2.0)\n",
+      "Requirement already satisfied: modopt>=1.2 in /arc/home/kilbinger/.local/lib/python3.7/site-packages (from shapepipe==1.0.1) (1.6.1)\n",
+      "Requirement already satisfied: numpy>=1.14 in /opt/conda/lib/python3.7/site-packages (from shapepipe==1.0.1) (1.21.6)\n",
+      "Requirement already satisfied: importlib-metadata>=3.7.0 in /opt/conda/lib/python3.7/site-packages (from modopt>=1.2->shapepipe==1.0.1) (4.11.4)\n",
+      "Requirement already satisfied: scipy>=1.5.4 in /opt/conda/lib/python3.7/site-packages (from modopt>=1.2->shapepipe==1.0.1) (1.7.3)\n",
+      "Requirement already satisfied: progressbar2>=3.53.1 in /arc/home/kilbinger/.local/lib/python3.7/site-packages (from modopt>=1.2->shapepipe==1.0.1) (4.2.0)\n",
+      "Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=3.7.0->modopt>=1.2->shapepipe==1.0.1) (4.5.0)\n",
+      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=3.7.0->modopt>=1.2->shapepipe==1.0.1) (3.15.0)\n",
+      "Requirement already satisfied: python-utils>=3.0.0 in /arc/home/kilbinger/.local/lib/python3.7/site-packages (from progressbar2>=3.53.1->modopt>=1.2->shapepipe==1.0.1) (3.5.2)\n",
+      "Building wheels for collected packages: shapepipe\n",
+      "  Building wheel for shapepipe (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25h  Created wheel for shapepipe: filename=shapepipe-1.0.1-py3-none-any.whl size=250346 sha256=bfed22a8e108ca0fa5f4243fe7723a481c32c2e5d6eef1af62a5262e63ab5f12\n",
+      "  Stored in directory: /tmp/pip-ephem-wheel-cache-nxs_30ro/wheels/83/8a/ec/cb0f2e7d438c89659ea7a80e8c13634e560f73bd6f82e48ce7\n",
+      "Successfully built shapepipe\n",
+      "Installing collected packages: shapepipe\n",
+      "  Attempting uninstall: shapepipe\n",
+      "    Found existing installation: shapepipe 1.0.1\n",
+      "    Uninstalling shapepipe-1.0.1:\n",
+      "      Successfully uninstalled shapepipe-1.0.1\n",
+      "Successfully installed shapepipe-1.0.1\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "!pip install ../.."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "196beca5-10a1-4cf5-9462-be145167cc70",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import shapepipe\n",
+    "from shapepipe.utilities.summary import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "ca63c72d-212c-463e-a792-71efbac0b908",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Setting\n",
+    "patch = \"P3\"\n",
+    "\n",
+    "retrieve = \"vos\"\n",
+    "verbose = False"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "dcb5604c-d61f-4705-8295-63875455cadb",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Load parameters\n",
+    "%run ~/shapepipe/scripts/python/summary_params_pre_v2 {patch}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "e69b7dab-1fea-4fcc-a8d9-0720e1d628c3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Checking main directory = /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P3\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Set job info for patch P3\n"
+     ]
+    }
+   ],
+   "source": [
+    "jobs, list_tile_IDs = set_jobs_v2_pre_v2(patch, retrieve, verbose)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "2c3a9dde-cf88-493f-926e-7ae7e8e10916",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Initialize runtime dicionary.\n",
+    "par_runtime = init_par_runtime(list_tile_IDs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "b7c63a22-ead1-4d6a-b081-a74ade515439",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "module                          expected     found   miss_expl missing uniq_miss  fr_found\n",
+      "====================================================================================================\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Start program\n",
+    "job_data.print_stats_header()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "4720ae18-0633-4646-b392-b1b24e0294c3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " (Job 1)\n",
+      "get_images_runner_run_1             2536      2536         0         0       0.0    100.0%\n",
+      "find_exposures_runner               1268      1268         0         0       0.0    100.0%\n",
+      "get_images_runner_run_2             3420      3420         0         0       0.0    100.0%\n"
+     ]
+    }
+   ],
+   "source": [
+    "for key in \"1\":\n",
+    "    job = jobs[key]\n",
+    "    job.print_intro()\n",
+    "    job.check_numbers(par_runtime=par_runtime, indices=[0, 1])\n",
+    "\n",
+    "    all_exposures = get_all_exposures(job._paths_in_dir[1], verbose=True)\n",
+    "    par_runtime[\"n_exposures\"] = len(all_exposures)\n",
+    "\n",
+    "    job.check_numbers(par_runtime, indices=[2])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "id": "f149f404-64e7-4d92-8f54-f300ed620130",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Update runtime dictionary with numbers of exposures\n",
+    "par_runtime = update_par_runtime_after_find_exp(par_runtime, all_exposures)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "607fed1b-005a-4d3a-811c-51b1c5049c19",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "print_par_runtime(par_runtime, verbose=verbose)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "78a9065f-8983-41cf-a34c-21892fc52dd2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Get all keys after \"1\"\n",
+    "keys = sorted(jobs.keys(), key=int)\n",
+    "_ = keys.pop(0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "79e39954-1155-4ca3-b0b2-64bc5670db53",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " (Job 2)\n",
+      "uncompress_fits_runner              1268      1268         0         0       0.0    100.0%\n",
+      "merge_headers_runner                   0         0         0         0       0.0    100.0%\n",
+      "split_exp_runner                  137940    137940         0         0       0.0    100.0%\n",
+      " (Job 4)\n",
+      "mask_runner                         1268      1268         0         0       0.0    100.0%\n",
+      " (Job 8)\n",
+      "mask_runner                        45600     45600         0         0       0.0    100.0%\n",
+      " (Job 16)\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m/tmp/ipykernel_48/963628441.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjobs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_intro\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_numbers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpar_runtime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpar_runtime\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/.local/lib/python3.7/site-packages/shapepipe/utilities/summary.py\u001b[0m in \u001b[0;36mcheck_numbers\u001b[0;34m(self, par_runtime, indices)\u001b[0m\n\u001b[1;32m    603\u001b[0m                 \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m                 \u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 605\u001b[0;31m                 \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    606\u001b[0m             )\n\u001b[1;32m    607\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.local/lib/python3.7/site-packages/shapepipe/utilities/summary.py\u001b[0m in \u001b[0;36mget_names_in_dir\u001b[0;34m(self, iterable, module, idx)\u001b[0m\n\u001b[1;32m    515\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"**** base_and_subdir {base_and_subdir}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 517\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbase_and_subdir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m                 \u001b[0mmatches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/opt/conda/lib/python3.7/genericpath.py\u001b[0m in \u001b[0;36misdir\u001b[0;34m(s)\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;34m\"\"\"Return true if the pathname refers to an existing directory.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m         \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "for key in keys:\n",
+    "    job = jobs[key]\n",
+    "    job.print_intro()\n",
+    "    job.check_numbers(par_runtime=par_runtime)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "id": "8d2243cb-5aa6-4e82-88f6-8689f15e5b61",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " (Job 64)\n",
+      "psfex_interp_runner                 1268       945         0       323     323.0     74.5%\n",
+      "vignetmaker_runner_run_1            1268       971         0       297     297.0     76.6%\n",
+      "spread_model_runner                 1268       173         0      1095    1095.0     13.6%\n",
+      "vignetmaker_runner_run_2            5072       690         0      4382    1095.5     13.6%\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Update some runs\n",
+    "for key in [\"64\"]:\n",
+    "    job = jobs[key]\n",
+    "    job.print_intro()\n",
+    "    job.check_numbers(par_runtime=par_runtime)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e8a38747-21b7-4f74-b667-62094f17154e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " (Job 64)\n",
+      "psfex_interp_runner                 1268       965         0       303     303.0     76.1%\n",
+      "vignetmaker_runner_run_1            1268       993         0       275     275.0     78.3%\n",
+      "spread_model_runner                 1268       187         0      1081    1081.0     14.7%\n"
+     ]
+    }
+   ],
+   "source": [
+    "for key in [\"64\"]:\n",
+    "    job = jobs[key]\n",
+    "    job.print_intro()\n",
+    "    job.check_numbers(par_runtime=par_runtime)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82cb7e0b-a8fa-4a55-840f-55ec0452bf7f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

From 1a81ae3b189a322589d837922f190da322be67bf Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 17 Jan 2024 15:27:54 +0000
Subject: [PATCH 089/133] psfex_interp: continue instead of error if one of the
 .psf files not found

---
 .../psfex_interp_package/psfex_interp.py      | 174 +++++++++---------
 1 file changed, 90 insertions(+), 84 deletions(-)

diff --git a/shapepipe/modules/psfex_interp_package/psfex_interp.py b/shapepipe/modules/psfex_interp_package/psfex_interp.py
index ea3db1de7..74064c296 100644
--- a/shapepipe/modules/psfex_interp_package/psfex_interp.py
+++ b/shapepipe/modules/psfex_interp_package/psfex_interp.py
@@ -29,82 +29,6 @@
 FILE_NOT_FOUND = 'File_not_found'
 
 
-def interpsfex(dotpsfpath, pos, thresh_star, thresh_chi2):
-    """Interpolate PSFEx.
-
-    Use PSFEx generated model to perform spatial PSF interpolation.
-
-    Parameters
-    ----------
-    dotpsfpath : str
-        Path to ``.psf`` file (PSFEx output)
-    pos : numpy.ndarray
-        Positions where the PSF model should be evaluated
-    thresh_star : int
-        Threshold of stars under which the PSF is not interpolated
-    thresh_chi2 : int
-        Threshold for chi squared
-
-    Returns
-    -------
-    numpy.ndarray
-        Array of PSFs, each row is the PSF image at the corresponding position
-        requested
-
-    """
-    if not os.path.exists(dotpsfpath):
-        return FILE_NOT_FOUND
-
-    # read PSF model and extract basis and polynomial degree and scale position
-    PSF_model = fits.open(dotpsfpath)[1]
-
-    # Check number of stars used to compute the PSF
-    if PSF_model.header['ACCEPTED'] < thresh_star:
-        return NOT_ENOUGH_STARS
-    if PSF_model.header['CHI2'] > thresh_chi2:
-        return BAD_CHI2
-
-    PSF_basis = np.array(PSF_model.data)[0][0]
-    try:
-        deg = PSF_model.header['POLDEG1']
-    except KeyError:
-        # constant PSF model
-        return PSF_basis[0, :, :]
-
-    # scale coordinates
-    x_interp, x_scale = (
-        PSF_model.header['POLZERO1'],
-        PSF_model.header['POLSCAL1']
-    )
-    y_interp, y_scale = (
-        PSF_model.header['POLZERO2'],
-        PSF_model.header['POLSCAL2']
-
-    )
-    xs, ys = (pos[:, 0] - x_interp) / x_scale, (pos[:, 1] - y_interp) / y_scale
-
-    # compute polynomial coefficients
-    coeffs = np.array([[x ** idx for idx in range(deg + 1)] for x in xs])
-    cross_coeffs = np.array([
-        np.concatenate([
-            [(x ** idx_j) * (y ** idx_i) for idx_j in range(deg - idx_i + 1)]
-            for idx_i in range(1, deg + 1)
-        ])
-        for x, y in zip(xs, ys)
-    ])
-    coeffs = np.hstack((coeffs, cross_coeffs))
-
-    # compute interpolated PSF
-    PSFs = np.array([
-        np.sum(
-            [coeff * atom for coeff, atom in zip(coeffs_posi, PSF_basis)],
-            axis=0,
-        )
-        for coeffs_posi in coeffs
-    ])
-
-    return PSFs
-
 
 class PSFExInterpolator(object):
     """The PSFEx Interpolator Class.
@@ -281,6 +205,88 @@ def _get_galaxy_positions(self):
             raise KeyError(pos_param_err)
         galcat.close()
 
+
+    def interpsfex(self, dotpsfpath, pos):
+        """Interpolate PSFEx.
+
+        Use PSFEx generated model to perform spatial PSF interpolation.
+
+        Parameters
+        ----------
+        dotpsfpath : str
+            Path to ``.psf`` file (PSFEx output)
+        pos : numpy.ndarray
+            Positions where the PSF model should be evaluated
+
+        Returns
+        -------
+        numpy.ndarray
+            Array of PSFs, each row is the PSF image at the corresponding position
+            requested
+    
+        """
+        if not os.path.exists(dotpsfpath):
+            return FILE_NOT_FOUND
+
+        # read PSF model and extract basis and polynomial degree and scale position
+        try:
+            PSF_model = fits.open(dotpsfpath)[1]
+        except OSError as err:
+            msg = f"Psf model file {dotpsfpath} empty or corrupt."
+            self._w_log.info(msg)
+            self._w_log.info(f"Error: {err=}, {type(err)=}")
+            raise
+
+        # Check number of stars used to compute the PSF
+        thresh_star = self._star_thresh
+        thresh_chi2 = self._chi2_thresh
+
+        if PSF_model.header['ACCEPTED'] < thresh_star:
+            return NOT_ENOUGH_STARS
+        if PSF_model.header['CHI2'] > thresh_chi2:
+            return BAD_CHI2
+
+        PSF_basis = np.array(PSF_model.data)[0][0]
+        try:
+            deg = PSF_model.header['POLDEG1']
+        except KeyError:
+            # constant PSF model
+            return PSF_basis[0, :, :]
+
+        # scale coordinates
+        x_interp, x_scale = (
+            PSF_model.header['POLZERO1'],
+            PSF_model.header['POLSCAL1']
+        )
+        y_interp, y_scale = (
+            PSF_model.header['POLZERO2'],
+            PSF_model.header['POLSCAL2']
+
+        )
+        xs, ys = (pos[:, 0] - x_interp) / x_scale, (pos[:, 1] - y_interp) / y_scale
+
+        # compute polynomial coefficients
+        coeffs = np.array([[x ** idx for idx in range(deg + 1)] for x in xs])
+        cross_coeffs = np.array([
+            np.concatenate([
+                [(x ** idx_j) * (y ** idx_i) for idx_j in range(deg - idx_i + 1)]
+                for idx_i in range(1, deg + 1)
+            ])
+            for x, y in zip(xs, ys)
+        ])
+        coeffs = np.hstack((coeffs, cross_coeffs))
+
+        # compute interpolated PSF
+        PSFs = np.array([
+            np.sum(
+                [coeff * atom for coeff, atom in zip(coeffs_posi, PSF_basis)],
+                axis=0,
+            )
+            for coeffs_posi in coeffs
+        ])
+
+        return PSFs
+
     def _interpolate(self):
         """Interpolate.
 
@@ -288,11 +294,9 @@ def _interpolate(self):
         positions.
 
         """
-        self.interp_PSFs = interpsfex(
+        self.interp_PSFs = self.interpsfex(
             self._dotpsf_path,
             self.gal_pos,
-            self._star_thresh,
-            self._chi2_thresh,
         )
 
     def _get_psfshapes(self):
@@ -608,7 +612,11 @@ def _interpolate_me(self):
                         found = True
                         break
                 if not found:
-                    raise ValueError("No dot psf file found")
+                    self._w_log.info(
+                        f"No .psf file found for exposure {exp_name} and"
+                        + f" ccd {ccd}"
+                    )
+                    continue
 
                 ind_obj = np.where(cat.get_data(hdu_index)['CCD_N'] == ccd)[0]
                 obj_id = all_id[ind_obj]
@@ -620,11 +628,9 @@ def _interpolate_me(self):
                     )
                 ).T
 
-                self.interp_PSFs = interpsfex(
+                self.interp_PSFs = self.interpsfex(
                     dot_psf_path,
                     gal_pos,
-                    self._star_thresh,
-                    self._chi2_thresh,
                 )
 
                 if (
@@ -632,7 +638,7 @@ def _interpolate_me(self):
                     and self.interp_PSFs == NOT_ENOUGH_STARS
                 ):
                     self._w_log.info(
-                        f'Not enough stars find in the ccd {ccd} of the '
+                        f'Not enough stars found on ccd {ccd} of '
                         + f'exposure {exp_name}. Object inside this ccd will '
                         + 'lose an epoch.'
                     )

From 93d6c6ef5a18334427ddcaf87d9ab868846a324a Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 17 Jan 2024 15:49:17 +0000
Subject: [PATCH 090/133] fixed symlink config files

---
 example/cfis/config_Gie_vos.ini          | 101 +++++++++++++++++++++++
 example/cfis/config_GitFeGie_symlink.ini |   6 +-
 example/cfis/config_GitFeGie_vos.ini     |   1 +
 3 files changed, 107 insertions(+), 1 deletion(-)
 create mode 100644 example/cfis/config_Gie_vos.ini

diff --git a/example/cfis/config_Gie_vos.ini b/example/cfis/config_Gie_vos.ini
new file mode 100644
index 000000000..a326434e7
--- /dev/null
+++ b/example/cfis/config_Gie_vos.ini
@@ -0,0 +1,101 @@
+# ShapePipe configuration file for: get images
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = False
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_Gie
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = get_images_runner            
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+# Get exposures
+[GET_IMAGES_RUNNER]
+
+INPUT_DIR = last:find_exposures_runner
+
+FILE_PATTERN = exp_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+
+# Paths
+
+# Output path (optional, default is [FILE]:OUTPUT_DIR
+# OUTPUT_PATH = input_images
+
+# Input path where original images are stored. Can be local path or vos url.
+# Single string or list of strings
+INPUT_PATH = vos:cfis/pitcairn, vos:cfis/weights, vos:cfis/flags
+# LSB images:
+#INPUT_PATH = vos:cfis/lsb_individual, vos:cfis/weights, vos:cfis/flags
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag
+# LSB images
+#INPUT_FILE_PATTERN = 000000s, 000000p.weight, 000000p.flag
+
+# Input file extensions
+INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{6}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = image-, weight-, flag-
+
+# Method to retrieve images, one in 'vos', 'symlink'
+RETRIEVE = vos
+
+# If RETRIEVE=vos, number of attempts to download
+# Optional, default=3
+N_TRY = 3
+
+# Retrieve command options, optional
+RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem
diff --git a/example/cfis/config_GitFeGie_symlink.ini b/example/cfis/config_GitFeGie_symlink.ini
index 971df44d4..6f84d1482 100644
--- a/example/cfis/config_GitFeGie_symlink.ini
+++ b/example/cfis/config_GitFeGie_symlink.ini
@@ -80,7 +80,7 @@ INPUT_NUMBERING = \d{3}\.\d{3}
 # Output file pattern without number
 OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight-
 
-CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output
+#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output
 
 # Copy/download method, one in 'vos', 'symlink'
 RETRIEVE = symlink
@@ -100,8 +100,12 @@ FILE_EXT = .fits
 # NUMBERING_SCHEME (optional) string with numbering pattern for input files
 NUMBERING_SCHEME = -000-000
 
+# Column number of exposure name in FITS header
 COLNUM = 3
 
+# Prefix to remove from exposure name
+EXP_PREFIX = p
+
 # Get exposures
 [GET_IMAGES_RUNNER_RUN_2]
 
diff --git a/example/cfis/config_GitFeGie_vos.ini b/example/cfis/config_GitFeGie_vos.ini
index eb06d372b..9ef72f198 100644
--- a/example/cfis/config_GitFeGie_vos.ini
+++ b/example/cfis/config_GitFeGie_vos.ini
@@ -86,6 +86,7 @@ RETRIEVE = vos
 # Copy command options, optional
 RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem
 
+CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output
 
 [FIND_EXPOSURES_RUNNER]
 

From 9c0b9a540666b2eaa0e8e5162d22327306acd195 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 17 Jan 2024 15:49:42 +0000
Subject: [PATCH 091/133] Updated summary run

---
 scripts/jupyter/summary_run.ipynb       | 185 +++++++++++++++---------
 scripts/python/summary_params_pre_v2.py |  34 ++---
 scripts/python/summary_run.py           |   3 +-
 3 files changed, 135 insertions(+), 87 deletions(-)

diff --git a/scripts/jupyter/summary_run.ipynb b/scripts/jupyter/summary_run.ipynb
index 26983d292..e57c72b1a 100644
--- a/scripts/jupyter/summary_run.ipynb
+++ b/scripts/jupyter/summary_run.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 17,
    "id": "130112a4-f2ca-4d26-b884-d8b054676f9c",
    "metadata": {
     "tags": []
@@ -22,50 +22,6 @@
     "%autoreload 2"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "c5e463a7-448f-4548-ab81-e660cfe57045",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "/arc/home/kilbinger/shapepipe/scripts/jupyter\n",
-      "Defaulting to user installation because normal site-packages is not writeable\n",
-      "Processing /arc/home/kilbinger/shapepipe\n",
-      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
-      "\u001b[?25hRequirement already satisfied: joblib>=0.13 in /opt/conda/lib/python3.7/site-packages (from shapepipe==1.0.1) (1.2.0)\n",
-      "Requirement already satisfied: modopt>=1.2 in /arc/home/kilbinger/.local/lib/python3.7/site-packages (from shapepipe==1.0.1) (1.6.1)\n",
-      "Requirement already satisfied: numpy>=1.14 in /opt/conda/lib/python3.7/site-packages (from shapepipe==1.0.1) (1.21.6)\n",
-      "Requirement already satisfied: importlib-metadata>=3.7.0 in /opt/conda/lib/python3.7/site-packages (from modopt>=1.2->shapepipe==1.0.1) (4.11.4)\n",
-      "Requirement already satisfied: scipy>=1.5.4 in /opt/conda/lib/python3.7/site-packages (from modopt>=1.2->shapepipe==1.0.1) (1.7.3)\n",
-      "Requirement already satisfied: progressbar2>=3.53.1 in /arc/home/kilbinger/.local/lib/python3.7/site-packages (from modopt>=1.2->shapepipe==1.0.1) (4.2.0)\n",
-      "Requirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=3.7.0->modopt>=1.2->shapepipe==1.0.1) (4.5.0)\n",
-      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=3.7.0->modopt>=1.2->shapepipe==1.0.1) (3.15.0)\n",
-      "Requirement already satisfied: python-utils>=3.0.0 in /arc/home/kilbinger/.local/lib/python3.7/site-packages (from progressbar2>=3.53.1->modopt>=1.2->shapepipe==1.0.1) (3.5.2)\n",
-      "Building wheels for collected packages: shapepipe\n",
-      "  Building wheel for shapepipe (setup.py) ... \u001b[?25ldone\n",
-      "\u001b[?25h  Created wheel for shapepipe: filename=shapepipe-1.0.1-py3-none-any.whl size=250346 sha256=bfed22a8e108ca0fa5f4243fe7723a481c32c2e5d6eef1af62a5262e63ab5f12\n",
-      "  Stored in directory: /tmp/pip-ephem-wheel-cache-nxs_30ro/wheels/83/8a/ec/cb0f2e7d438c89659ea7a80e8c13634e560f73bd6f82e48ce7\n",
-      "Successfully built shapepipe\n",
-      "Installing collected packages: shapepipe\n",
-      "  Attempting uninstall: shapepipe\n",
-      "    Found existing installation: shapepipe 1.0.1\n",
-      "    Uninstalling shapepipe-1.0.1:\n",
-      "      Successfully uninstalled shapepipe-1.0.1\n",
-      "Successfully installed shapepipe-1.0.1\n"
-     ]
-    }
-   ],
-   "source": [
-    "\n",
-    "!pip install ../.."
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 18,
@@ -81,7 +37,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 19,
    "id": "ca63c72d-212c-463e-a792-71efbac0b908",
    "metadata": {
     "tags": []
@@ -89,10 +45,9 @@
    "outputs": [],
    "source": [
     "# Setting\n",
-    "patch = \"P3\"\n",
+    "patch = \"P1\"\n",
     "\n",
-    "retrieve = \"vos\"\n",
-    "verbose = False"
+    "verbose = True"
    ]
   },
   {
@@ -110,7 +65,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 21,
    "id": "e69b7dab-1fea-4fcc-a8d9-0720e1d628c3",
    "metadata": {
     "tags": []
@@ -120,24 +75,24 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Checking main directory = /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P3\n"
+      "Checking main directory = /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P1\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Set job info for patch P3\n"
+      "Set job info for patch P1\n"
      ]
     }
    ],
    "source": [
-    "jobs, list_tile_IDs = set_jobs_v2_pre_v2(patch, retrieve, verbose)"
+    "jobs, list_tile_IDs = set_jobs_v2_pre_v2(patch, verbose)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 22,
    "id": "2c3a9dde-cf88-493f-926e-7ae7e8e10916",
    "metadata": {
     "tags": []
@@ -150,7 +105,34 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 23,
+   "id": "1e9c1487-3cec-4394-9fcf-c12e92a0f984",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===========\n",
+      "par_runtime\n",
+      "-----------\n",
+      "n_tile_IDs                       3527\n",
+      "list_tile_IDs                   3527 entries\n",
+      "===========\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "print_par_runtime(par_runtime, verbose=verbose)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
    "id": "b7c63a22-ead1-4d6a-b081-a74ade515439",
    "metadata": {
     "tags": []
@@ -172,7 +154,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 27,
    "id": "4720ae18-0633-4646-b392-b1b24e0294c3",
    "metadata": {
     "tags": []
@@ -183,9 +165,39 @@
      "output_type": "stream",
      "text": [
       " (Job 1)\n",
-      "get_images_runner_run_1             2536      2536         0         0       0.0    100.0%\n",
-      "find_exposures_runner               1268      1268         0         0       0.0    100.0%\n",
-      "get_images_runner_run_2             3420      3420         0         0       0.0    100.0%\n"
+      "get_images_runner_run_1             7054      4402         0      2652    1326.0     62.4%\n",
+      "find_exposures_runner               3527       875         0      2652    2652.0     24.8%\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "** module get_images_runner_run_1\n",
+      "*** subdirs ['']\n",
+      "**** base_and_subdir /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P1/output//.\n",
+      "Matching entries:  [<DirEntry 'run_sp_Git'>, <DirEntry 'run_sp_GitFeGie_2024-01-16_22-24-51'>]\n",
+      "** module find_exposures_runner\n",
+      "*** subdirs ['']\n",
+      "**** base_and_subdir /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P1/output//.\n",
+      "Matching entries:  [<DirEntry 'run_sp_Git'>, <DirEntry 'run_sp_GitFeGie_2024-01-16_22-24-51'>]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "get_images_runner_run_2             7119      1509         0      5610    1870.0     21.2%\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "** module get_images_runner_run_2\n",
+      "*** subdirs ['']\n",
+      "**** base_and_subdir /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P1/output//.\n",
+      "Matching entries:  [<DirEntry 'run_sp_Git'>, <DirEntry 'run_sp_GitFeGie_2024-01-16_22-24-51'>]\n"
      ]
     }
    ],
@@ -195,15 +207,16 @@
     "    job.print_intro()\n",
     "    job.check_numbers(par_runtime=par_runtime, indices=[0, 1])\n",
     "\n",
-    "    all_exposures = get_all_exposures(job._paths_in_dir[1], verbose=True)\n",
+    "    all_exposures = get_all_exposures(job._paths_in_dir[1], verbose=verbose)\n",
     "    par_runtime[\"n_exposures\"] = len(all_exposures)\n",
+    "    par_runtime[\"list_exposures\"] = all_exposures\n",
     "\n",
     "    job.check_numbers(par_runtime, indices=[2])"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 28,
    "id": "f149f404-64e7-4d92-8f54-f300ed620130",
    "metadata": {
     "tags": []
@@ -216,12 +229,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 29,
    "id": "607fed1b-005a-4d3a-811c-51b1c5049c19",
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===========\n",
+      "par_runtime\n",
+      "-----------\n",
+      "n_tile_IDs                       3527\n",
+      "list_tile_IDs                   3527 entries\n",
+      "n_exposures                      2373\n",
+      "list_exposures                  2373 entries\n",
+      "n_shdus                         94920\n",
+      "list_shdus                     94920 entries\n",
+      "n_3*n_shdus+n_exposures        287133\n",
+      "===========\n",
+      "\n"
+     ]
+    }
+   ],
    "source": [
     "print_par_runtime(par_runtime, verbose=verbose)"
    ]
@@ -313,7 +346,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 35,
    "id": "e8a38747-21b7-4f74-b667-62094f17154e",
    "metadata": {},
    "outputs": [
@@ -324,7 +357,8 @@
       " (Job 64)\n",
       "psfex_interp_runner                 1268       965         0       303     303.0     76.1%\n",
       "vignetmaker_runner_run_1            1268       993         0       275     275.0     78.3%\n",
-      "spread_model_runner                 1268       187         0      1081    1081.0     14.7%\n"
+      "spread_model_runner                 1268       187         0      1081    1081.0     14.7%\n",
+      "vignetmaker_runner_run_2            5072       749         0      4323    1080.8     14.8%\n"
      ]
     }
    ],
@@ -340,6 +374,27 @@
    "execution_count": null,
    "id": "82cb7e0b-a8fa-4a55-840f-55ec0452bf7f",
    "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " (Job 64)\n"
+     ]
+    }
+   ],
+   "source": [
+    "for key in [\"64\"]:\n",
+    "    job = jobs[key]\n",
+    "    job.print_intro()\n",
+    "    job.check_numbers(par_runtime=par_runtime)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c57c576f-b7e8-4735-a047-5b63e245f9a4",
+   "metadata": {},
    "outputs": [],
    "source": []
   }
diff --git a/scripts/python/summary_params_pre_v2.py b/scripts/python/summary_params_pre_v2.py
index 635dc15da..31b63cb3c 100644
--- a/scripts/python/summary_params_pre_v2.py
+++ b/scripts/python/summary_params_pre_v2.py
@@ -32,7 +32,7 @@ def update_par_runtime_after_find_exp(par_runtime, all_exposures):
     return par_runtime
 
 
-def set_jobs_v2_pre_v2(patch, retrieve, verbose):
+def set_jobs_v2_pre_v2(patch, verbose):
     """ Return information about shapepipe jobs
     
     """
@@ -53,12 +53,6 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
 
     logging.info(f"Checking main directory = {path_main}")
 
-    # Number of links created for retrieved images
-    if retrieve == "vos":
-        n_link = 2
-    else:
-        n_link = 1
-
     # Tile IDs
     tile_ID_path = f"{path_main}/tile_numbers.txt"
 
@@ -67,8 +61,14 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
 
     jobs = {}
 
-#        "run_sp_GitFeGie_",
     # Set the first job (retrieve images)
+    
+    # With "CFIS_" only the linked images are counted. The original
+    # ones do not match the IDdash pattern.
+    # If images were downloaded in several runs:
+    # - Only copy original images, then (re-)set links in SP numbering format
+    # - get_images_runner_run_[12] consistent
+    # - remove previous output dirs since only last is searched
     jobs["1"] = job_data(
         1,
          "run_sp_Git",
@@ -79,33 +79,27 @@ def set_jobs_v2_pre_v2(patch, retrieve, verbose):
         ],
         ["tile_IDs", "tile_IDs", "exposures"],
         pattern=["CFIS_", "", ""],
-        n_mult=[1 * n_link, 1, 3],
+        n_mult=[2, 1, 3],
         path_main=path_main,
         path_left="output",
         verbose=verbose,
     )
 
+    #        n_mult=[1, 1, 1],
     jobs["2"] = job_data(
         2,
         ["run_sp_Uz", "run_sp_exp_SpMh", "run_sp_exp_SpMh_2023-12"],
         ["uncompress_fits_runner", "merge_headers_runner", "split_exp_runner"],
         ["tile_IDs", 0, "3*n_shdus+n_exposures"],
-        n_mult=[1, 1, 1],
         path_main=path_main,
         path_left="output",
         verbose=verbose,
     )
 
-    if patch == "PX":
-        run_dir_mask_tiles = "run_sp_combined_flag"
-        run_dir_mask_exp = run_dir_mask_tiles
-        mask_module_tiles = "mask_runner_run_1"
-        mask_module_exp = "mask_runner_run_2"
-    else:
-        run_dir_mask_tiles = "run_sp_tile_Ma"
-        run_dir_mask_exp = "run_sp_exp_Ma"
-        mask_module_tiles = "mask_runner"
-        mask_module_exp = "mask_runner"
+    run_dir_mask_tiles = "run_sp_tile_Ma"
+    run_dir_mask_exp = "run_sp_exp_Ma"
+    mask_module_tiles = "mask_runner"
+    mask_module_exp = "mask_runner"
 
     jobs["4"] = job_data(
         4,
diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index 973d74bd4..b790300af 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -10,12 +10,11 @@ def main(argv=None):
 
     patch = argv[1]
 
-    retrieve = "vos"
     verbose = False
     
     import summary_params_pre_v2 as params
 
-    jobs, list_tile_IDs_dot = params.set_jobs_v2_pre_v2(patch, retrieve, verbose)
+    jobs, list_tile_IDs_dot = params.set_jobs_v2_pre_v2(patch, verbose)
     
     list_tile_IDs = job_data.replace_dot_dash(list_tile_IDs_dot)
     

From b19ad035208b5a53a9618ec62e08fcf9efb866c4 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 17 Jan 2024 15:50:14 +0000
Subject: [PATCH 092/133] revert to main psfex in link exp for tiles script

---
 scripts/python/link_to_exp_for_tile.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index fc8ca9a69..0ad205adf 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -252,7 +252,7 @@ def get_paths(exp_base_dir, exp_shdu_IDs, pattern):
 
         if n_subdirs != 1:
             msg = (
-                f"Exactly on directory matching {pattern} in {path} expected,"
+                f"Exactly one directory matching {pattern} in {path} expected,"
                 + f"  not {n_subdirs}"
             )
             print(msg)
@@ -324,7 +324,8 @@ def main(argv=None):
     exp_IDs = get_exp_IDs(tile_base_dir, tile_ID, verbose=verbose)
     exp_shdu_IDs = get_exp_single_HDU_IDs(exp_IDs, n_CPU)
 
-    patterns = ["run_sp_exp_SxSePsf"] #, "run_sp_exp_Pi"]
+    # Note: psfex P3 is mostly run_sp_exp_SxSePsf
+    patterns = ["run_sp_exp_SxSePsfPi"] #, "run_sp_exp_Pi"]
     for pattern in patterns:
         paths, number = get_paths(exp_base_dir, exp_shdu_IDs, pattern)
 

From af3855608a58c903fc8924fd88c9ffd4b93dfbfa Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 17 Jan 2024 15:50:47 +0000
Subject: [PATCH 093/133] curl job script: remoging session logs

---
 scripts/sh/curl_canfar_local.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index 4b85d7cef..ed4d21bd5 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -142,6 +142,8 @@ if [ "$dry_run" == 2 ]; then
 
 else
 
+  rm -rf session_IDs.txt session_image_IDs.txt
+
   if [ "$ID" == "-1" ]; then
 
     for ID in `cat $file_IDs`; do

From 8c9921f80f10e7e887e0f85ae01407af9309f7ae Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 17 Jan 2024 15:51:22 +0000
Subject: [PATCH 094/133] comments

---
 shapepipe/modules/mccd_package/mccd_interpolation_script.py | 2 --
 shapepipe/utilities/summary.py                              | 5 +++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/shapepipe/modules/mccd_package/mccd_interpolation_script.py b/shapepipe/modules/mccd_package/mccd_interpolation_script.py
index 40e644e8b..c899b2b8f 100644
--- a/shapepipe/modules/mccd_package/mccd_interpolation_script.py
+++ b/shapepipe/modules/mccd_package/mccd_interpolation_script.py
@@ -435,8 +435,6 @@ def _interpolate_me(self):
                 ).T
 
                 self.interp_PSFs = interp_MCCD(mccd_model_path, gal_pos, ccd)
-                # self.interp_PSFs = interpsfex(
-                # dot_psf_path, gal_pos, self._star_thresh, self._chi2_thresh)
 
                 if (
                     isinstance(self.interp_PSFs, str)
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index e594e7e8f..001e975f6 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -412,8 +412,7 @@ def output_missing(
                 )
             else:
                 output_path = self._output_path_missing_IDs[idx]
-
-        self.write_IDs_to_file(output_path, missing_IDs_unique)
+            self.write_IDs_to_file(output_path, missing_IDs_unique)
 
         return missing_IDs_unique
 
@@ -541,6 +540,8 @@ def get_names_in_dir(self, iterable, module, idx):
                     directory = self.get_module_output_dir(
                         full_path, module
                     )
+                    if self._verbose:
+                        print(f"**** Output dir = {directory}")
 
                     # Find matching file names and paths
                     self.get_matches_final(directory, idx)

From 4276feaa62d958b2bd1dde065b0d742ef7d654df Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 19 Jan 2024 16:49:33 +0100
Subject: [PATCH 095/133] Update canfar.md

Started md and cleanup of canfar doc
---
 docs/source/canfar.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/source/canfar.md b/docs/source/canfar.md
index 991d53ea7..d1779a153 100644
--- a/docs/source/canfar.md
+++ b/docs/source/canfar.md
@@ -1,5 +1,12 @@
+# Running `shapepipe` on the canfar science portal
+
+## Introduction
+
+## Steps from testing to parallel running
+
+Before starting a batch remote session job on a large number of images (step 6.),
+it is recommended to perform some or all of the testing steps (1. - 5.).
 
-Steps from testing to parallel running:
 
 1. In ID directory, run shapepipe with config file
 

From 3f1553315dd6ca71c3c533bb2744786797294893 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 19 Jan 2024 16:53:13 +0100
Subject: [PATCH 096/133] Update canfar.md

continue editing
---
 docs/source/canfar.md | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/docs/source/canfar.md b/docs/source/canfar.md
index d1779a153..e743e72bc 100644
--- a/docs/source/canfar.md
+++ b/docs/source/canfar.md
@@ -8,13 +8,19 @@ Before starting a batch remote session job on a large number of images (step 6.)
 it is recommended to perform some or all of the testing steps (1. - 5.).
 
 
-1. In ID directory, run shapepipe with config file
-
-2. In ID directory, run job
-
-   job_sp_canfar -j JOB
-
-3. In base directory, run single image
+1. Run the basic `shapepipe` runner script to test (one or several) modules in question, specified by a given config file, on one image.
+   This step has to be run in the image run directory. The command is
+   ```bash
+   shapepipe_run -c config.ini
+   ```
+
+2. Run the job script to test the job management, on one image.
+   This step has to be run in the image run directory. The command is
+   ```bash
+   job_sp_canfar -j JOB [OPTIONS]
+   ```
+
+5. In base directory, run single image
 
   First in dry mode
 

From cfefcfb000f6607b0279d48bad40bf340b6e8d31 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 19 Jan 2024 17:40:52 +0100
Subject: [PATCH 097/133] Update canfar.md

added monitoring and status
---
 docs/source/canfar.md | 80 ++++++++++++++++++++++++++++++-------------
 1 file changed, 56 insertions(+), 24 deletions(-)

diff --git a/docs/source/canfar.md b/docs/source/canfar.md
index e743e72bc..7a83e47e6 100644
--- a/docs/source/canfar.md
+++ b/docs/source/canfar.md
@@ -4,8 +4,8 @@
 
 ## Steps from testing to parallel running
 
-Before starting a batch remote session job on a large number of images (step 6.),
-it is recommended to perform some or all of the testing steps (1. - 5.).
+Before starting a batch remote session job on a large number of images (step 5.),
+it is recommended to perform some or all of the testing steps (1. - 4.).
 
 
 1. Run the basic `shapepipe` runner script to test (one or several) modules in question, specified by a given config file, on one image.
@@ -20,25 +20,57 @@ it is recommended to perform some or all of the testing steps (1. - 5.).
    job_sp_canfar -j JOB [OPTIONS]
    ```
 
-5. In base directory, run single image
-
-  First in dry mode
-
-  init_run_exclusive_canfar.sh -j 64 -e 271.281 -p psfex -k tile -n
-
-  Then for real, remove option -n
-
-4. In base dir, run single image with curl
-
-    First in dry-mode=2 (showing curl command)
-    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile -n 2
-
-    First in dry-mode=1 (using curl command)
-    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile -n 1
-
-    Then real run
-    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile
-
-5. In base dir, run collection of images
-
-    curl_canfar_local.sh -j 64 -f tile_numbers.txt -p psfex -k tile
+3. Run the pipeline script to test the processing step(s), on one image.
+   This step has to be run in the patch base directory.
+
+   1. First, run in dry mode:
+      ```bash
+      init_run_exclusive_canfar.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n
+      ```
+   2. Next, perform a real run with
+      ```bash
+      init_run_exclusive_canfar.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n
+      ```
+
+4. Run remote session script to test job submission using docker images, on one image.
+   This step has to be run in the patch base directory.
+   1. First, run in dry mode=2, to display curl command, with
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n 2
+      ```
+
+   2. Next, run in dry mode=1, to use curl command without processing:
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n 1
+      ```
+   3. Then, perform a real run, to use curl with processing:
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp]
+      ```
+   
+5. Full run: Call remote session script and docker image with collection of images
+      ```bash
+      curl_canfar_local.sh -j JOB -f path_IDs -p [psfex|mccd] -k [tile|exp]
+      ```
+      with `path_IDs` being a text file with one image ID per line.
+
+## Monitoring
+
+Monitoring of the currently active remote session can be performed using the session IDs `session_IDs.txt` written by the
+remote session script `curl_canfar_local.sh`. In the patch main directory, run
+```bash
+curl_canfar_monitor.sh logs
+```
+to print `stdout` of the remotely run pipeline script, and
+```bash
+curl_canfar_monitor.sh events
+```
+to display the remotely started docker image status.
+
+## Post-hoc summary
+
+In the patch main directory, run
+```bash
+summary_run PATCH
+```
+to print a summary with missing image IDs per job and module.

From 24480e2b1da1612cd9e390488954d817ef08c4f1 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 19 Jan 2024 17:41:07 +0100
Subject: [PATCH 098/133] Update canfar.md

monitoring
---
 docs/source/canfar.md | 80 ++++++++++++++++++++++++++++++-------------
 1 file changed, 56 insertions(+), 24 deletions(-)

diff --git a/docs/source/canfar.md b/docs/source/canfar.md
index e743e72bc..7a83e47e6 100644
--- a/docs/source/canfar.md
+++ b/docs/source/canfar.md
@@ -4,8 +4,8 @@
 
 ## Steps from testing to parallel running
 
-Before starting a batch remote session job on a large number of images (step 6.),
-it is recommended to perform some or all of the testing steps (1. - 5.).
+Before starting a batch remote session job on a large number of images (step 5.),
+it is recommended to perform some or all of the testing steps (1. - 4.).
 
 
 1. Run the basic `shapepipe` runner script to test (one or several) modules in question, specified by a given config file, on one image.
@@ -20,25 +20,57 @@ it is recommended to perform some or all of the testing steps (1. - 5.).
    job_sp_canfar -j JOB [OPTIONS]
    ```
 
-5. In base directory, run single image
-
-  First in dry mode
-
-  init_run_exclusive_canfar.sh -j 64 -e 271.281 -p psfex -k tile -n
-
-  Then for real, remove option -n
-
-4. In base dir, run single image with curl
-
-    First in dry-mode=2 (showing curl command)
-    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile -n 2
-
-    First in dry-mode=1 (using curl command)
-    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile -n 1
-
-    Then real run
-    curl_canfar_local.sh -j 64 -e 271.282 -p psfex -k tile
-
-5. In base dir, run collection of images
-
-    curl_canfar_local.sh -j 64 -f tile_numbers.txt -p psfex -k tile
+3. Run the pipeline script to test the processing step(s), on one image.
+   This step has to be run in the patch base directory.
+
+   1. First, run in dry mode:
+      ```bash
+      init_run_exclusive_canfar.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n
+      ```
+   2. Next, perform a real run with
+      ```bash
+      init_run_exclusive_canfar.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n
+      ```
+
+4. Run remote session script to test job submission using docker images, on one image.
+   This step has to be run in the patch base directory.
+   1. First, run in dry mode=2, to display curl command, with
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n 2
+      ```
+
+   2. Next, run in dry mode=1, to use curl command without processing:
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp] -n 1
+      ```
+   3. Then, perform a real run, to use curl with processing:
+      ```bash
+      curl_canfar_local.sh -j JOB -e ID -p [psfex|mccd] -k [tile|exp]
+      ```
+   
+5. Full run: Call remote session script and docker image with collection of images
+      ```bash
+      curl_canfar_local.sh -j JOB -f path_IDs -p [psfex|mccd] -k [tile|exp]
+      ```
+      with `path_IDs` being a text file with one image ID per line.
+
+## Monitoring
+
+Monitoring of the currently active remote session can be performed using the session IDs `session_IDs.txt` written by the
+remote session script `curl_canfar_local.sh`. In the patch main directory, run
+```bash
+curl_canfar_monitor.sh logs
+```
+to print `stdout` of the remotely run pipeline script, and
+```bash
+curl_canfar_monitor.sh events
+```
+to display the remotely started docker image status.
+
+## Post-hoc summary
+
+In the patch main directory, run
+```bash
+summary_run PATCH
+```
+to print a summary with missing image IDs per job and module.

From 9a964c5fcf94f1bbc0ddd3176b8aa2a21a1a8adc Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 26 Jan 2024 10:55:39 +0000
Subject: [PATCH 099/133] updated canfar doc

---
 docs/source/canfar.md | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/docs/source/canfar.md b/docs/source/canfar.md
index 7a83e47e6..1f490c504 100644
--- a/docs/source/canfar.md
+++ b/docs/source/canfar.md
@@ -56,16 +56,28 @@ it is recommended to perform some or all of the testing steps (1. - 4.).
 
 ## Monitoring
 
+
+### Status and output of submitted job 
+
 Monitoring of the currently active remote session can be performed using the session IDs `session_IDs.txt` written by the
 remote session script `curl_canfar_local.sh`. In the patch main directory, run
 ```bash
+curl_canfar_monitor.sh events
+```
+to display the remotely started docker image status, and
+```bash
 curl_canfar_monitor.sh logs
 ```
-to print `stdout` of the remotely run pipeline script, and
+to print `stdout` of the remotely run pipeline script.
+
+### Number of submitted running jobs
+
+The script
 ```bash
-curl_canfar_monitor.sh events
+stats_headless_canfar.py
 ```
-to display the remotely started docker image status.
+returns the number of actively running headless jobs.
+
 
 ## Post-hoc summary
 
@@ -74,3 +86,9 @@ In the patch main directory, run
 summary_run PATCH
 ```
 to print a summary with missing image IDs per job and module.
+
+## Deleting jobs
+
+```bash
+ for id in `cat session_IDs.txt`; do echo $id; curl -X DELETE -E /arc/home/kilbinger/.ssl/cadcproxy.pem https://ws-uv.canfar.net/skaha/v0/session/$id;  done
+ ```

From e0100cad201ca91a0fec9f75a53ee6cc52eacdd6 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 26 Jan 2024 10:56:18 +0000
Subject: [PATCH 100/133] script with akaha lib to count headlerss jobs

---
 environment.yml                         |  1 +
 scripts/python/stats_headless_canfar.py | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+)
 create mode 100755 scripts/python/stats_headless_canfar.py

diff --git a/environment.yml b/environment.yml
index ea42dda00..548c7e618 100644
--- a/environment.yml
+++ b/environment.yml
@@ -22,6 +22,7 @@ dependencies:
     - PyQt5
     - pyqtgraph
     - reproject
+    - skaha
     - sip_tpv
     - sf_tools
     - sqlitedict
diff --git a/scripts/python/stats_headless_canfar.py b/scripts/python/stats_headless_canfar.py
new file mode 100755
index 000000000..1ede78eed
--- /dev/null
+++ b/scripts/python/stats_headless_canfar.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+# Name: stats_headless_canfar.py
+
+import sys
+from skaha.session import Session
+
+
+def main(argv=None):
+    session = Session()
+
+    n_headless = session.stats()["instances"]["headless"]
+
+    print(n_headless)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

From 984ec90c259fd02cab49f6c5c10c275810a779d5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 26 Jan 2024 10:56:59 +0000
Subject: [PATCH 101/133] cleaned up curl submit script

---
 scripts/sh/curl_canfar_local.sh | 92 ++++++++++++++++++++++++++++-----
 1 file changed, 80 insertions(+), 12 deletions(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index ed4d21bd5..b02f9205f 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -119,46 +119,114 @@ RESOURCES="ram=4&cores=$N_SMP"
 
 # TODO: dir as command line argument to this script
 dir=`pwd`
-arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+#arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+
+
+# Return argument for local script to be called via curl
+function set_arg() {
+  my_arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+  echo $my_arg
+}
+
+
+function call_curl() {
+  my_arg=$1
+
+}
+
+function update_session_logs() {
+  echo $my_session >> session_IDs.txt
+  echo "$my_session $ID" >> session_image_IDs.txt
+
+}
+
+function submit_batch() {
+  path=$1
+
+  for ID in `cat $path`; do
+    my_arg=$(set_arg)
+    my_session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$my_arg"`
+    update_session_logs
+  done
+
+}
+
+n_max=200
+batch=20
+sleep=300
+
+((n_thresh=n_max- batch))
 
 
 if [ "$dry_run" == 2 ]; then
 
+  # Do not call curl (dry run = 2)
   echo "Running command dry run:"
 
   if [ "$ID" == "-1" ]; then
 
+    # Submit file (dry run = 2)
     for ID in `cat $file_IDs`; do
-      arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+      arg=$(set_arg)
       echo curl -E $SSL $SESSION?$RESOURCES -d \"image=$IMAGE:$version\" -d \"name=${NAME}\" -d \"cmd=$cmd_remote\" --data-urlencode \"args=$arg\"
     done
 
   else
 
-    arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+    # Submit image (dry run = 2)
+    arg=$(set_arg)
     echo curl -E $SSL $SESSION?$RESOURCES -d \"image=$IMAGE:$version\" -d \"name=${NAME}\" -d \"cmd=$cmd_remote\" --data-urlencode \"args=$arg\"
 
   fi
 
 else
 
+  # Call curl
   rm -rf session_IDs.txt session_image_IDs.txt
 
   if [ "$ID" == "-1" ]; then
 
-    for ID in `cat $file_IDs`; do
-      arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
-      session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg"`
-      echo $session >> session_IDs.txt
-      echo "$session $ID" >> session_image_IDs.txt
-    done
+    # Submit file
+    n_jobs=`cat $file_IDs | wc -l`
+    if [ "$n_jobs" -gt "$n_max" ]; then
+
+      # Split into batches 
+      prefix="${file_IDs}_split_"
+      split -d -l $batch $file_IDs $prefix
+      n_split=`ls -l $prefix* | wc -l`
+      echo "Split '$file_IDs' into $n_split batches of size $batch"
+
+      count=1
+      for batch in $prefix*; do
+        echo "Number of running jobs = $n_running"
+        echo "Submitting batch $batch ($count/$n_split)"
+        submit_batch $batch
+        ((count=count+1))
+
+        n_running=`stats_headless_canfar.py`
+
+        while [ "$n_running" -gt "$n_thresh" ]; do
+          echo "Wait for #jobs = $n_running jobs to go < $n_thresh ..."
+          sleep $sleep
+          n_running=`stats_headless_canfar.py`
+        done
+
+      done
+
+    else
+
+      # Submit entire file (single batch)
+      echo "Submit '$file_IDs' in single batch"
+      submit_batch $file_IDs
+
+    fi
 
   else
 
-    arg="-j $job -e $ID -N $N_SMP -k $kind $arg_dry_run -d $dir"
+    # Submit image
+    arg=$(set_arg)
     session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg"`
-    echo $session >> session_IDs.txt
-    echo "$session $ID" >> session_image_IDs.txt
+    update_session_logs
 
   fi
 

From 842e3824e4ac6dc4ccaf5cac9f26f47dd73f46e0 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 26 Jan 2024 10:57:56 +0000
Subject: [PATCH 102/133] minor modifs to summary

---
 scripts/jupyter/summary_run.ipynb       | 235 +++++++-----------------
 scripts/python/summary_params_pre_v2.py |   6 +-
 shapepipe/utilities/summary.py          |   7 +-
 3 files changed, 77 insertions(+), 171 deletions(-)

diff --git a/scripts/jupyter/summary_run.ipynb b/scripts/jupyter/summary_run.ipynb
index e57c72b1a..8c42e3865 100644
--- a/scripts/jupyter/summary_run.ipynb
+++ b/scripts/jupyter/summary_run.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 20,
    "id": "130112a4-f2ca-4d26-b884-d8b054676f9c",
    "metadata": {
     "tags": []
@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 21,
    "id": "196beca5-10a1-4cf5-9462-be145167cc70",
    "metadata": {
     "tags": []
@@ -37,7 +37,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 22,
    "id": "ca63c72d-212c-463e-a792-71efbac0b908",
    "metadata": {
     "tags": []
@@ -45,14 +45,14 @@
    "outputs": [],
    "source": [
     "# Setting\n",
-    "patch = \"P1\"\n",
+    "patch = \"P3\"\n",
     "\n",
-    "verbose = True"
+    "verbose = False"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 23,
    "id": "dcb5604c-d61f-4705-8295-63875455cadb",
    "metadata": {
     "tags": []
@@ -65,7 +65,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 24,
    "id": "e69b7dab-1fea-4fcc-a8d9-0720e1d628c3",
    "metadata": {
     "tags": []
@@ -75,14 +75,14 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Checking main directory = /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P1\n"
+      "Checking main directory = /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P3\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Set job info for patch P1\n"
+      "Set job info for patch P3\n"
      ]
     }
    ],
@@ -92,7 +92,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 25,
    "id": "2c3a9dde-cf88-493f-926e-7ae7e8e10916",
    "metadata": {
     "tags": []
@@ -105,34 +105,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 26,
    "id": "1e9c1487-3cec-4394-9fcf-c12e92a0f984",
    "metadata": {
     "tags": []
    },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "===========\n",
-      "par_runtime\n",
-      "-----------\n",
-      "n_tile_IDs                       3527\n",
-      "list_tile_IDs                   3527 entries\n",
-      "===========\n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "print_par_runtime(par_runtime, verbose=verbose)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 27,
    "id": "b7c63a22-ead1-4d6a-b081-a74ade515439",
    "metadata": {
     "tags": []
@@ -154,7 +139,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 28,
    "id": "4720ae18-0633-4646-b392-b1b24e0294c3",
    "metadata": {
     "tags": []
@@ -165,58 +150,27 @@
      "output_type": "stream",
      "text": [
       " (Job 1)\n",
-      "get_images_runner_run_1             7054      4402         0      2652    1326.0     62.4%\n",
-      "find_exposures_runner               3527       875         0      2652    2652.0     24.8%\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "** module get_images_runner_run_1\n",
-      "*** subdirs ['']\n",
-      "**** base_and_subdir /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P1/output//.\n",
-      "Matching entries:  [<DirEntry 'run_sp_Git'>, <DirEntry 'run_sp_GitFeGie_2024-01-16_22-24-51'>]\n",
-      "** module find_exposures_runner\n",
-      "*** subdirs ['']\n",
-      "**** base_and_subdir /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P1/output//.\n",
-      "Matching entries:  [<DirEntry 'run_sp_Git'>, <DirEntry 'run_sp_GitFeGie_2024-01-16_22-24-51'>]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "get_images_runner_run_2             7119      1509         0      5610    1870.0     21.2%\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "** module get_images_runner_run_2\n",
-      "*** subdirs ['']\n",
-      "**** base_and_subdir /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P1/output//.\n",
-      "Matching entries:  [<DirEntry 'run_sp_Git'>, <DirEntry 'run_sp_GitFeGie_2024-01-16_22-24-51'>]\n"
+      "get_images_runner_run_1             2536      2536         0         0       0.0    100.0%\n",
+      "find_exposures_runner               1268      1268         0         0       0.0    100.0%\n",
+      "get_images_runner_run_2             3420      3420         0         0       0.0    100.0%\n"
      ]
     }
    ],
    "source": [
     "for key in \"1\":\n",
-    "    job = jobs[key]\n",
-    "    job.print_intro()\n",
-    "    job.check_numbers(par_runtime=par_runtime, indices=[0, 1])\n",
+    "    jobs[key].print_intro()\n",
+    "    jobs[key].check_numbers(par_runtime=par_runtime, indices=[0, 1])\n",
     "\n",
-    "    all_exposures = get_all_exposures(job._paths_in_dir[1], verbose=verbose)\n",
+    "    all_exposures = get_all_exposures(jobs[key]._paths_in_dir[1], verbose=verbose)\n",
     "    par_runtime[\"n_exposures\"] = len(all_exposures)\n",
     "    par_runtime[\"list_exposures\"] = all_exposures\n",
     "\n",
-    "    job.check_numbers(par_runtime, indices=[2])"
+    "    jobs[key].check_numbers(par_runtime, indices=[2])"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 29,
    "id": "f149f404-64e7-4d92-8f54-f300ed620130",
    "metadata": {
     "tags": []
@@ -229,39 +183,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 30,
    "id": "607fed1b-005a-4d3a-811c-51b1c5049c19",
    "metadata": {
     "tags": []
    },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "===========\n",
-      "par_runtime\n",
-      "-----------\n",
-      "n_tile_IDs                       3527\n",
-      "list_tile_IDs                   3527 entries\n",
-      "n_exposures                      2373\n",
-      "list_exposures                  2373 entries\n",
-      "n_shdus                         94920\n",
-      "list_shdus                     94920 entries\n",
-      "n_3*n_shdus+n_exposures        287133\n",
-      "===========\n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "print_par_runtime(par_runtime, verbose=verbose)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 31,
    "id": "78a9065f-8983-41cf-a34c-21892fc52dd2",
    "metadata": {
     "tags": []
@@ -275,7 +209,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 32,
    "id": "79e39954-1155-4ca3-b0b2-64bc5670db53",
    "metadata": {
     "tags": []
@@ -293,34 +227,45 @@
       "mask_runner                         1268      1268         0         0       0.0    100.0%\n",
       " (Job 8)\n",
       "mask_runner                        45600     45600         0         0       0.0    100.0%\n",
-      " (Job 16)\n"
-     ]
-    },
-    {
-     "ename": "KeyboardInterrupt",
-     "evalue": "",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
-      "\u001b[0;32m/tmp/ipykernel_48/963628441.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjobs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_intro\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_numbers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpar_runtime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpar_runtime\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
-      "\u001b[0;32m~/.local/lib/python3.7/site-packages/shapepipe/utilities/summary.py\u001b[0m in \u001b[0;36mcheck_numbers\u001b[0;34m(self, par_runtime, indices)\u001b[0m\n\u001b[1;32m    603\u001b[0m                 \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m                 \u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 605\u001b[0;31m                 \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    606\u001b[0m             )\n\u001b[1;32m    607\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m~/.local/lib/python3.7/site-packages/shapepipe/utilities/summary.py\u001b[0m in \u001b[0;36mget_names_in_dir\u001b[0;34m(self, iterable, module, idx)\u001b[0m\n\u001b[1;32m    515\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"**** base_and_subdir {base_and_subdir}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 517\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbase_and_subdir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m                 \u001b[0mmatches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;32m/opt/conda/lib/python3.7/genericpath.py\u001b[0m in \u001b[0;36misdir\u001b[0;34m(s)\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;34m\"\"\"Return true if the pathname refers to an existing directory.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m         \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
-      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+      " (Job 16)\n",
+      "sextractor_runner                   2536      2536         0         0       0.0    100.0%\n",
+      " (Job 32)\n",
+      "sextractor_runner                  91200     91200         0         0       0.0    100.0%\n",
+      "setools_runner                     91200     91032         0       168      84.0     99.8%\n",
+      "psfex_runner                       91200     91032         0       168      84.0     99.8%\n",
+      " (Job 33)\n",
+      "psfex_interp_runner                45600     41132         0      4468    4468.0     90.2%\n",
+      " (Job 64)\n",
+      "psfex_interp_runner                 1268      1268         0         0       0.0    100.0%\n",
+      "vignetmaker_runner_run_1            1268      1268         0         0       0.0    100.0%\n",
+      "spread_model_runner                 1268      1268         0         0       0.0    100.0%\n",
+      "vignetmaker_runner_run_2            5072      5072         0         0       0.0    100.0%\n",
+      " (Job 128)\n",
+      "ngmix_runner                        1268       189         0      1079    1079.0     14.9%\n",
+      "ngmix_runner                        1268       202         0      1066    1066.0     15.9%\n",
+      "ngmix_runner                        1268       203         0      1065    1065.0     16.0%\n",
+      "ngmix_runner                        1268       183         0      1085    1085.0     14.4%\n",
+      "ngmix_runner                        1268       204         0      1064    1064.0     16.1%\n",
+      "ngmix_runner                        1268       185         0      1083    1083.0     14.6%\n",
+      "ngmix_runner                        1268       199         0      1069    1069.0     15.7%\n",
+      "ngmix_runner                        1268       196         0      1072    1072.0     15.5%\n",
+      " (Job 256)\n",
+      "merge_sep_cats_runner               1268         0         0      1268    1268.0      0.0%\n",
+      "make_cat_runner                     1268         0         0      1268    1268.0      0.0%\n",
+      " (Job 1024)\n",
+      "psfex_interp_runner                45600     41132         0      4468    4468.0     90.2%\n"
      ]
     }
    ],
    "source": [
     "for key in keys:\n",
-    "    job = jobs[key]\n",
-    "    job.print_intro()\n",
-    "    job.check_numbers(par_runtime=par_runtime)"
+    "    jobs[key].print_intro()\n",
+    "    jobs[key].check_numbers(par_runtime=par_runtime)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": null,
    "id": "8d2243cb-5aa6-4e82-88f6-8689f15e5b61",
    "metadata": {},
    "outputs": [
@@ -328,72 +273,28 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      " (Job 64)\n",
-      "psfex_interp_runner                 1268       945         0       323     323.0     74.5%\n",
-      "vignetmaker_runner_run_1            1268       971         0       297     297.0     76.6%\n",
-      "spread_model_runner                 1268       173         0      1095    1095.0     13.6%\n",
-      "vignetmaker_runner_run_2            5072       690         0      4382    1095.5     13.6%\n"
+      " (Job 128)\n",
+      "ngmix_runner                        1268       413         0       855     855.0     32.6%\n",
+      "ngmix_runner                        1268       417         0       851     851.0     32.9%\n",
+      "ngmix_runner                        1268       425         0       843     843.0     33.5%\n",
+      "ngmix_runner                        1268       419         0       849     849.0     33.0%\n",
+      "ngmix_runner                        1268       436         0       832     832.0     34.4%\n",
+      "ngmix_runner                        1268       427         0       841     841.0     33.7%\n",
+      "ngmix_runner                        1268       422         0       846     846.0     33.3%\n"
      ]
     }
    ],
    "source": [
     "# Update some runs\n",
-    "for key in [\"64\"]:\n",
-    "    job = jobs[key]\n",
-    "    job.print_intro()\n",
-    "    job.check_numbers(par_runtime=par_runtime)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 35,
-   "id": "e8a38747-21b7-4f74-b667-62094f17154e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      " (Job 64)\n",
-      "psfex_interp_runner                 1268       965         0       303     303.0     76.1%\n",
-      "vignetmaker_runner_run_1            1268       993         0       275     275.0     78.3%\n",
-      "spread_model_runner                 1268       187         0      1081    1081.0     14.7%\n",
-      "vignetmaker_runner_run_2            5072       749         0      4323    1080.8     14.8%\n"
-     ]
-    }
-   ],
-   "source": [
-    "for key in [\"64\"]:\n",
-    "    job = jobs[key]\n",
-    "    job.print_intro()\n",
-    "    job.check_numbers(par_runtime=par_runtime)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "82cb7e0b-a8fa-4a55-840f-55ec0452bf7f",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      " (Job 64)\n"
-     ]
-    }
-   ],
-   "source": [
-    "for key in [\"64\"]:\n",
-    "    job = jobs[key]\n",
-    "    job.print_intro()\n",
-    "    job.check_numbers(par_runtime=par_runtime)"
+    "for key in [\"128\"]:\n",
+    "    jobs[key].print_intro()\n",
+    "    jobs[key].check_numbers(par_runtime=par_runtime)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "c57c576f-b7e8-4735-a047-5b63e245f9a4",
+   "id": "67b50a61-e3cc-4559-941d-f39c6a200294",
    "metadata": {},
    "outputs": [],
    "source": []
diff --git a/scripts/python/summary_params_pre_v2.py b/scripts/python/summary_params_pre_v2.py
index 31b63cb3c..204e9a157 100644
--- a/scripts/python/summary_params_pre_v2.py
+++ b/scripts/python/summary_params_pre_v2.py
@@ -141,12 +141,14 @@ def set_jobs_v2_pre_v2(patch, verbose):
             "run_sp_exp_SxSePsf",
             "run_sp_exp_SxSePsf",
             "run_sp_exp_SxSePsf",
-        ],  # "run_sp_exp_Pi"],
+            #"run_sp_exp_Pi"
+        ],
         [
             "sextractor_runner",
             "setools_runner",
             "psfex_runner",
-        ],  # "psfex_interp_runner"],
+            # "psfex_interp_runner"],
+        ],
         "shdus",
         n_mult=[2, 2, 2],  # 1],
         path_main=path_main,
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index 001e975f6..cdffc1f23 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -464,7 +464,7 @@ def get_module_output_dir(self, full_path, module):
 
     def get_matches_final(self, directory, idx):
 
-        # Look over files
+        # Loop over files
         # os.path.whether exists is twice faster than try/except
 
         if os.path.exists(directory):
@@ -475,6 +475,7 @@ def get_matches_final(self, directory, idx):
                     and (
                         fnmatch.fnmatch(entry2.name, pattern)
                     )
+                    and entry2.stat().st_size > 0
                 ):
                     # Append matching files
                     self._names_in_dir[idx].append(entry2.name)
@@ -530,7 +531,9 @@ def get_names_in_dir(self, iterable, module, idx):
                         continue
 
                     if self._verbose:
-                        print("Matching entries: ", matches)
+                        print("**** Matching entries: ", end="")
+                        for match in matches:
+                            print(match.name)
 
                     full_path = self.get_last_full_path(
                         base_and_subdir, matches

From d9846e91bdb2bad93a9bdecb7478bc86c2d52eb5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 26 Jan 2024 10:59:03 +0000
Subject: [PATCH 103/133] Removed VM_HOME; jon sp canfar cleaned up

---
 example/cfis/config_Gie_vos.ini       |   4 +-
 example/cfis/config_GitFeGie_vos.ini  |   4 +-
 example/cfis/config_GitFe_symlink.ini | 105 ++++++++++++++++++++++++++
 example/cfis/config_Git_vos.ini       |  93 +++++++++++++++++++++++
 scripts/sh/job_sp_canfar.bash         |  83 +++++++++++++++-----
 5 files changed, 269 insertions(+), 20 deletions(-)
 create mode 100644 example/cfis/config_GitFe_symlink.ini
 create mode 100644 example/cfis/config_Git_vos.ini

diff --git a/example/cfis/config_Gie_vos.ini b/example/cfis/config_Gie_vos.ini
index a326434e7..ee4b92f30 100644
--- a/example/cfis/config_Gie_vos.ini
+++ b/example/cfis/config_Gie_vos.ini
@@ -98,4 +98,6 @@ RETRIEVE = vos
 N_TRY = 3
 
 # Retrieve command options, optional
-RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem
+RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem
+
+CHECK_EXISTING_DIR = $SP_RUN/data_exp
diff --git a/example/cfis/config_GitFeGie_vos.ini b/example/cfis/config_GitFeGie_vos.ini
index 9ef72f198..75044bf44 100644
--- a/example/cfis/config_GitFeGie_vos.ini
+++ b/example/cfis/config_GitFeGie_vos.ini
@@ -84,7 +84,7 @@ OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight-
 RETRIEVE = vos
 
 # Copy command options, optional
-RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem
+RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem
 
 CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Git/get_images_runner_run_1/output
 
@@ -152,4 +152,4 @@ RETRIEVE = vos
 N_TRY = 3
 
 # Retrieve command options, optional
-RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem
+RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem
diff --git a/example/cfis/config_GitFe_symlink.ini b/example/cfis/config_GitFe_symlink.ini
new file mode 100644
index 000000000..4e7ce75cb
--- /dev/null
+++ b/example/cfis/config_GitFe_symlink.ini
@@ -0,0 +1,105 @@
+# ShapePipe configuration file for: get images and find exposures
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = False
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_GitFe
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = get_images_runner, find_exposures_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+# Get tiles
+[GET_IMAGES_RUNNER]
+
+FILE_PATTERN = tile_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = 
+
+# Paths
+
+# Input path where original images are stored. Can be local path or vos url.
+# Single string or list of strings
+INPUT_PATH = $SP_RUN/data_tiles, $SP_RUN/data_tiles
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight
+
+# Input file extensions
+INPUT_FILE_EXT = .fits, .fits.fz
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{3}\.\d{3}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight-
+
+# Copy/download method, one in 'vos', 'symlink'
+RETRIEVE = symlink
+
+# Copy command options, optional
+RETRIEVE_OPTIONS = -L
+
+
+[FIND_EXPOSURES_RUNNER]
+
+INPUT_MODULE = get_images_runner
+
+FILE_PATTERN = CFIS_image
+
+FILE_EXT = .fits
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = -000-000
+
+# Column number of exposure name in FITS header
+COLNUM = 3
+
+# Prefix to remove from exposure name
+EXP_PREFIX = p
diff --git a/example/cfis/config_Git_vos.ini b/example/cfis/config_Git_vos.ini
new file mode 100644
index 000000000..26edf3efc
--- /dev/null
+++ b/example/cfis/config_Git_vos.ini
@@ -0,0 +1,93 @@
+# ShapePipe configuration file for: get images
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = False
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_Git
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = True
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = get_images_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 1
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+
+# Get tiles
+[GET_IMAGES_RUNNER]
+
+FILE_PATTERN = tile_numbers
+
+FILE_EXT = .txt
+
+# NUMBERING_SCHEME (optional) string with numbering pattern for input files
+NUMBERING_SCHEME = 
+
+# Paths
+
+# Input path where original images are stored. Can be local path or vos url.
+# Single string or list of strings
+INPUT_PATH = vos:cfis/tiles_DR5, vos:cfis/tiles_DR5
+
+# Input file pattern including tile number as dummy template
+INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight
+
+# Input file extensions
+INPUT_FILE_EXT = .fits, .fits.fz
+
+# Input numbering scheme, python regexp
+INPUT_NUMBERING = \d{3}\.\d{3}
+
+# Output file pattern without number
+OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight-
+
+# Copy/download method, one in 'vos', 'symlink'
+RETRIEVE = vos
+
+# If RETRIEVE=vos, number of attempts to download
+# Optional, default=3
+N_TRY = 3
+
+# Copy command options, optional
+RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem
+
+CHECK_EXISTING_DIR = $SP_RUN/data_tiles
diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
index 91f67619b..f2b9c03d1 100755
--- a/scripts/sh/job_sp_canfar.bash
+++ b/scripts/sh/job_sp_canfar.bash
@@ -160,9 +160,6 @@ export SP_RUN=`pwd`
 export SP_CONFIG=$SP_RUN/cfis
 export SP_CONFIG_MOD=$SP_RUN/cfis_mod
 
-# To find ssl certificate
-export VM_HOME=$HOME
-
 ## Other variables
 
 # Output
@@ -341,17 +338,31 @@ fi
 if [[ $do_job != 0 ]]; then
 
   ### Retrieve files
-  command_sp "shapepipe_run -c $SP_CONFIG/config_GitFeGie_$retrieve.ini" "Retrieve images"
+  command_cfg_shapepipe \
+    "config_GitFeGie_$retrieve.ini" \
+     "Retrieve images" \
+     -1 \
+     $exclusive
+
+  #if [[ ! -d "data_tiles" ]]; then
+    #echo "Directory or link 'data_tiles' does not exist, exiting"
+    #exit 1
+  #fi
+  #command_cfg_shapepipe "config_Git_vos.ini" "Retrieve tiles" -1 $n_exclusive
 
   ### Retrieve and save star catalogues for masking
   if [ "$star_cat_for_mask" == "save" ]; then
     #### For tiles
     mkdir $SP_RUN/star_cat_tiles
-    command_sp "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_1/output $SP_RUN/star_cat_tiles" "Save star cats for masking (tile)"
+    command_sp \
+      "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_1/output $SP_RUN/star_cat_tiles" \
+      "Save star cats for masking (tile)"
 
     #### For single-exposures
     mkdir $SP_RUN/star_cat_exp
-    command_sp "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_2/output $SP_RUN/star_cat_exp exp" "Save star cats for masking (exp)"
+    command_sp \
+      "create_star_cat $SP_RUN/output/run_sp_GitFeGie_*/get_images_runner_run_2/output $SP_RUN/star_cat_exp exp" \
+      "Save star cats for masking (exp)"
   fi
 
 fi
@@ -364,7 +375,11 @@ if [[ $do_job != 0 ]]; then
   command_cfg_shapepipe "config_tile_Uz.ini" "Run shapepipe (uncompress tile weights)" $n_smp $exclusive
 
   ### Split images into single-HDU files, merge headers for WCS info
-  command_cfg_shapepipe "config_exp_SpMh.ini" "Run shapepipe (split images, merge headers)" $n_smp $exclusive
+  command_cfg_shapepipe \
+    "config_exp_SpMh.ini" \
+    "Run shapepipe (split images, merge headers)" \
+    $n_smp \
+    $exclusive
 
 fi
 
@@ -373,7 +388,11 @@ fi
 if [[ $do_job != 0 ]]; then
 
   ### Mask tiles
-  command_cfg_shapepipe "config_tile_Ma_$star_cat_for_mask.ini" "Run shapepipe (mask tiles)" $n_smp $exclusive
+  command_cfg_shapepipe \
+    "config_tile_Ma_$star_cat_for_mask.ini" \
+    "Run shapepipe (mask tiles)" \
+    $n_smp \
+    $exclusive
 
 fi
 
@@ -382,7 +401,11 @@ fi
 if [[ $do_job != 0 ]]; then
 
   ### Mask exposures
-  command_cfg_shapepipe "config_exp_Ma_$star_cat_for_mask.ini" "Run shapepipe (mask exposures)" $n_smp $exclusive
+  command_cfg_shapepipe \
+    "config_exp_Ma_$star_cat_for_mask.ini" \
+    "Run shapepipe (mask exposures)" \
+    $n_smp \
+    $exclusive
 
 fi
 
@@ -392,7 +415,11 @@ fi
 if [[ $do_job != 0 ]]; then
 
   ### Object detection on tiles
-  command_cfg_shapepipe "config_tile_Sx.ini" "Run shapepipe (tile detection)" $n_smp $exclusive
+  command_cfg_shapepipe \
+    "config_tile_Sx.ini" \
+    "Run shapepipe (tile detection)" \
+    $n_smp \
+    $exclusive
 
 fi
 
@@ -403,7 +430,11 @@ if [[ $do_job != 0 ]]; then
   ### Star detection, selection, PSF model. setools can exit with an error for CCD with insufficient stars,
   ### the script should continue
   STOP=0
-  command_cfg_shapepipe "config_exp_${psf}.ini" "Run shapepipe (exp $psf)" $n_smp $exclusive
+  command_cfg_shapepipe \
+    "config_exp_${psf}.ini" \
+    "Run shapepipe (exp $psf)" \
+    $n_smp \
+    $exclusive
   STOP=1
 
 fi
@@ -415,7 +446,11 @@ if [[ $do_job != 0 ]]; then
   ### PSF model letter: 'P' (psfex) or 'M' (mccd)
   letter=${psf:0:1}
   Letter=${letter^}
-  command_cfg_shapepipe "config_tile_${Letter}iViSmVi_canfar.ini" "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)" $n_smp $exclusive
+  command_cfg_shapepipe \
+    "config_tile_${Letter}iViSmVi_canfar.ini" \
+    "Run shapepipe (tile PsfInterp=$Letter}: up to ngmix+galsim)" \
+    $n_smp \
+    $exclusive
 
 fi
 
@@ -454,13 +489,19 @@ if [[ $do_job != 0 ]]; then
         ngmix_out="$ngmix_run/output"
         n_out=`ls -rlt $ngmix_out | wc -l`
         if [ "$n_out" -lt 2 ]; then
-          command "rm -rf $OUTPUT/run_sp_tile_ngmix_Ng${k}u" "Re-running existing empty ngmix subrun $k"
-          command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" "Run shapepipe (tile: ngmix $k)" &
+          command \
+            "rm -rf $OUTPUT/run_sp_tile_ngmix_Ng${k}u" \
+            "Re-running existing empty ngmix subrun $k"
+          command_sp \
+            "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" \
+            "Run shapepipe (tile: ngmix $k)" &
         else
           echo "Skipping existing non-empty ngmix subrun $k"
         fi
       else
-        command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" "Run shapepipe (tile: ngmix $k)" &
+        command_sp \
+          "shapepipe_run -c $SP_CONFIG_MOD/config_tile_Ng${k}u.ini" \
+          "Run shapepipe (tile: ngmix $k)" &
       fi
   done
   wait
@@ -478,9 +519,17 @@ if [[ $do_job != 0 ]]; then
       > $SP_CONFIG_MOD/config_merge_sep_cats.ini
  
   ### Merge separated shapes catalogues
-  command_sp "shapepipe_run -c $SP_CONFIG_MOD/config_merge_sep_cats.ini" "Run shapepipe (tile: merge sep cats)" "$VERBOSE" "$ID"
+  command_sp \
+    "shapepipe_run -c $SP_CONFIG_MOD/config_merge_sep_cats.ini" \
+    "Run shapepipe (tile: merge sep cats)" \
+    "$VERBOSE" \
+    "$ID"
 
   ### Merge all relevant information into final catalogue
-  command_sp "shapepipe_run -c $SP_CONFIG/config_make_cat_$psf.ini" "Run shapepipe (tile: create final cat $psf)" "$VERBOSE" "$ID"
+  command_sp \
+    "shapepipe_run -c $SP_CONFIG/config_make_cat_$psf.ini" \
+    "Run shapepipe (tile: create final cat $psf)" \
+    "$VERBOSE" \
+    "$ID"
 
 fi

From c5deb345df7654734f269328442f613e1a94c3d5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 30 Jan 2024 10:38:29 +0100
Subject: [PATCH 104/133] Dockerfile cleand up

---
 Dockerfile | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4cfd891d8..afde6114c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -28,10 +28,10 @@ RUN apt-get update --allow-releaseinfo-change && \
     apt-get install vim -y && \
     apt-get install locate -y && \
     apt-get install curl -y && \
+    apt-get install acl -y && \
+    apt-get install sssd -y && \
     apt-get clean
 
-RUN apt-get install acl -y && \
-    apt-get install sssd -y
 ADD nsswitch.conf /etc/
 
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
@@ -52,9 +52,6 @@ RUN conda env create --file environment.yml
 COPY shapepipe ./shapepipe
 COPY scripts ./scripts
 
-# Activate conda environment using "source"
-#RUN . /opt/conda/etc/profile.d/conda.sh
-#RUN conda init bash
-RUN ./scripts/sh/init_canfar.sh
-RUN source activate shapepipe
+# Make RUN commands use the new environment:
+SHELL ["conda", "run", "-n", "shapepipe", "/bin/bash", "-c"]
 RUN pip install jupyter

From d507780efc0868531576178426092c3b0e954e21 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 1 Feb 2024 15:25:54 +0000
Subject: [PATCH 105/133] summary params minor bug fixed

---
 scripts/python/summary_params_pre_v2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/python/summary_params_pre_v2.py b/scripts/python/summary_params_pre_v2.py
index 204e9a157..953f6190a 100644
--- a/scripts/python/summary_params_pre_v2.py
+++ b/scripts/python/summary_params_pre_v2.py
@@ -88,7 +88,7 @@ def set_jobs_v2_pre_v2(patch, verbose):
     #        n_mult=[1, 1, 1],
     jobs["2"] = job_data(
         2,
-        ["run_sp_Uz", "run_sp_exp_SpMh", "run_sp_exp_SpMh_2023-12"],
+        ["run_sp_Uz", "run_sp_exp_SpMh", "run_sp_exp_SpMh"],
         ["uncompress_fits_runner", "merge_headers_runner", "split_exp_runner"],
         ["tile_IDs", 0, "3*n_shdus+n_exposures"],
         path_main=path_main,

From b4e0aa04aa30a7c0b45791dcb9f017f490faf5a1 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 1 Feb 2024 15:26:46 +0000
Subject: [PATCH 106/133] major bugx fixed: N_SMP was n_SMP, not propagated

---
 scripts/sh/curl_canfar_local.sh | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index b02f9205f..3ecacc3cd 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -17,6 +17,7 @@ N_SMP=1
 kind=-1
 version=1.0
 cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
+batch_max=200
 dry_run=0
 
 # TODO psf
@@ -40,6 +41,8 @@ usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS]
     \tversion of docker image, default='$version'\n
    -C, --command_remote\n
     \tremote command to run on canfar, default='$cmd_remote'\n
+   -b, --batch_max\n
+    \tmaximum batch size = number of jobs run simultaneously\n
    -n, --dry_run LEVEL\n
     \tdry run, from LEVEL=2 (no processing) to 0 (full run)\n
 "
@@ -70,13 +73,17 @@ while [ $# -gt 0 ]; do
       shift
       ;;
     -N|--N_SMP)
-      n_SMP="$2"
+      N_SMP="$2"
       shift
       ;;
     -k|--kind)
       kind="$2"
       shift
       ;;
+    -b|--batch_max)
+      batch_max="$2"
+      shift
+      ;;
     -n|--dry_run)
       dry_run="$2"
       shift
@@ -151,11 +158,10 @@ function submit_batch() {
 
 }
 
-n_max=200
 batch=20
-sleep=300
+sleep=600
 
-((n_thresh=n_max- batch))
+((n_thresh=batch_max-batch))
 
 
 if [ "$dry_run" == 2 ]; then
@@ -188,7 +194,7 @@ else
 
     # Submit file
     n_jobs=`cat $file_IDs | wc -l`
-    if [ "$n_jobs" -gt "$n_max" ]; then
+    if [ "$n_jobs" -gt "$batch_max" ]; then
 
       # Split into batches 
       prefix="${file_IDs}_split_"

From f8a07d7f4c5fee351c3cafe715ed852a25c34b07 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 1 Feb 2024 15:27:36 +0000
Subject: [PATCH 107/133] major bugx fixed: N_SMP was n_SMP, not propagated

---
 scripts/sh/init_run_exclusive_canfar.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 05ca2a848..65ecfe8c1 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -57,7 +57,7 @@ while [ $# -gt 0 ]; do
       shift                                                                     
       ;;
     -N|--N_SMP)                                                                 
-      n_SMP="$2"                                                                
+      N_SMP="$2"                                                                
       shift                                                                     
       ;;                                                                        
     -k|--kind)                                                                 
@@ -177,7 +177,6 @@ done
 if [[ $do_job != 0 ]]; then
   if [ "$kind" == "tile" ]; then
     cd ../../..
-    #command "rm -rf tile_runs/$ID/output/run_exp_SxSePsf*" $dry_run
     command "link_to_exp_for_tile.py -t $ID -i tile_runs -I exp_runs" $dry_run
     cd ${kind}_runs/$ID/output
 
@@ -188,6 +187,9 @@ if [[ $do_job != 0 ]]; then
       echo "removing $n_remove duplicate old job-32 runs"
       rm -rf `ls -rt1d run_sp_tile_Sx_* | head -$n_remove`
     fi
+
+    # Remove previous runs of this job
+    rm -rf run_sp_tile_PsViSmVi*
   fi
 fi
 

From 30116b844f5e71859a850f2a7e2f26288a1e0b40 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Thu, 1 Feb 2024 15:29:26 +0000
Subject: [PATCH 108/133] summary nb

---
 scripts/jupyter/summary_run.ipynb | 80 ++++++++++++++-----------------
 1 file changed, 37 insertions(+), 43 deletions(-)

diff --git a/scripts/jupyter/summary_run.ipynb b/scripts/jupyter/summary_run.ipynb
index 8c42e3865..6b4007172 100644
--- a/scripts/jupyter/summary_run.ipynb
+++ b/scripts/jupyter/summary_run.ipynb
@@ -2,21 +2,12 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 1,
    "id": "130112a4-f2ca-4d26-b884-d8b054676f9c",
    "metadata": {
     "tags": []
    },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The autoreload extension is already loaded. To reload it, use:\n",
-      "  %reload_ext autoreload\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "%load_ext autoreload\n",
     "%autoreload 2"
@@ -24,7 +15,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 2,
    "id": "196beca5-10a1-4cf5-9462-be145167cc70",
    "metadata": {
     "tags": []
@@ -37,7 +28,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 3,
    "id": "ca63c72d-212c-463e-a792-71efbac0b908",
    "metadata": {
     "tags": []
@@ -52,7 +43,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 4,
    "id": "dcb5604c-d61f-4705-8295-63875455cadb",
    "metadata": {
     "tags": []
@@ -65,7 +56,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 5,
    "id": "e69b7dab-1fea-4fcc-a8d9-0720e1d628c3",
    "metadata": {
     "tags": []
@@ -92,7 +83,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 6,
    "id": "2c3a9dde-cf88-493f-926e-7ae7e8e10916",
    "metadata": {
     "tags": []
@@ -105,7 +96,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 7,
    "id": "1e9c1487-3cec-4394-9fcf-c12e92a0f984",
    "metadata": {
     "tags": []
@@ -117,7 +108,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 8,
    "id": "b7c63a22-ead1-4d6a-b081-a74ade515439",
    "metadata": {
     "tags": []
@@ -139,7 +130,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 9,
    "id": "4720ae18-0633-4646-b392-b1b24e0294c3",
    "metadata": {
     "tags": []
@@ -152,7 +143,7 @@
       " (Job 1)\n",
       "get_images_runner_run_1             2536      2536         0         0       0.0    100.0%\n",
       "find_exposures_runner               1268      1268         0         0       0.0    100.0%\n",
-      "get_images_runner_run_2             3420      3420         0         0       0.0    100.0%\n"
+      "get_images_runner_run_2             3420         0         0      3420    1140.0      0.0%\n"
      ]
     }
    ],
@@ -170,7 +161,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 10,
    "id": "f149f404-64e7-4d92-8f54-f300ed620130",
    "metadata": {
     "tags": []
@@ -183,7 +174,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 11,
    "id": "607fed1b-005a-4d3a-811c-51b1c5049c19",
    "metadata": {
     "tags": []
@@ -195,7 +186,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 12,
    "id": "78a9065f-8983-41cf-a34c-21892fc52dd2",
    "metadata": {
     "tags": []
@@ -209,7 +200,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 13,
    "id": "79e39954-1155-4ca3-b0b2-64bc5670db53",
    "metadata": {
     "tags": []
@@ -241,14 +232,14 @@
       "spread_model_runner                 1268      1268         0         0       0.0    100.0%\n",
       "vignetmaker_runner_run_2            5072      5072         0         0       0.0    100.0%\n",
       " (Job 128)\n",
-      "ngmix_runner                        1268       189         0      1079    1079.0     14.9%\n",
-      "ngmix_runner                        1268       202         0      1066    1066.0     15.9%\n",
-      "ngmix_runner                        1268       203         0      1065    1065.0     16.0%\n",
-      "ngmix_runner                        1268       183         0      1085    1085.0     14.4%\n",
-      "ngmix_runner                        1268       204         0      1064    1064.0     16.1%\n",
-      "ngmix_runner                        1268       185         0      1083    1083.0     14.6%\n",
-      "ngmix_runner                        1268       199         0      1069    1069.0     15.7%\n",
-      "ngmix_runner                        1268       196         0      1072    1072.0     15.5%\n",
+      "ngmix_runner                        1268       759         0       509     509.0     59.9%\n",
+      "ngmix_runner                        1268       745         0       523     523.0     58.8%\n",
+      "ngmix_runner                        1268       755         0       513     513.0     59.5%\n",
+      "ngmix_runner                        1268       744         0       524     524.0     58.7%\n",
+      "ngmix_runner                        1268       752         0       516     516.0     59.3%\n",
+      "ngmix_runner                        1268       750         0       518     518.0     59.1%\n",
+      "ngmix_runner                        1268       743         0       525     525.0     58.6%\n",
+      "ngmix_runner                        1268       758         0       510     510.0     59.8%\n",
       " (Job 256)\n",
       "merge_sep_cats_runner               1268         0         0      1268    1268.0      0.0%\n",
       "make_cat_runner                     1268         0         0      1268    1268.0      0.0%\n",
@@ -266,26 +257,21 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "8d2243cb-5aa6-4e82-88f6-8689f15e5b61",
-   "metadata": {},
+   "id": "b3d51a05-ecca-420b-b8b3-1fb2b1ec9fe3",
+   "metadata": {
+    "tags": []
+   },
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      " (Job 128)\n",
-      "ngmix_runner                        1268       413         0       855     855.0     32.6%\n",
-      "ngmix_runner                        1268       417         0       851     851.0     32.9%\n",
-      "ngmix_runner                        1268       425         0       843     843.0     33.5%\n",
-      "ngmix_runner                        1268       419         0       849     849.0     33.0%\n",
-      "ngmix_runner                        1268       436         0       832     832.0     34.4%\n",
-      "ngmix_runner                        1268       427         0       841     841.0     33.7%\n",
-      "ngmix_runner                        1268       422         0       846     846.0     33.3%\n"
+      " (Job 128)\n"
      ]
     }
    ],
    "source": [
-    "# Update some runs\n",
+    "## Update some runs\n",
     "for key in [\"128\"]:\n",
     "    jobs[key].print_intro()\n",
     "    jobs[key].check_numbers(par_runtime=par_runtime)"
@@ -298,6 +284,14 @@
    "metadata": {},
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4d08ba73-0b84-4660-8e19-2b78cbe80618",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

From 9e44fa6646f21563c19481bc13740f64bfb26133 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 11 Feb 2024 10:58:21 +0000
Subject: [PATCH 109/133] sumamry updated

---
 scripts/jupyter/summary_run.ipynb       | 141 ++++++++++++++++++------
 scripts/python/summary_params_pre_v2.py |  27 +++--
 scripts/python/summary_run.py           |  25 ++---
 shapepipe/utilities/summary.py          |  27 ++---
 4 files changed, 141 insertions(+), 79 deletions(-)

diff --git a/scripts/jupyter/summary_run.ipynb b/scripts/jupyter/summary_run.ipynb
index 6b4007172..451c591d0 100644
--- a/scripts/jupyter/summary_run.ipynb
+++ b/scripts/jupyter/summary_run.ipynb
@@ -10,7 +10,8 @@
    "outputs": [],
    "source": [
     "%load_ext autoreload\n",
-    "%autoreload 2"
+    "%autoreload 3\n",
+    "%reload_ext autoreload"
    ]
   },
   {
@@ -36,7 +37,7 @@
    "outputs": [],
    "source": [
     "# Setting\n",
-    "patch = \"P3\"\n",
+    "patch = \"P7\"\n",
     "\n",
     "verbose = False"
    ]
@@ -51,7 +52,7 @@
    "outputs": [],
    "source": [
     "# Load parameters\n",
-    "%run ~/shapepipe/scripts/python/summary_params_pre_v2 {patch}"
+    "%run ~/shapepipe/scripts/python/summary_params_pre_v2"
    ]
   },
   {
@@ -66,14 +67,14 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Checking main directory = /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P3\n"
+      "Checking main directory = /arc/home/kilbinger/cosmostat/v2/pre_v2/psfex/P7\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Set job info for patch P3\n"
+      "Set job info for patch P7\n"
      ]
     }
    ],
@@ -103,7 +104,8 @@
    },
    "outputs": [],
    "source": [
-    "print_par_runtime(par_runtime, verbose=verbose)"
+    "# No effect in notebook\n",
+    "#print_par_runtime(par_runtime, verbose=verbose)"
    ]
   },
   {
@@ -141,9 +143,9 @@
      "output_type": "stream",
      "text": [
       " (Job 1)\n",
-      "get_images_runner_run_1             2536      2536         0         0       0.0    100.0%\n",
-      "find_exposures_runner               1268      1268         0         0       0.0    100.0%\n",
-      "get_images_runner_run_2             3420         0         0      3420    1140.0      0.0%\n"
+      "get_images_runner_run_1              464       462         0         2       1.0     99.6%\n",
+      "find_exposures_runner                232       231         0         1       1.0     99.6%\n",
+      "get_images_runner_run_2              537         0         0       537     179.0      0.0%\n"
      ]
     }
    ],
@@ -172,18 +174,6 @@
     "par_runtime = update_par_runtime_after_find_exp(par_runtime, all_exposures)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "607fed1b-005a-4d3a-811c-51b1c5049c19",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "print_par_runtime(par_runtime, verbose=verbose)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 12,
@@ -224,22 +214,20 @@
       "sextractor_runner                  91200     91200         0         0       0.0    100.0%\n",
       "setools_runner                     91200     91032         0       168      84.0     99.8%\n",
       "psfex_runner                       91200     91032         0       168      84.0     99.8%\n",
-      " (Job 33)\n",
-      "psfex_interp_runner                45600     41132         0      4468    4468.0     90.2%\n",
       " (Job 64)\n",
       "psfex_interp_runner                 1268      1268         0         0       0.0    100.0%\n",
       "vignetmaker_runner_run_1            1268      1268         0         0       0.0    100.0%\n",
       "spread_model_runner                 1268      1268         0         0       0.0    100.0%\n",
       "vignetmaker_runner_run_2            5072      5072         0         0       0.0    100.0%\n",
       " (Job 128)\n",
-      "ngmix_runner                        1268       759         0       509     509.0     59.9%\n",
-      "ngmix_runner                        1268       745         0       523     523.0     58.8%\n",
-      "ngmix_runner                        1268       755         0       513     513.0     59.5%\n",
-      "ngmix_runner                        1268       744         0       524     524.0     58.7%\n",
-      "ngmix_runner                        1268       752         0       516     516.0     59.3%\n",
-      "ngmix_runner                        1268       750         0       518     518.0     59.1%\n",
-      "ngmix_runner                        1268       743         0       525     525.0     58.6%\n",
-      "ngmix_runner                        1268       758         0       510     510.0     59.8%\n",
+      "ngmix_runner                        1268      1225         0        43      43.0     96.6%\n",
+      "ngmix_runner                        1268      1216         0        52      52.0     95.9%\n",
+      "ngmix_runner                        1268      1216         0        52      52.0     95.9%\n",
+      "ngmix_runner                        1268      1217         0        51      51.0     96.0%\n",
+      "ngmix_runner                        1268      1228         0        40      40.0     96.8%\n",
+      "ngmix_runner                        1268      1216         0        52      52.0     95.9%\n",
+      "ngmix_runner                        1268      1216         0        52      52.0     95.9%\n",
+      "ngmix_runner                        1268      1216         0        52      52.0     95.9%\n",
       " (Job 256)\n",
       "merge_sep_cats_runner               1268         0         0      1268    1268.0      0.0%\n",
       "make_cat_runner                     1268         0         0      1268    1268.0      0.0%\n",
@@ -256,7 +244,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 65,
    "id": "b3d51a05-ecca-420b-b8b3-1fb2b1ec9fe3",
    "metadata": {
     "tags": []
@@ -266,7 +254,15 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      " (Job 128)\n"
+      " (Job 128)\n",
+      "ngmix_runner                        1268      1267         0         1       1.0     99.9%\n",
+      "ngmix_runner                        1268      1265         0         3       3.0     99.8%\n",
+      "ngmix_runner                        1268      1267         0         1       1.0     99.9%\n",
+      "ngmix_runner                        1268      1267         0         1       1.0     99.9%\n",
+      "ngmix_runner                        1268      1267         0         1       1.0     99.9%\n",
+      "ngmix_runner                        1268      1266         0         2       2.0     99.8%\n",
+      "ngmix_runner                        1268      1268         0         0       0.0    100.0%\n",
+      "ngmix_runner                        1268      1266         0         2       2.0     99.8%\n"
      ]
     }
    ],
@@ -279,16 +275,89 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 64,
    "id": "67b50a61-e3cc-4559-941d-f39c6a200294",
    "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " (Job 128)\n",
+      "ngmix_runner                        1268      1267         0         1       1.0     99.9%\n",
+      "ngmix_runner                        1268      1265         0         3       3.0     99.8%\n",
+      "ngmix_runner                        1268      1267         0         1       1.0     99.9%\n",
+      "ngmix_runner                        1268      1267         0         1       1.0     99.9%\n",
+      "ngmix_runner                        1268      1267         0         1       1.0     99.9%\n",
+      "ngmix_runner                        1268      1266         0         2       2.0     99.8%\n",
+      "ngmix_runner                        1268      1268         0         0       0.0    100.0%\n",
+      "ngmix_runner                        1268      1266         0         2       2.0     99.8%\n"
+     ]
+    }
+   ],
+   "source": [
+    "for key in [\"128\"]:\n",
+    "    jobs[key].print_intro()\n",
+    "    \n",
+    "    \n",
+    "    jobs[key].check_numbers(par_runtime=par_runtime)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "affa8293-daf9-4d2b-9215-fe19f8e2c1e2",
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
-   "source": []
+   "source": [
+    "session = Session()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "id": "269496d1-cd89-4d13-a5e4-41b897669e22",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "ids = [session[\"id\"] for session in session.fetch(kind=\"headless\")]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "80af8dff-98c7-4db4-8bcc-06936e1875cf",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "ename": "RuntimeError",
+     "evalue": "This event loop is already running",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
+      "\u001b[0;32m/tmp/ipykernel_69/559116804.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdestroy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/.local/lib/python3.7/site-packages/skaha/session.py\u001b[0m in \u001b[0;36mdestroy\u001b[0;34m(self, id)\u001b[0m\n\u001b[1;32m    264\u001b[0m             \u001b[0marguments\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m\"url\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserver\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m         \u001b[0mloop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_event_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscale\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marguments\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m         \u001b[0mresponses\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midentity\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/opt/conda/lib/python3.7/asyncio/base_events.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m    561\u001b[0m         \"\"\"\n\u001b[1;32m    562\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_closed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 563\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_runnung\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    564\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    565\u001b[0m         \u001b[0mnew_task\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mfutures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfuture\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfuture\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/opt/conda/lib/python3.7/asyncio/base_events.py\u001b[0m in \u001b[0;36m_check_runnung\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    521\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_check_runnung\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    522\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 523\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'This event loop is already running'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    524\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mevents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_running_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m             raise RuntimeError(\n",
+      "\u001b[0;31mRuntimeError\u001b[0m: This event loop is already running"
+     ]
+    }
+   ],
+   "source": [
+    "session.destroy(ids[0])"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "4d08ba73-0b84-4660-8e19-2b78cbe80618",
+   "id": "66a3ed14-8aaf-4028-b933-10ecb7376d68",
    "metadata": {},
    "outputs": [],
    "source": []
diff --git a/scripts/python/summary_params_pre_v2.py b/scripts/python/summary_params_pre_v2.py
index 953f6190a..5b67ee0ca 100644
--- a/scripts/python/summary_params_pre_v2.py
+++ b/scripts/python/summary_params_pre_v2.py
@@ -46,7 +46,10 @@ def set_jobs_v2_pre_v2(patch, verbose):
     if not os.path.isdir(path):
         os.mkdir(path)
     log_file_name = f"{path}/summary_log.txt"
-    handlers = [logging.FileHandler(log_file_name), logging.StreamHandler()]
+    handlers = [
+        logging.FileHandler(log_file_name, mode="w"),
+        logging.StreamHandler()
+    ]
     logging.basicConfig(
         level=logging.INFO, format="%(message)s", handlers=handlers
     )
@@ -159,17 +162,17 @@ def set_jobs_v2_pre_v2(patch, verbose):
     )
 
     # For P3
-    jobs["33"] = job_data(
-        33,
-        "run_sp_exp_Pi",
-        ["psfex_interp_runner"],
-        "shdus",
-        path_main=path_main,
-        path_left="exp_runs",
-        output_subdirs="shdus",
-        path_right="output",
-        verbose=verbose,
-    )
+    #jobs["33"] = job_data(
+    #    33,
+    #    "run_sp_exp_Pi",
+    #    ["psfex_interp_runner"],
+    #    "shdus",
+    #    path_main=path_main,
+    #    path_left="exp_runs",
+    #    output_subdirs="shdus",
+    #    path_right="output",
+    #    verbose=verbose,
+    #)
 
     jobs["64"] = job_data(
         "64",
diff --git a/scripts/python/summary_run.py b/scripts/python/summary_run.py
index b790300af..331764b24 100755
--- a/scripts/python/summary_run.py
+++ b/scripts/python/summary_run.py
@@ -5,47 +5,46 @@
 
 from shapepipe.utilities.summary import *
 
+from summary_params_pre_v2 import *
+
 
 def main(argv=None):
 
     patch = argv[1]
 
     verbose = False
-    
-    import summary_params_pre_v2 as params
 
-    jobs, list_tile_IDs_dot = params.set_jobs_v2_pre_v2(patch, verbose)
+    jobs, list_tile_IDs_dot = set_jobs_v2_pre_v2(patch, verbose)
     
     list_tile_IDs = job_data.replace_dot_dash(list_tile_IDs_dot)
     
     # Numbers updated at runtime
-    par_runtime = params.init_par_runtime(list_tile_IDs)
+    par_runtime = init_par_runtime(list_tile_IDs)
     
     job_data.print_stats_header()
 
     for key in "1":
-        job = jobs[key]
-        job.print_intro()
-        job.check_numbers(par_runtime=par_runtime, indices=[0, 1])
+        jobs[key].print_intro()
+        jobs[key].check_numbers(par_runtime=par_runtime, indices=[0, 1])
 
-        all_exposures = get_all_exposures(job._paths_in_dir[1], verbose=True)
+        all_exposures = get_all_exposures(jobs[key]._paths_in_dir[1], verbose=True)
         par_runtime["n_exposures"] = len(all_exposures)
         par_runtime["list_exposures"] = all_exposures
 
-        job.check_numbers(par_runtime, indices=[2])
+        jobs[key].check_numbers(par_runtime, indices=[2])
 
-    par_runtime = params.update_par_runtime_after_find_exp(par_runtime, all_exposures)
+    par_runtime = update_par_runtime_after_find_exp(par_runtime, all_exposures)
     
     print_par_runtime(par_runtime, verbose=verbose)
 
+
     # Get all keys after "1"
     keys = sorted(jobs.keys(), key=int)
     _ = keys.pop(0)
 
     for key in keys:
-        job = jobs[key]
-        job.print_intro()
-        job.check_numbers(par_runtime=par_runtime)
+        jobs[key].print_intro()
+        jobs[key].check_numbers(par_runtime=par_runtime)
 
     return 0
 
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index cdffc1f23..42b7ebf93 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -15,6 +15,8 @@
 
 from tqdm import tqdm
 
+print("summaary v1.0")
+
 
 def get_IDs_from_file(path):
     """Get IDs From File.
@@ -408,16 +410,20 @@ def output_missing(
         if n_unique > 0:
             if not self._output_path_missing_IDs:
                 output_path = (
-                    f"{self._path_main}/summary/missing_job_{self._bit}_{module}.txt"
+                    f"{self._path_main}/summary/missing_job_{self._bit}"
+                    + f"_{module}.txt"
                 )
             else:
                 output_path = self._output_path_missing_IDs[idx]
+            #print("MKDEBUG", missing_IDs_unique)
             self.write_IDs_to_file(output_path, missing_IDs_unique)
 
         return missing_IDs_unique
 
     def output_missing_job(self):
-        output_path = f"{self._path_main}/summary/missing_job_{self._bit}_all.txt"
+        output_path = (
+            f"{self._path_main}/summary/missing_job_{self._bit}_all.txt"
+        )
 
         missing_IDs_all = set(self._missing_IDs_job)
 
@@ -483,21 +489,6 @@ def get_matches_final(self, directory, idx):
                         os.path.join(directory, entry2.name)
                     )
 
-        #if os.path.exists(directory):
-            #with os.scandir(directory) as entries2:
-                #files = [
-                    #entry2.name
-                    #for entry2 in entries2
-                    #if entry2.name.startswith(self._pattern[idx])
-                #]
-
-                ## Append matching files
-                #self._names_in_dir[idx].extend(files)
-                #self._paths_in_dir[idx].extend(
-                    #[os.path.join(directory, file)
-                    #for file in files]
-                #)
-
     def get_names_in_dir(self, iterable, module, idx):
 
         # Initialise output file names and paths
@@ -687,7 +678,7 @@ def get_par_runtime(par_runtime, key, kind="n"):
 
 def print_par_runtime(par_runtime, verbose=True):
     # Print runtime parameter values
-    if verbose:
+    if True:
         logging.info("")
         logging.info("===========")
         logging.info("par_runtime")

From 3ab923f2bcfae07bbf1e141002aef2fb47ca8a0d Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 11 Feb 2024 10:59:28 +0000
Subject: [PATCH 110/133] terminal title from within curl script

---
 auxdir/CFIS/tiles_202106/tiles_P7.txt   |  1 +
 scripts/sh/curl_canfar_local.sh         |  7 ++++---
 scripts/sh/curl_canfar_monitor_local.sh |  8 ++++++++
 scripts/sh/job_curl_canfar_local.sh     | 17 -----------------
 4 files changed, 13 insertions(+), 20 deletions(-)
 delete mode 100644 scripts/sh/job_curl_canfar_local.sh

diff --git a/auxdir/CFIS/tiles_202106/tiles_P7.txt b/auxdir/CFIS/tiles_202106/tiles_P7.txt
index 535084a2e..93a1dfdb6 100644
--- a/auxdir/CFIS/tiles_202106/tiles_P7.txt
+++ b/auxdir/CFIS/tiles_202106/tiles_P7.txt
@@ -229,3 +229,4 @@
 094.300
 094.301
 095.300
+000.000
diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index 3ecacc3cd..3e03dcd5a 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -26,7 +26,7 @@ dry_run=0
 usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS]
 \n\nOptions:\n
    -h\tthis message\n
-   -j, --job JOB\tRUnning JOB, bit-coded\n
+   -j, --job JOB\tRunning JOB, bit-coded\n
    -e, --exclusive ID
     \timage ID\n
    -f, --file_IDs path
@@ -42,7 +42,7 @@ usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS]
    -C, --command_remote\n
     \tremote command to run on canfar, default='$cmd_remote'\n
    -b, --batch_max\n
-    \tmaximum batch size = number of jobs run simultaneously\n
+    \tmaximum batch size = number of jobs run simultaneously, default=$batch_max\n
    -n, --dry_run LEVEL\n
     \tdry run, from LEVEL=2 (no processing) to 0 (full run)\n
 "
@@ -159,7 +159,7 @@ function submit_batch() {
 }
 
 batch=20
-sleep=600
+sleep=300
 
 ((n_thresh=batch_max-batch))
 
@@ -206,6 +206,7 @@ else
       for batch in $prefix*; do
         echo "Number of running jobs = $n_running"
         echo "Submitting batch $batch ($count/$n_split)"
+        echo -ne "\033]0;curl patch=$patch job=$job $count/$n_split\007"
         submit_batch $batch
         ((count=count+1))
 
diff --git a/scripts/sh/curl_canfar_monitor_local.sh b/scripts/sh/curl_canfar_monitor_local.sh
index 4ba0bf875..793d80262 100755
--- a/scripts/sh/curl_canfar_monitor_local.sh
+++ b/scripts/sh/curl_canfar_monitor_local.sh
@@ -9,6 +9,14 @@ type=$1
 
 echo "type=$type"
 
+for session_ID in `cat session_IDs.txt`; do
+  cmd="curl -E $SSL $SESSION/$session_ID?view=$type"
+  echo $cmd
+  $cmd
+done
+
+exit 0
+
 while [ 1 ]; do
   session_ID=`tail -n 1 session_IDs.txt`
   cmd="curl -E $SSL $SESSION/$session_ID?view=$type"
diff --git a/scripts/sh/job_curl_canfar_local.sh b/scripts/sh/job_curl_canfar_local.sh
deleted file mode 100644
index 665f6ea79..000000000
--- a/scripts/sh/job_curl_canfar_local.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-# Usage:
-# Edit the file "all.txt"
-# screen
-# bash run_curl.sh kind job
-
-rm -f session_IDs.txt session_image_IDs.txt
-
-script_local=~/shapepipe/scripts/sh/curl_canfar_local.sh
-version="1.0"
-cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
-N_SMP=1
-kind="$1"
-job="$2"
-
-echo $kind $job
-cat all.txt | xargs -n 1 -P 1 $script_local -v $version -c $cmd_remote -N $N_SMP -k $kind -j $job -e
-

From ee5b3ad96f6b8633b6a973654864b48674395486 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Sun, 11 Feb 2024 12:03:01 +0100
Subject: [PATCH 111/133] Update tiles_P7.txt

remove 000.000
---
 auxdir/CFIS/tiles_202106/tiles_P7.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/auxdir/CFIS/tiles_202106/tiles_P7.txt b/auxdir/CFIS/tiles_202106/tiles_P7.txt
index 93a1dfdb6..535084a2e 100644
--- a/auxdir/CFIS/tiles_202106/tiles_P7.txt
+++ b/auxdir/CFIS/tiles_202106/tiles_P7.txt
@@ -229,4 +229,3 @@
 094.300
 094.301
 095.300
-000.000

From 1d928d461c56873983e9a4244681488f415c3e88 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Fri, 23 Feb 2024 10:44:41 +0000
Subject: [PATCH 112/133] monitoring and job handling scripts; small modifs,
 output; canfar pipeline job description to docs/source

---
 auxdir/CFIS/tiles_202106/tiles_P7.txt         |  1 +
 docs/source/pipeline_canfar.md                | 91 +++++++++++++++++++
 .../cfis/config_merge_sep_cats_template.ini   |  3 -
 scripts/jupyter/summary_run.ipynb             |  4 +-
 scripts/python/stats_headless_canfar.py       |  9 ++
 scripts/python/summary_params_pre_v2.py       | 12 ++-
 scripts/sh/combine_runs.bash                  |  2 +-
 scripts/sh/curl_canfar_local.sh               | 18 ++--
 scripts/sh/missing_unique.sh                  | 27 ++++++
 scripts/sh/stats_jobs_canfar.sh               | 90 ++++++++++++++++++
 10 files changed, 244 insertions(+), 13 deletions(-)
 create mode 100644 docs/source/pipeline_canfar.md
 create mode 100644 scripts/sh/missing_unique.sh
 create mode 100755 scripts/sh/stats_jobs_canfar.sh

diff --git a/auxdir/CFIS/tiles_202106/tiles_P7.txt b/auxdir/CFIS/tiles_202106/tiles_P7.txt
index 535084a2e..cd8b6b5af 100644
--- a/auxdir/CFIS/tiles_202106/tiles_P7.txt
+++ b/auxdir/CFIS/tiles_202106/tiles_P7.txt
@@ -1,3 +1,4 @@
+000.000
 052.332
 053.331
 053.332
diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md
new file mode 100644
index 000000000..23213f7cd
--- /dev/null
+++ b/docs/source/pipeline_canfar.md
@@ -0,0 +1,91 @@
+patch="P7"
+psf="psfex"
+N_SMP=16
+
+# Terminal title
+echo -ne "\033]0;$patch\007"
+
+# Run directory
+dir=~/cosmostat/v2/pre_v2/$psf/$patch
+cd $dir
+
+# Get tile number list
+ln -s ~/shapepipe/auxdir/CFIS/tiles_202106/tiles_$patch.txt tile_numbers.txt
+
+
+# Get images
+
+## Download and link separately
+
+### Download
+### Create and link to central image storage directory
+mkdir -p ~/cosmostat/v2/data_tiles/$patch
+ln -s ~/cosmostat/v2/data_tiles/$patch data_tiles
+
+### Download and move tiles 
+ln -s ~/shapepipe/example/cfis
+mkdir -p output
+export SP_RUN=`pwd`
+
+shapepipe_run -c cfis/config_Git_vos.ini
+mv -i output/run_sp_Git_*/get_images_runner/output/CFIS.???.???.*fits* data_tiles
+rm -rf output/run_sp_tiles_Git_*
+update_run_log_file.py
+# repeat the above block
+
+### Find exposures; this run can be stopped after Fe
+shapepipe_run -c cfis/config_GitFe_symlink.ini
+
+### Download and move exposures
+
+shapepipe_run -c cfis/config_Gie_vos.ini
+mv -i output/run_sp_Gie_*/get_images_runner/output/*.fits*fz data_exp
+rm -rf  output/run_sp_Gie_*
+update_run_log_file.py
+# repeat the above
+
+### Create links (and re-run Fe, not necessary)
+job_sp_canfar.bash -p $psf `cat tile_numbers.txt` -j 1 -r symlink
+
+# Uncompress weights,  split exposures into single HDUs
+job_sp_canfar.bash -p $psf -n $N_SMP -j 2
+
+# Mask tiles
+job_sp_canfar.bash -p $psf -n $N_SMP -j 4
+
+# Mask exposures
+job_sp_canfar.bash -p $psf -n $N_SMP -j 8
+
+
+# Tile detection
+curl_canfar_local.sh -j 16 -f tile_numbers.txt -k tile -p $psf -N $N_SMP
+
+
+# Exposure detection
+## Get single-HDU single-exposure IDs
+~/shapepipe/scripts/python/summary_run.py
+
+cp summary/missing_job_32_sextractor.txt all.txt
+curl_canfar_local.sh -j 32 -f all.txt -k exp -p $psf -N $N_SMP
+
+# Tile preparation
+curl_canfar_local.sh -j 64 -f tile_numbers.txt -k tile -p $psf -N $N_SMP
+
+# Tile shape measurement
+curl_canfar_local.sh -j 128 -f tile_numbers.txt -k tile -p $psf -N 8
+
+# Merge subcatalogues, and create final cat
+job_sp_canfar.bash -p $psf -n 1 -j 256
+
+# Combine all final cats in common output dir as links
+combine_runs.bash -c final -p psfex
+
+# Merge all final cats
+# (use 192GB RAM)
+merge_final_cat -i output/run_sp_combined_final/make_catalog_runner/output -p cfis/final_cat.param -v
+
+
+# Delete jobs
+SSL=~/.ssl/cadcproxy.pem
+SESSION=https://ws-uv.canfar.net/skaha/v0/session
+for ID in `cat session_IDs.txt`; do echo $ID; curl -X DELETE -E $SSL $SESSION/$ID; done
diff --git a/example/cfis/config_merge_sep_cats_template.ini b/example/cfis/config_merge_sep_cats_template.ini
index e14e5fe74..36b99e8da 100644
--- a/example/cfis/config_merge_sep_cats_template.ini
+++ b/example/cfis/config_merge_sep_cats_template.ini
@@ -35,7 +35,6 @@ RUN_LOG_NAME = log_run_sp
 
 # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
 INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output
-#, ./output/run_sp_tile_ngmix_Ng1u/galsim_shapes_v2_runner/output
 
 # Output directory
 OUTPUT_DIR = ./output
@@ -58,11 +57,9 @@ TIMEOUT = 96:00:00
 # Input file pattern(s), list of strings with length matching number of expected input file types
 # Cannot contain wild cards
 FILE_PATTERN = ngmix
-#, galsim
 
 # FILE_EXT (optional) list of string extensions to identify input files
 FILE_EXT = .fits
-#, .fits
 
 # Numbering convention, string that exemplifies a numbering pattern.
 NUMBERING_SCHEME = -000-000
diff --git a/scripts/jupyter/summary_run.ipynb b/scripts/jupyter/summary_run.ipynb
index 451c591d0..f527e4159 100644
--- a/scripts/jupyter/summary_run.ipynb
+++ b/scripts/jupyter/summary_run.ipynb
@@ -143,8 +143,8 @@
      "output_type": "stream",
      "text": [
       " (Job 1)\n",
-      "get_images_runner_run_1              464       462         0         2       1.0     99.6%\n",
-      "find_exposures_runner                232       231         0         1       1.0     99.6%\n",
+      "get_images_runner_run_1              462       462         0         0       0.0    100.0%\n",
+      "find_exposures_runner                231       231         0         0       0.0    100.0%\n",
       "get_images_runner_run_2              537         0         0       537     179.0      0.0%\n"
      ]
     }
diff --git a/scripts/python/stats_headless_canfar.py b/scripts/python/stats_headless_canfar.py
index 1ede78eed..1f728949d 100755
--- a/scripts/python/stats_headless_canfar.py
+++ b/scripts/python/stats_headless_canfar.py
@@ -2,11 +2,20 @@
 
 # Name: stats_headless_canfar.py
 
+# Caution: Does not show all running or pending
+# headless jobs, for some reason.
+
 import sys
 from skaha.session import Session
 
 
 def main(argv=None):
+
+    print(
+        "# Depreciated, does not show pending jobs; use stats_jobs_canfar.sh",
+        file=sys.stderr,
+    )
+
     session = Session()
 
     n_headless = session.stats()["instances"]["headless"]
diff --git a/scripts/python/summary_params_pre_v2.py b/scripts/python/summary_params_pre_v2.py
index 5b67ee0ca..d8d08201f 100644
--- a/scripts/python/summary_params_pre_v2.py
+++ b/scripts/python/summary_params_pre_v2.py
@@ -74,7 +74,7 @@ def set_jobs_v2_pre_v2(patch, verbose):
     # - remove previous output dirs since only last is searched
     jobs["1"] = job_data(
         1,
-         "run_sp_Git",
+         "run_sp_GitFeGie",
        [
             "get_images_runner_run_1",
             "find_exposures_runner",
@@ -220,6 +220,16 @@ def set_jobs_v2_pre_v2(patch, verbose):
     )
 
     # Post-processing
+    jobs["512"] = job_data(
+        "512",
+        ["run_sp_combined_final"],
+        ["make_cat_runner"],
+        "tile_IDs",
+        path_main=path_main,
+        path_left="output",
+        verbose=verbose,
+    )
+
     jobs["1024"] = job_data(
         "1024",
         "run_sp_combined_psf",
diff --git a/scripts/sh/combine_runs.bash b/scripts/sh/combine_runs.bash
index 89e844c63..2226e3e44 100755
--- a/scripts/sh/combine_runs.bash
+++ b/scripts/sh/combine_runs.bash
@@ -175,7 +175,7 @@ for dir in $run_in; do
 done
 
 #echo " $n_files target files, $i links created/skipped"
-echo " $i total, "n_skipped skipped, "n_created links created"
+echo " $i total, "$n_skipped skipped, "$n_created links created"
 
 # Update log file
 update_runs_log_file.py
diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index 3e03dcd5a..a8a095ac1 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -15,7 +15,7 @@ ID=-1
 file_IDs=-1
 N_SMP=1
 kind=-1
-version=1.0
+version="1.1"
 cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
 batch_max=200
 dry_run=0
@@ -92,7 +92,7 @@ while [ $# -gt 0 ]; do
   shift
 done
 
-# Check options                                                                 
+## Check options                                                                 
 if [ "$job" == "-1" ]; then                                                     
   echo "No job indicated, use option -j"                                        
   exit 2                                                                        
@@ -136,11 +136,13 @@ function set_arg() {
 }
 
 
+# MKDEBUG TODO
 function call_curl() {
   my_arg=$1
 
 }
 
+# Add session and image IDs to log files
 function update_session_logs() {
   echo $my_session >> session_IDs.txt
   echo "$my_session $ID" >> session_image_IDs.txt
@@ -152,7 +154,7 @@ function submit_batch() {
 
   for ID in `cat $path`; do
     my_arg=$(set_arg)
-    my_session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$my_arg"`
+    my_session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$my_arg" &> /dev/null`
     update_session_logs
   done
 
@@ -203,6 +205,8 @@ else
       echo "Split '$file_IDs' into $n_split batches of size $batch"
 
       count=1
+      #n_running=`stats_headless_canfar.py`
+      n_running=`stats_jobs_canfar.sh`
       for batch in $prefix*; do
         echo "Number of running jobs = $n_running"
         echo "Submitting batch $batch ($count/$n_split)"
@@ -210,12 +214,14 @@ else
         submit_batch $batch
         ((count=count+1))
 
-        n_running=`stats_headless_canfar.py`
+        #n_running=`stats_headless_canfar.py`
+        n_running=`stats_jobs_canfar.sh`
 
         while [ "$n_running" -gt "$n_thresh" ]; do
           echo "Wait for #jobs = $n_running jobs to go < $n_thresh ..."
           sleep $sleep
-          n_running=`stats_headless_canfar.py`
+          #n_running=`stats_headless_canfar.py`
+          n_running=`stats_jobs_canfar.sh`
         done
 
       done
@@ -232,7 +238,7 @@ else
 
     # Submit image
     arg=$(set_arg)
-    session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg"`
+    session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg" &> /dev/null`
     update_session_logs
 
   fi
diff --git a/scripts/sh/missing_unique.sh b/scripts/sh/missing_unique.sh
new file mode 100644
index 000000000..8820d26f5
--- /dev/null
+++ b/scripts/sh/missing_unique.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+FILES=("summary/missing_job_128_ngmix_runner_*.txt")
+temp="temp_temp.tmp"
+temp2="temp_temp2.tmp"
+out="missing_job_128_ngmix_runner_cut.txt"
+
+i=0
+for file in ${FILES[@]}; do
+
+  echo $file $i
+
+  if [ "$i" == "0" ]; then
+    cp $file $temp
+  else
+    comm -12  <(sort $file) <(sort $temp) > $temp2
+    cp $temp2 $temp
+  fi
+
+  wc $file $temp
+
+  ((i=i+1))
+
+done
+
+mv $temp $out
+rm $temp2
diff --git a/scripts/sh/stats_jobs_canfar.sh b/scripts/sh/stats_jobs_canfar.sh
new file mode 100755
index 000000000..3c1332f59
--- /dev/null
+++ b/scripts/sh/stats_jobs_canfar.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+# Name: stats_jobs_canfar.sh
+# Author: Martin Kilbinger <martin.kilbinger@cea.fr>
+# Description: Handles headless jobs on canfar
+
+
+# Global variables
+
+## Temporary files
+tmpfile_jobs="jobinfo.txt"
+tmpfile_ids="ids.txt"
+
+## curl options
+SSL=~/.ssl/cadcproxy.pem
+SESSION=https://ws-uv.canfar.net/skaha/v0/session
+
+
+# Command line arguments
+
+## Default values
+mode="count"
+
+## Help string
+usage="Usage: $(basename "$0") -j JOB -[e ID |-f file_IDs] -k KIND [OPTIONS]
+\n\nOptions:\n
+   -h\tthis message\n
+   -m, --mode MODE\n
+   \tmode, allowed are 'count' (default), 'delete'\n
+"
+
+## Parse command line
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -h)
+      echo -ne $usage
+      exit 0
+      ;;
+    -m|--mode)
+      mode="$2"
+      shift
+      ;;
+  esac
+  shift
+done
+
+## Check options
+case $mode in
+  "count"|"delete")
+    # valid option
+    ;;
+  *)
+    echo "Invalid mode $mode"
+    exit 1
+    ;;
+esac
+
+
+# Main program
+
+# Get all instances
+curl -E $SSL $SESSION &> /dev/null > $tmpfile_jobs
+
+# Get headless job IDs
+cat $tmpfile_jobs | grep headless -B 4 -A 12 | grep \"id | perl -F\" -ane 'print "$F[3]\n"' > $tmpfile_ids
+
+# Number of jobs
+n_headless=`cat $tmpfile_ids | wc -l`
+
+if [ "$mode" == "count" ]; then
+
+  echo $n_headless
+
+elif [ "$mode" == "delete" ]; then
+
+  echo -n "Delete $n_headless jobs? [y|n] "
+  read answer
+  if [ "$answer" == "y" ]; then
+    for ID in `cat $tmpfile_ids`; do
+      echo $ID
+      # Delete headless jobs
+      #curl -X DELETE -E $SSL $SESSION/$ID
+    done
+  fi
+
+fi
+
+
+# Remove temporary files
+rm -f $tmpfile_jobs $tmpfile_ids

From a6d8b521cbe3921c7d3015001df194b8f012397f Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Tue, 5 Mar 2024 13:11:28 +0000
Subject: [PATCH 113/133] combine run script: flag_tile and flag_exp as diff
 options

---
 scripts/sh/combine_runs.bash | 44 ++++++++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/scripts/sh/combine_runs.bash b/scripts/sh/combine_runs.bash
index 2226e3e44..c5a99cde7 100755
--- a/scripts/sh/combine_runs.bash
+++ b/scripts/sh/combine_runs.bash
@@ -17,9 +17,10 @@ usage="Usage: $(basename "$0") [OPTIONS]
 \n\nOptions:\n
    -h\tthis message\n
    -p, --psf MODEL\n
-    \tPSF model, one in ['psfex'|'mccd'|'setools'], default='$psf'\n
+    \tPSF model, allowed are 'psfex', 'mccd', 'setools', default='$psf'\n
    -c, --cat TYPE\n
-    \tCatalogue type, one in ['final'|'flag'|'psf'|'image'], default='$cat'\n
+    \tCatalogue type, allowed are 'final', 'flag_tile', 'flag_exp', \n
+    \t'psf', 'image', default='$cat'\n
 "
 
 ## Parse command line
@@ -48,10 +49,11 @@ done
 
 ## Check options
 if [ "$cat" != "final" ] \
-  && [ "$cat" != "flag" ] \
+  && [ "$cat" != "flag_tile" ] \
+  && [ "$cat" != "flag_exp" ] \
   && [ "$cat" != "psf" ] \
   && [ "$cat" != "image" ]; then
-  echo "cat (option -c) needs to be 'final', 'flag', 'psf', or 'image'"
+  echo "cat (option -c) needs to be 'final', 'flag_tile', 'flag_exp', 'psf', or 'image'"
   exit 2
 fi
 
@@ -106,11 +108,27 @@ if [ "$cat" == "final" ]; then
   module="make_catalog_runner"
   pattern="final_cat-*"
 
-elif [ "$cat" == "flag" ]; then
+elif [ "$cat" == "flag_tile" ]; then
 
-  run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
-  module="mask_runner_run_1"
-  pattenr="pipeline_flag-*"
+  # v1
+  #run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
+  # v2
+  run_in="$pwd/$out_base/run_sp_tile_Ma_*"
+  run_out="run_sp_tile_Ma"
+
+  module="mask_runner"
+  pattern="pipeline_flag-*"
+
+elif [ "$cat" == "flag_exp" ]; then
+
+  # v1
+  #run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_2"
+  # v2
+  run_in="$pwd/$out_base/run_sp_exp_Ma_*"
+  run_out="run_sp_exp_Ma"
+
+  module="mask_runner"
+  pattern="pipeline_flag-*"
 
 elif [ "$cat" == "image" ]; then
 
@@ -162,13 +180,15 @@ i=0
 for dir in $run_in; do
   FILES=(`find $dir -type f -name "$pattern" -print0 | xargs -0 echo`)
 
+  echo "$dir $pattern"
+
   ## Look over source files
   for file in ${FILES[@]}; do
 
-  target=$file
-  link_name=$outdir/`basename $file`
-  link_s $target $link_name
-  ((i=i+1))
+    target=$file
+    link_name=$outdir/`basename $file`
+    link_s $target $link_name
+    ((i=i+1))
 
   done
 

From 1a2d47d6bfc375ba82a96c89ab1791711066b7f8 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 09:02:27 +0000
Subject: [PATCH 114/133] mask runer excluding processed files working

---
 docs/source/pipeline_canfar.md                   |  4 ++++
 example/cfis/config_tile_Sx.ini                  |  2 +-
 scripts/sh/curl_canfar_local.sh                  |  2 --
 shapepipe/modules/mask_package/__init__.py       |  3 +++
 shapepipe/modules/mask_package/mask.py           | 16 ++++++++++++++++
 shapepipe/modules/mask_runner.py                 | 10 ++++++++++
 .../merge_sep_cats_package/merge_sep_cats.py     |  6 ++++++
 shapepipe/utilities/summary.py                   |  7 ++++---
 8 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md
index 23213f7cd..7227cf463 100644
--- a/docs/source/pipeline_canfar.md
+++ b/docs/source/pipeline_canfar.md
@@ -53,6 +53,10 @@ job_sp_canfar.bash -p $psf -n $N_SMP -j 2
 # Mask tiles
 job_sp_canfar.bash -p $psf -n $N_SMP -j 4
 
+# If not finshed:
+combine_runs.bash -p psfex -c flag
+mv output/run_sp_combined_flag output/run_sp_exp_Ma
+
 # Mask exposures
 job_sp_canfar.bash -p $psf -n $N_SMP -j 8
 
diff --git a/example/cfis/config_tile_Sx.ini b/example/cfis/config_tile_Sx.ini
index 1c418245d..a079c2488 100644
--- a/example/cfis/config_tile_Sx.ini
+++ b/example/cfis/config_tile_Sx.ini
@@ -55,7 +55,7 @@ TIMEOUT = 96:00:00
 
 [SEXTRACTOR_RUNNER]
 
-INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, run_sp_tile_Ma:mask_runner
+INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, run_sp_Ma_tile:mask_runner
 
 FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag
 
diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index a8a095ac1..e745ebd06 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -214,13 +214,11 @@ else
         submit_batch $batch
         ((count=count+1))
 
-        #n_running=`stats_headless_canfar.py`
         n_running=`stats_jobs_canfar.sh`
 
         while [ "$n_running" -gt "$n_thresh" ]; do
           echo "Wait for #jobs = $n_running jobs to go < $n_thresh ..."
           sleep $sleep
-          #n_running=`stats_headless_canfar.py`
           n_running=`stats_jobs_canfar.sh`
         done
 
diff --git a/shapepipe/modules/mask_package/__init__.py b/shapepipe/modules/mask_package/__init__.py
index af2dc5c5c..ccd55fa9a 100644
--- a/shapepipe/modules/mask_package/__init__.py
+++ b/shapepipe/modules/mask_package/__init__.py
@@ -63,6 +63,9 @@
     Prefix to be appended to output file name ``flag``;
     helps to distinguish the file patterns of newly created and external
     mask files
+CHECK_EXISTING_DIR : str, optional
+    If given, search this directory for existing mask files; the
+    corresponding images will then not be processed
 
 Mask config file
 ================
diff --git a/shapepipe/modules/mask_package/mask.py b/shapepipe/modules/mask_package/mask.py
index e34eb9583..a81ce18d0 100644
--- a/shapepipe/modules/mask_package/mask.py
+++ b/shapepipe/modules/mask_package/mask.py
@@ -45,6 +45,8 @@ class Mask(object):
         Path to external flag file, default is ``None`` (not used)
     outname_base : str, optional
        Output file name base, default is ``flag``
+    check_existing_dir : str, optional
+        If not ``None`` (default), search path for existing mask files
     star_cat_path : str, optional
         Path to external star catalogue, default is ``None`` (not used;
         instead the star catalogue is produced on the fly at run time)
@@ -64,6 +66,7 @@ def __init__(
         w_log,
         path_external_flag=None,
         outname_base='flag',
+        check_existing_dir=None,
         star_cat_path=None,
         hdu=0,
     ):
@@ -98,6 +101,9 @@ def __init__(
         # Output file base name
         self._outname_base = outname_base
 
+        # Search path for existing mask files
+        self._check_existing_dir= check_existing_dir
+
         # Set external star catalogue path if given
         if star_cat_path is not None:
             self._star_cat_path = star_cat_path
@@ -308,6 +314,16 @@ def make_mask(self):
         Main function to create the mask.
 
         """
+        output_file_name = (
+            f'{self._img_prefix}'
+            + f'{self._outname_base}{self._img_number}.fits'
+        )
+        if (
+            os.path.exists(f"{self._check_existing_dir}//{output_file_name}")
+        ):
+            print("MKDEBUG skipping ", output_file_name)
+            return None, None
+
         if self._config['MD']['make']:
             self.missing_data()
 
diff --git a/shapepipe/modules/mask_runner.py b/shapepipe/modules/mask_runner.py
index 3683076e4..c3215107c 100644
--- a/shapepipe/modules/mask_runner.py
+++ b/shapepipe/modules/mask_runner.py
@@ -91,6 +91,15 @@ def mask_runner(
 
     outname_base = 'flag'
 
+    # Path to check for already created mask files
+    if config.has_option(module_config_sec, 'CHECK_EXISTING_DIR'):
+        check_existing_dir = config.getexpanded(
+            module_config_sec,
+            'CHECK_EXISTING_DIR'
+        )
+    else:
+        check_existing_dir = None
+
     # Create instance of Mask
     mask_inst = Mask(
         *input_file_list[:2],
@@ -101,6 +110,7 @@ def mask_runner(
         path_external_flag=ext_flag_name,
         outname_base=outname_base,
         star_cat_path=ext_star_cat,
+        check_existing_dir=check_existing_dir,
         hdu=hdu,
         w_log=w_log,
     )
diff --git a/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py b/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py
index 85f7e194f..d9200529e 100644
--- a/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py
+++ b/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py
@@ -87,6 +87,12 @@ def process(self):
             cat0.open()
             list_ext_name = cat0.get_ext_name()
             list_col_name = cat0.get_col_names()
+            # MKDEBUG: Some input ngmix catalogues have multiple of 5 HDUs
+            # if reprocessed and not deleted but appended
+            if len(list_ext_name) > 6:
+                wmsg = f"Cropping input HDUs from {len(list_ext_name)} to 5"
+                self._w_log.info(wmsg)
+                list_ext_name = list_ext_name[:6]
             cat0.close()
 
             # Create empty dictionary
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index 42b7ebf93..0aedb4475 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -430,7 +430,8 @@ def output_missing_job(self):
         if len(missing_IDs_all) > 0:
             self.write_IDs_to_file(output_path, missing_IDs_all)
         else:
-            logging.warning("no missing IDs in output_missing_job")
+            #logging.warning("no missing IDs in output_missing_job")
+            os.unlink(output_path)
 
     @classmethod
     def get_last_full_path(self, base_and_subdir, matches):
@@ -648,8 +649,8 @@ def check_numbers(self, par_runtime=None, indices=None):
                 self._missing_IDs_job.extend(missing_IDs)
 
         # Write missing IDs for entire job to file
-        if n_missing_job > 0:
-            self.output_missing_job()
+        #if n_missing_job > 0:
+        self.output_missing_job()
 
 
 def get_par_runtime(par_runtime, key, kind="n"):

From de82961945be79ec0a0536ec8a9e29f13ce0dc96 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 09:09:16 +0000
Subject: [PATCH 115/133] combine runs for masks: changed output target dir
 name

---
 scripts/sh/combine_runs.bash | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/sh/combine_runs.bash b/scripts/sh/combine_runs.bash
index c5a99cde7..3b8a27824 100755
--- a/scripts/sh/combine_runs.bash
+++ b/scripts/sh/combine_runs.bash
@@ -114,7 +114,7 @@ elif [ "$cat" == "flag_tile" ]; then
   #run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
   # v2
   run_in="$pwd/$out_base/run_sp_tile_Ma_*"
-  run_out="run_sp_tile_Ma"
+  run_out="run_sp_Ma_tile"
 
   module="mask_runner"
   pattern="pipeline_flag-*"
@@ -125,7 +125,7 @@ elif [ "$cat" == "flag_exp" ]; then
   #run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_2"
   # v2
   run_in="$pwd/$out_base/run_sp_exp_Ma_*"
-  run_out="run_sp_exp_Ma"
+  run_out="run_sp_Ma_exp"
 
   module="mask_runner"
   pattern="pipeline_flag-*"

From 8928c86c2ed9aaec5199b80dedbf92c7bcfbe1f2 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 09:56:54 +0000
Subject: [PATCH 116/133] summary bug fixed

---
 shapepipe/utilities/summary.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index 0aedb4475..ae574e5c4 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -431,7 +431,8 @@ def output_missing_job(self):
             self.write_IDs_to_file(output_path, missing_IDs_all)
         else:
             #logging.warning("no missing IDs in output_missing_job")
-            os.unlink(output_path)
+            if os.path.exists(output_path):
+                os.unlink(output_path)
 
     @classmethod
     def get_last_full_path(self, base_and_subdir, matches):

From fc5a7df2ae9910dc01566f1b47d88a335a6f18ab Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 09:57:54 +0000
Subject: [PATCH 117/133] comment removed

---
 scripts/sh/curl_canfar_local.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/sh/curl_canfar_local.sh b/scripts/sh/curl_canfar_local.sh
index e745ebd06..3c2c73658 100755
--- a/scripts/sh/curl_canfar_local.sh
+++ b/scripts/sh/curl_canfar_local.sh
@@ -205,7 +205,6 @@ else
       echo "Split '$file_IDs' into $n_split batches of size $batch"
 
       count=1
-      #n_running=`stats_headless_canfar.py`
       n_running=`stats_jobs_canfar.sh`
       for batch in $prefix*; do
         echo "Number of running jobs = $n_running"

From e882df3376ab7c54471692c688ab68704c676018 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 09:59:15 +0000
Subject: [PATCH 118/133] style

---
 shapepipe/modules/mask_package/mask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shapepipe/modules/mask_package/mask.py b/shapepipe/modules/mask_package/mask.py
index a81ce18d0..fc40d1f30 100644
--- a/shapepipe/modules/mask_package/mask.py
+++ b/shapepipe/modules/mask_package/mask.py
@@ -102,7 +102,7 @@ def __init__(
         self._outname_base = outname_base
 
         # Search path for existing mask files
-        self._check_existing_dir= check_existing_dir
+        self._check_existing_dir = check_existing_dir
 
         # Set external star catalogue path if given
         if star_cat_path is not None:

From 93e86e934c338ec35b79eea042249d6eb1abc9e5 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 13:25:59 +0100
Subject: [PATCH 119/133] Delete scripts/sh/init_canfar.sh

---
 scripts/sh/init_canfar.sh | 13 -------------
 1 file changed, 13 deletions(-)
 delete mode 100755 scripts/sh/init_canfar.sh

diff --git a/scripts/sh/init_canfar.sh b/scripts/sh/init_canfar.sh
deleted file mode 100755
index dd3346f94..000000000
--- a/scripts/sh/init_canfar.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-echo "start init canfar"
-
-echo init_canfar > ~/init_canfar.log
-date >> ~/init_canfar.log
-
-. /opt/conda/etc/profile.d/conda.sh
-
-conda activate shapepipe
-
-echo "end init canfar"
-

From a6bf8a52b174d6f7a49a70c578fc1f2e57b9643a Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 15:30:08 +0100
Subject: [PATCH 120/133] remove centos ci

---
 .github/workflows/ci-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
index ba9fe6cce..cc4ae4e93 100644
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-tests.yml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, centos-latest]
+        os: [ubuntu-latest, macos-latest]
         python-version: [3.9]
 
     steps:

From bddc491c4e58876ca4af76717455f0a9e7e45757 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 16:29:12 +0100
Subject: [PATCH 121/133] repaired plot spectro notebook

---
 scripts/jupyter/plot_spectro_areas.ipynb | 76 ++----------------------
 1 file changed, 5 insertions(+), 71 deletions(-)

diff --git a/scripts/jupyter/plot_spectro_areas.ipynb b/scripts/jupyter/plot_spectro_areas.ipynb
index 6497c57a6..b881d208c 100644
--- a/scripts/jupyter/plot_spectro_areas.ipynb
+++ b/scripts/jupyter/plot_spectro_areas.ipynb
@@ -9,27 +9,9 @@
   },
   {
    "cell_type": "code",
-<<<<<<< HEAD
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'skymapper'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn [1], line 12\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mastropy\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m units \u001b[38;5;28;01mas\u001b[39;00m u\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[0;32m---> 12\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mskymapper\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mskm\u001b[39;00m\n",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'skymapper'"
-     ]
-    }
-   ],
-=======
    "execution_count": null,
    "metadata": {},
    "outputs": [],
->>>>>>> origin/develop
    "source": [
     "import os\n",
     "import sys\n",
@@ -63,11 +45,7 @@
   },
   {
    "cell_type": "code",
-<<<<<<< HEAD
-   "execution_count": 2,
-=======
    "execution_count": null,
->>>>>>> origin/develop
    "metadata": {},
    "outputs": [],
    "source": [
@@ -113,11 +91,7 @@
   },
   {
    "cell_type": "code",
-<<<<<<< HEAD
-   "execution_count": 3,
-=======
    "execution_count": null,
->>>>>>> origin/develop
    "metadata": {},
    "outputs": [],
    "source": [
@@ -210,17 +184,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "DESI: 17450894 objects in total, 4835230/17450894 = 27.7% potentially in UNIONS footprint\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "surveys = []\n",
     "\n",
@@ -319,17 +285,7 @@
     "if 'DECaLS-groups'in use:\n",
     "    surveys.append(Survey.from_fits('DECaLS-groups', 'cyan',\n",
     "                                    '{}/DESI_NGC_group_cut.txt.fits'.format(cat_home),\n",
-<<<<<<< HEAD
-    "                                    key_ra='RA[deg]', key_dec='DEC[deg]'))\n",
-    "    \n",
-    "if 'DESI-Iron'in use:\n",
-    "    surveys.append(Survey.from_fits('DESI', 'cyan',\n",
-    "                                    '{}/galaxies_desi_iron_v0.1.fits'.format(cat_home),\n",
-    "                                    key_ra='TARGET_RA', key_dec='TARGET_DEC'))\n",
-    "    "
-=======
     "                                    key_ra='RA[deg]', key_dec='DEC[deg]'))"
->>>>>>> origin/develop
    ]
   },
   {
@@ -341,7 +297,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -406,21 +362,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "AttributeError",
-     "evalue": "module 'cs_util.cfis' has no attribute 'get_image_list'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[6], line 24\u001b[0m\n\u001b[1;32m     22\u001b[0m images \u001b[38;5;241m=\u001b[39m {}\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m band \u001b[38;5;129;01min\u001b[39;00m bands:\n\u001b[0;32m---> 24\u001b[0m     images[band] \u001b[38;5;241m=\u001b[39m \u001b[43mcfis\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_image_list\u001b[49m(tiles_ID[band], band, image_type, input_format\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mID_only\u001b[39m\u001b[38;5;124m'\u001b[39m, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
-      "\u001b[0;31mAttributeError\u001b[0m: module 'cs_util.cfis' has no attribute 'get_image_list'"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Variables\n",
     "unit = 'deg'\n",
@@ -684,15 +628,9 @@
  ],
  "metadata": {
   "kernelspec": {
-<<<<<<< HEAD
-   "display_name": "shapepipe",
-   "language": "python",
-   "name": "shapepipe"
-=======
    "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
->>>>>>> origin/develop
   },
   "language_info": {
    "codemirror_mode": {
@@ -704,11 +642,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-<<<<<<< HEAD
-   "version": "3.9.13"
-=======
    "version": "3.9.7"
->>>>>>> origin/develop
   }
  },
  "nbformat": 4,

From 4dffd478a61223ca1a10ce0d7100c2cfe020db61 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 16:58:13 +0100
Subject: [PATCH 122/133] added doc string

---
 scripts/python/link_to_exp_for_tile.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/python/link_to_exp_for_tile.py b/scripts/python/link_to_exp_for_tile.py
index 0ad205adf..929c90940 100755
--- a/scripts/python/link_to_exp_for_tile.py
+++ b/scripts/python/link_to_exp_for_tile.py
@@ -2,7 +2,10 @@
 
 # -*- coding: utf-8 -*-
 
-"""Script link_to_esp_for_tile.py
+"""Script link_to_exp_for_tile.py
+
+:Description: Link to exposure and PSF catalogue
+ for a given tile.
 
 :Author: Martin Kilbinger
 

From 5675c0654d0b2aafe222eaaa8d42151b58867126 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 18:54:04 +0100
Subject: [PATCH 123/133] python example runner: added back mccd and pysap

---
 shapepipe/modules/python_example_runner.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shapepipe/modules/python_example_runner.py b/shapepipe/modules/python_example_runner.py
index fc5565ecc..d468c10e0 100644
--- a/shapepipe/modules/python_example_runner.py
+++ b/shapepipe/modules/python_example_runner.py
@@ -19,8 +19,10 @@
         'astropy',
         'galsim',
         'joblib',
+        'mccd',
         'ngmix',
         'pandas',
+        'pysap',
         'scipy',
         'sf_tools',
         'sip_tpv',

From 14569b28b75d4b283260f9f6e146f44123010623 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 6 Mar 2024 18:57:41 +0100
Subject: [PATCH 124/133] corrected typo in doc string

---
 shapepipe/pipeline/args.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shapepipe/pipeline/args.py b/shapepipe/pipeline/args.py
index 21eecef84..818f54163 100644
--- a/shapepipe/pipeline/args.py
+++ b/shapepipe/pipeline/args.py
@@ -131,7 +131,7 @@ def create_arg_parser():
     optional.add_argument(
         '-e',
         '--exclusive',
-        help='exclusive input filer number string',
+        help='exclusive input file number string',
     )
     # Return parser
     return parser.parse_args()

From b406438e51a4e990b67994a1ca5a2b7bf2d9f2f4 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Mar 2024 07:01:38 +0000
Subject: [PATCH 125/133] updated canfar pipeline doc

---
 docs/source/pipeline_canfar.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md
index 7227cf463..36a1a2025 100644
--- a/docs/source/pipeline_canfar.md
+++ b/docs/source/pipeline_canfar.md
@@ -79,13 +79,13 @@ curl_canfar_local.sh -j 64 -f tile_numbers.txt -k tile -p $psf -N $N_SMP
 curl_canfar_local.sh -j 128 -f tile_numbers.txt -k tile -p $psf -N 8
 
 # Merge subcatalogues, and create final cat
-job_sp_canfar.bash -p $psf -n 1 -j 256
+curl_canfar_local.sh -j 256 -f tile_numbers.txt -k tile -p $psf -N $N_SMP
 
 # Combine all final cats in common output dir as links
 combine_runs.bash -c final -p psfex
 
 # Merge all final cats
-# (use 192GB RAM)
+# (W3: 140GB RAM)
 merge_final_cat -i output/run_sp_combined_final/make_catalog_runner/output -p cfis/final_cat.param -v
 
 

From fe6683158ccb3a6a201816997242054478e58d90 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Mar 2024 07:02:10 +0000
Subject: [PATCH 126/133] fixed combined mask input dir

---
 example/cfis/config_exp_psfex.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/example/cfis/config_exp_psfex.ini b/example/cfis/config_exp_psfex.ini
index 52ca81f24..a5ac5d8a1 100644
--- a/example/cfis/config_exp_psfex.ini
+++ b/example/cfis/config_exp_psfex.ini
@@ -58,7 +58,7 @@ TIMEOUT = 96:00:00
 [SEXTRACTOR_RUNNER]
 
 # Input from two modules
-INPUT_DIR = last:split_exp_runner, run_sp_exp_Ma:mask_runner
+INPUT_DIR = last:split_exp_runner, run_sp_Ma_exp:mask_runner
 
 # Read pipeline flag files created by mask module
 FILE_PATTERN = image, weight, pipeline_flag

From 282d76b4294609c4fe1124cadb744fe5a33aab8e Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Mar 2024 07:02:24 +0000
Subject: [PATCH 127/133] added parameters to final cat

---
 example/cfis/final_cat.param | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/example/cfis/final_cat.param b/example/cfis/final_cat.param
index 6372a13e8..a70de962d 100644
--- a/example/cfis/final_cat.param
+++ b/example/cfis/final_cat.param
@@ -76,12 +76,23 @@ NGMIX_FLUX_ERR_2M
 NGMIX_FLUX_ERR_2P
 NGMIX_FLUX_ERR_NOSHEAR
 
-# magnitude, mainly for plots
+# magnitudes
 MAG_AUTO
-
-# SNR from SExtractor, used for cuts on GALSIM shapes
+MAGERR_AUTO
+MAG_WIN
+MAGERR_WIN
+FLUX_AUTO
+FLUXERR_AUTO
+FLUX_APER
+FLUXERR_APER
+FLUX_RADIUS
+
+# SNR from SExtractor
 SNR_WIN
 
+FWHM_IMAGE
+FWHM_WORLD
+
 # PSF size measured on original image
 NGMIX_T_PSFo_NOSHEAR
 

From 9ac6bb475440d2848542d799969ff5f65f82bd6c Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Mar 2024 07:02:47 +0000
Subject: [PATCH 128/133] summary

---
 scripts/python/summary_params_pre_v2.py | 3 +--
 shapepipe/utilities/summary.py          | 7 +++++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/scripts/python/summary_params_pre_v2.py b/scripts/python/summary_params_pre_v2.py
index d8d08201f..dc16b59a2 100644
--- a/scripts/python/summary_params_pre_v2.py
+++ b/scripts/python/summary_params_pre_v2.py
@@ -88,7 +88,6 @@ def set_jobs_v2_pre_v2(patch, verbose):
         verbose=verbose,
     )
 
-    #        n_mult=[1, 1, 1],
     jobs["2"] = job_data(
         2,
         ["run_sp_Uz", "run_sp_exp_SpMh", "run_sp_exp_SpMh"],
@@ -223,7 +222,7 @@ def set_jobs_v2_pre_v2(patch, verbose):
     jobs["512"] = job_data(
         "512",
         ["run_sp_combined_final"],
-        ["make_cat_runner"],
+        ["make_catalog_runner"],
         "tile_IDs",
         path_main=path_main,
         path_left="output",
diff --git a/shapepipe/utilities/summary.py b/shapepipe/utilities/summary.py
index ae574e5c4..8a1776905 100755
--- a/shapepipe/utilities/summary.py
+++ b/shapepipe/utilities/summary.py
@@ -15,7 +15,7 @@
 
 from tqdm import tqdm
 
-print("summaary v1.0")
+print("summaary v1.1")
 
 
 def get_IDs_from_file(path):
@@ -243,7 +243,7 @@ def print_intro(self):
         Print header line for job statistics.
 
         """
-        logging.info(f" (Job {self._bit})")
+        logging.info(f" # Job {self._bit}:")
 
     @classmethod
     def print_stats_header(self):
@@ -649,6 +649,9 @@ def check_numbers(self, par_runtime=None, indices=None):
                 n_missing_job += n_missing
                 self._missing_IDs_job.extend(missing_IDs)
 
+        # Empty line after job
+        logging.info("")
+
         # Write missing IDs for entire job to file
         #if n_missing_job > 0:
         self.output_missing_job()

From 25ef4e22bdb1abb2d7c29213dd24bd8768b71815 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Mar 2024 07:03:22 +0000
Subject: [PATCH 129/133] Removed rarely used ngmix njob/ngal options; splits
 computed automatically

---
 scripts/sh/job_sp_canfar.bash | 27 +++------------------------
 1 file changed, 3 insertions(+), 24 deletions(-)

diff --git a/scripts/sh/job_sp_canfar.bash b/scripts/sh/job_sp_canfar.bash
index f2b9c03d1..3ba748c69 100755
--- a/scripts/sh/job_sp_canfar.bash
+++ b/scripts/sh/job_sp_canfar.bash
@@ -20,8 +20,6 @@ star_cat_for_mask='onthefly'
 exclusive=''
 results='cosmostat/kilbinger/results_v2'
 n_smp=-1
-nsh_step=-1
-nsh_max=-1
 nsh_jobs=8
 
 ## Help string
@@ -53,14 +51,9 @@ usage="Usage: $(basename "$0") [OPTIONS] [TILE_ID]
    \toutput (upload) directory on vos:cfis, default='$results'\n
    -n, --n_smp N_SMP\n
    \tnumber of jobs (SMP mode only), default from original config files\n
-   --nsh_step NSTEP\n
-   \tnumber of shape measurement parallel jobs, default=$nsh_jobs\n
    --nsh_jobs NJOB\n
    \tnumber of objects per parallel shape module call, \n
    \tdefault: optimal number is computed\n
-   --nsh_max NMAX\n
-   \tmax number of objects per parallel shape module call, \n
-   \tdefault: unlimited; has precedent over --nsh_step\n
    TILE_ID_i\n
    \ttile ID(s), e.g. 283.247 214.242\n
 "
@@ -110,14 +103,6 @@ while [ $# -gt 0 ]; do
       n_smp="$2"
       shift
       ;;
-    --nsh_max)
-      nsh_max="$2"
-      shift
-      ;;
-    --nsh_step)
-      nsh_step="$2"
-      shift
-      ;;
     --nsh_jobs)
       nsh_jobs="$2"
       shift
@@ -142,10 +127,6 @@ if [ "$retrieve" != "vos" ] && [ "$retrieve" != "symlink" ]; then
   exit 5
 fi
 
-if [ $nsh_max != -1 ]; then
-  nsh_step=$nsh_max
-fi
-
 # For tar archives. TODO: Should be unique to each job
 export ID="test"
 
@@ -460,10 +441,8 @@ if [[ $do_job != 0 ]]; then
 
   ### Prepare config files
   n_min=0
-  if [[ $nsh_step == -1 ]]; then
-    n_obj=`get_number_objects`
-    nsh_step=`echo "$(($n_obj/$nsh_jobs))"`
-  fi
+  n_obj=`get_number_objects`
+  nsh_step=`echo "$(($n_obj/$nsh_jobs))"`
 
   n_max=$((nsh_step - 1))
   for k in $(seq 1 $nsh_jobs); do
@@ -472,7 +451,7 @@ if [[ $do_job != 0 ]]; then
         's/(ID_OBJ_MIN =) X/$1 '$n_min'/; s/(ID_OBJ_MAX =) X/$1 '$n_max'/; s/NgXu/Ng'$k'u/; s/X_interp/'$psf'_interp/g; print' \
         > $SP_CONFIG_MOD/config_tile_Ng${k}u.ini
     n_min=$((n_min + nsh_step))
-    if [ "$k" == $((nsh_jobs - 1)) ] && [ $nsh_max == -1 ]; then
+    if [ "$k" == $((nsh_jobs - 1)) ];  then
       n_max=-1
     else
       n_max=$((n_min + nsh_step - 1))

From 75fb975f39cc7d05ec4cfca26d3a7ab5f67a0e94 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Mar 2024 07:04:10 +0000
Subject: [PATCH 130/133] canfar init + run job: removed prev Ms and Mc dirs
 for job 256

---
 scripts/sh/init_run_exclusive_canfar.sh | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/scripts/sh/init_run_exclusive_canfar.sh b/scripts/sh/init_run_exclusive_canfar.sh
index 65ecfe8c1..dbaac2444 100755
--- a/scripts/sh/init_run_exclusive_canfar.sh
+++ b/scripts/sh/init_run_exclusive_canfar.sh
@@ -181,9 +181,9 @@ if [[ $do_job != 0 ]]; then
     cd ${kind}_runs/$ID/output
 
     # Remove duplicate job-32 runs (tile detection)
-    n_16=`ls -rt1d run_sp_tile_Sx_* | wc -l`
-    if [ "$n_16" != "1" ]; then
-      n_remove="$(($n_16-1))"
+    n_32=`ls -rt1d run_sp_tile_Sx_* | wc -l`
+    if [ "$n_32" != "1" ]; then
+      n_remove="$(($n_32-1))"
       echo "removing $n_remove duplicate old job-32 runs"
       rm -rf `ls -rt1d run_sp_tile_Sx_* | head -$n_remove`
     fi
@@ -193,6 +193,15 @@ if [[ $do_job != 0 ]]; then
   fi
 fi
 
+(( do_job= $job & 256 ))
+if [[ $do_job != 0 ]]; then
+
+  # Remove previous runs of this job
+  rm -rf run_sp_Ms_20??_*
+  rm -rf run_sp_Mc_20??_*
+
+fi
+
 cd ..
 
 # Update log file

From 376bbf1b2337e8e746a7955b89a8167b68a577d2 Mon Sep 17 00:00:00 2001
From: Martin Kilbinger <martin.kilbinger@cea.fr>
Date: Wed, 13 Mar 2024 07:04:44 +0000
Subject: [PATCH 131/133] merge sep cats: error if ngmix #HDUs < 6

---
 .../modules/merge_sep_cats_package/merge_sep_cats.py   | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py b/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py
index d9200529e..4c82d1a8c 100644
--- a/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py
+++ b/shapepipe/modules/merge_sep_cats_package/merge_sep_cats.py
@@ -87,6 +87,16 @@ def process(self):
             cat0.open()
             list_ext_name = cat0.get_ext_name()
             list_col_name = cat0.get_col_names()
+
+
+            # Inupt ngmix files sometimes have not all sheared versions
+            # (HDUs 1 - 5 = 1M, 1P, 2M, 2P, NOSHEAR) due to IO errors
+            if len(list_ext_name) < 6:
+                raise IndexError(
+                    f"Input ngmix catalogue {input_file} has only"
+                    + f" {len(list_ext_name)} HDUs, required are 6"
+                )
+
             # MKDEBUG: Some input ngmix catalogues have multiple of 5 HDUs
             # if reprocessed and not deleted but appended
             if len(list_ext_name) > 6:

From 21deebc692952574b2641628d8ad4cb64cb72551 Mon Sep 17 00:00:00 2001
From: martinkilbinger <martin.kilbinger@cea.fr>
Date: Wed, 3 Apr 2024 09:29:32 +0200
Subject: [PATCH 132/133] env-dev updated to min versions on canfar (docker
 image SP 1.1)

---
 environment-dev.yml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/environment-dev.yml b/environment-dev.yml
index cd2977ab8..703c19172 100644
--- a/environment-dev.yml
+++ b/environment-dev.yml
@@ -8,29 +8,29 @@ name: shapepipe-dev
 channels:
   - conda-forge
 dependencies:
-  - python=3.11
-  - pip>=21.2.4
-  - astropy>=5.0
+  - python>=3.9.18
+  - pip>=23.3.1
+  - astropy>=5.1.1
   - autoconf>=2.71
-  - automake>=1.16
-  - cmake>=3.27
-  - galsim>=2.5
-  - joblib>=1.3
+  - automake>=1.16.5
+  - cmake>=3.27.7
+  - galsim>=2.5.1
+  - joblib>=1.3.2
   - libtool>=2.4
-  - matplotlib>=3.8
-  - numba>=0.57
+  - matplotlib>=3.8.1
+  - numba>=0.58.1
   - pandas>=2.1
-  - pyqt5-sip>=12.12
-  - pyqtgraph>=0.13
+  - pyqt5-sip>=12.13
+  - pyqtgraph>=0.13.3
   - reproject>=0.12
   - sqlitedict>=2.1
   - termcolor>=2.3
-  - tqdm>=4.66
-  - treecorr>=4.3
+  - tqdm>=4.66.1
+  - treecorr>=4.3.3
   - pip:
     - cs_util==0.0.5
-    - mccd==1.2.3
-    - modopt==1.6.0
+    - mccd==1.2.4
+    - modopt==1.6.1
     - sip_tpv==1.1
     - sf_tools==2.0.4
     - git+https://github.com/CEA-COSMIC/pysap@develop

From f2a5a069497fb071e8489452ebd52d8fc9cdf55c Mon Sep 17 00:00:00 2001
From: martinkilbinger <martinkilbinger@cea.fr>
Date: Wed, 10 Apr 2024 16:32:13 +0200
Subject: [PATCH 133/133] minor changes

---
 install_shapepipe | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/install_shapepipe b/install_shapepipe
index e169f4859..85deab531 100755
--- a/install_shapepipe
+++ b/install_shapepipe
@@ -119,7 +119,7 @@ check_conda() {
   CONDA_VERSION_MAJOR=$(cut -d'.' -f1 <<<$CONDA_VERSION)
   CONDA_VERSION_MINOR=$(cut -d'.' -f2 <<<$CONDA_VERSION)
   CONDA_VERSION_PATCH=$(cut -d'.' -f3 <<<$CONDA_VERSION)
-  CONDA_SH=/opt/conda/etc/profile.d/conda.sh
+  CONDA_SH=/etc/profile.d/conda.sh
   # Check Conda major version
   if [ "$CONDA_VERSION_MAJOR" -lt 4 ]
   then
@@ -127,14 +127,14 @@ check_conda() {
     exit 1
   fi
   # Look for conda.sh file
-  if [ -f "$CONDA_PREFIX_1$CONDA_SH" ]
+  if [ -f "/opt/conda/$CONDA_SH" ]
   then
-    source "$CONDA_PREFIX_1$CONDA_SH"
+    source "/opt/conda/$CONDA_SH"
   elif [ -f "$CONDA_PREFIX$CONDA_SH" ]
   then
     source "$CONDA_PREFIX$CONDA_SH"
   else
-    echo -ne "${RED}ERROR: Could not find $CONDA_SH in \$CONDA_PREFIX.${NC}\n"
+    echo -ne "${RED}ERROR: Could not find $CONDA_SH in /opt/conda or \$CONDA_PREFIX.${NC}\n"
     echo -ne "${RED}Activate the base/root Conda environment and try again.${NC}\n"
     exit 1
   fi