Merge branch 'main' into feat/identity

.github/workflows/compiler_benchmark.yml

@@ -83,11 +83,8 @@ jobs:
           echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}"
           echo "CUDACXX=$CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}"
 
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
 
       - name: Build compiler benchmarks
         run: |

.github/workflows/compiler_build_and_test_cpu.yml

@@ -63,6 +63,9 @@ jobs:
           submodules: recursive
           token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
 
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
+
       - name: Create build dir
         run: mkdir build
 
@@ -83,6 +86,7 @@ jobs:
             ${{ env.DOCKER_GPU_OPTION }}
           shell: bash
           run: |
+            rustup toolchain install nightly-2024-01-31
             set -e
             cd /concrete/compilers/concrete-compiler/compiler
             rm -rf /build/*
@@ -136,6 +140,7 @@ jobs:
           shell: bash
           run: |
             set -e
+            rustup toolchain install nightly-2024-01-31
             cd /concrete/compilers/concrete-compiler/compiler
             pip install pytest
             dnf install -y libzstd libzstd-devel

.github/workflows/compiler_build_and_test_gpu.yml

@@ -28,7 +28,7 @@ on:
 
 env:
   DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
-  CUDA_PATH: /usr/local/cuda-11.7
+  CUDA_PATH: /usr/local/cuda-12.3
   GCC_VERSION: 11
 
 jobs:
@@ -63,11 +63,8 @@ jobs:
           submodules: recursive
           token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
 
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
 
       - name: Create build dir
         run: mkdir build
@@ -89,6 +86,7 @@ jobs:
             --gpus all
           shell: bash
           run: |
+            rustup toolchain install nightly-2024-01-31
             set -e
             cd /concrete/compilers/concrete-compiler/compiler
             rm -rf /build/*

.github/workflows/compiler_macos_build_and_test.yml

@@ -34,11 +34,8 @@ jobs:
           submodules: recursive
           token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
 
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
 
       - name: Install Deps
         run: |

.github/workflows/compiler_publish_docker_images.yml

@@ -116,7 +116,7 @@ jobs:
     runs-on: ${{ github.event.inputs.runner_name }}
     env:
       IMAGE: ghcr.io/zama-ai/cuda
-      TAG: "11-7"
+      TAG: "12-3"
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/concrete_cpu_test.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Rust install
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly
+          toolchain: nightly-2024-01-31
           override: true
           components: rustfmt, clippy
 
@@ -65,11 +65,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Rust install
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
 
       - name: Download cargo cache
         uses: Swatinem/rust-cache@v2
@@ -86,11 +83,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Rust install
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
 
       - name: Download cargo cache
         uses: Swatinem/rust-cache@v2

.github/workflows/concrete_python_release.yml

@@ -98,6 +98,7 @@ jobs:
           shell: bash
           run: |
             set -e
+            rustup toolchain install nightly-2024-01-31
             rm -rf /build/*
             
             export PYTHON=${{ format('python{0}', matrix.python-version) }}
@@ -145,7 +146,8 @@ jobs:
           python -m venv .testenv
           source .testenv/bin/activate
           
-          sudo apt install graphviz graphviz-devel
+          sudo apt update -y
+          sudo apt install -y graphviz libgraphviz-dev
           
           pip install $CONCRETE_PYTHON/dist/*manylinux*.whl
           pip install -r $CONCRETE_PYTHON/requirements.dev.txt
@@ -191,11 +193,8 @@ jobs:
         run: |
           brew install ninja ccache graphviz
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
 
       - name: Set release version (nightly)
         if: ${{ env.RELEASE_TYPE == 'nightly' }}
@@ -265,6 +264,7 @@ jobs:
           export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python
 
           export TEST_TMP_DIR=$(mktemp -d)
+          echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}"
           cd $TEST_TMP_DIR
 
           export PYTHON_VERSION=${{ format('python{0}', matrix.python-version) }}
@@ -289,22 +289,16 @@ jobs:
           # we fix it by using a single one (from torch)
           # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8
           export SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/
-          export CONCRETE_LIB_OMP=$SITE_PACKAGES/concrete/.dylibs/libomp.dylib
-          # Where is libiomp? Possibilities ordered in increasing priorities
-          export TORCH_LIB_IOMP="this shouldn't exist"
-          ls $SITE_PACKAGES/torch/lib/libiomp5.dylib && export TORCH_LIB_IOMP=$SITE_PACKAGES/torch/lib/libiomp5.dylib
-          ls $SITE_PACKAGES/torch/.dylibs/libiomp5.dylib && export TORCH_LIB_IOMP=$SITE_PACKAGES/torch/.dylibs/libiomp5.dylib
-          ls $SITE_PACKAGES/functorch/.dylibs/libiomp5.dylib && export TORCH_LIB_IOMP=$SITE_PACKAGES/functorch/.dylibs/libiomp5.dylib
-          ls $TORCH_LIB_IOMP \
-          && rm $CONCRETE_LIB_OMP \
-          && ln -s $TORCH_LIB_IOMP $CONCRETE_LIB_OMP
+          find $SITE_PACKAGES/site-packages -not \( -path $SITE_PACKAGES/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1  ln -f -s $SITE_PACKAGES/concrete/.dylibs/libomp.dylib
 
           cp -R $CONCRETE_PYTHON/tests .
           cp -R $CONCRETE_PYTHON/pytest.ini .
 
-          pytest tests -svv -n auto -m "not dataflow" --key-cache /tmp/KeySetCache
-          rm -rf /tmp/KeySetCache
+          pytest tests -svv -n auto -m "not dataflow" --key-cache "./KeySetCache"
 
+      - name: Cleanup host
+        if: success() || failure()
+        run: |
           rm -rf $TEST_TMP_DIR
 
   push:

.github/workflows/concrete_python_test_macos.yml

@@ -36,11 +36,8 @@ jobs:
         run: |
           brew install ninja ccache graphviz
 
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
 
       - name: Cache Compilation (push)
         if: github.event_name == 'push'
@@ -123,7 +120,7 @@ jobs:
           # we fix it by using a single one (from torch)
           # see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8
           
-          find .testenv/lib/python3.10/site-packages -name 'lib*omp5.dylib' | xargs -n 1  ln -f -s $(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib
+          find .testenv/lib/python3.10/site-packages -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1  ln -f -s $(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib
 
           cp -R $GITHUB_WORKSPACE/frontends/concrete-python/tests ./tests
           pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow"

.github/workflows/concrete_python_tests_linux.yml

@@ -25,7 +25,7 @@ on:
 
 env:
   DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
-  CUDA_PATH: /usr/local/cuda-11.7
+  CUDA_PATH: /usr/local/cuda-12.3
   GCC_VERSION: 11
   GLIB_VER: 2_28
 
@@ -62,6 +62,9 @@ jobs:
       - name: Create build directory
         run: mkdir build
 
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
+
       - name: Build bindings
         uses: addnab/docker-run-action@v3
         if: ${{ !contains(inputs.instance_type, 'p3') }}
@@ -78,6 +81,7 @@ jobs:
             -e SSH_AUTH_SOCK=/ssh.socket
           shell: bash
           run: |
+            rustup toolchain install nightly-2024-01-31
             set -e
             rm -rf /build/*
 

.github/workflows/main.yml

@@ -7,8 +7,9 @@ on:
     branches:
       - 'main'
       - 'release/*'
+      - 'force-docker-images'
     tags:
-      - 'v[0-9]+\.[0-9]+\.[0-9]+(-.*)?'
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
   schedule:
     # Nightly Release @ 3AM after each work day
     - cron: "0 3 * * 2-6"
@@ -98,7 +99,9 @@ jobs:
         id: concrete-compiler-docker-images-workflow
         uses: tj-actions/changed-files@0c6f2ac4885216b1830ecbcc3573209980c97716
         with:
-          files: ./.github/workflows/compiler_publish_docker_images.yml
+          files: |
+            ./.github/workflows/compiler_publish_docker_images.yml
+            ./docker/**
 
       - name: Check if concrete_cpu_test workflow has changed
         id: concrete-cpu-workflow
@@ -132,41 +135,41 @@ jobs:
 
       - name: Set some github event outputs
         id: github
-        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'release'))
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release'))
         run: echo "push-main=true" >> "$GITHUB_OUTPUT"
 
 #################################################
 # Compiler jobs #################################
   compiler-compliance:
     needs: file-change
-    if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-format-and-linting-workflow == 'true' || needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/tags/')
+    if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-format-and-linting-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
     uses: ./.github/workflows/compiler_format_and_linting.yml
 
   compiler-cpu-build:
     needs: file-change
-    if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cpu-api == 'true'|| needs.file-change.outputs.concrete-compiler-cpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/tags/')
+    if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cpu-api == 'true'|| needs.file-change.outputs.concrete-compiler-cpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
     uses: ./.github/workflows/start_slab.yml
     secrets: inherit
     with:
       command: compiler-cpu-build
 
   compiler-gpu-build:
     needs: file-change
-    if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cuda-api == 'true' || needs.file-change.outputs.concrete-compiler-gpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/tags/')
+    if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cuda-api == 'true' || needs.file-change.outputs.concrete-compiler-gpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
     uses: ./.github/workflows/start_slab.yml
     secrets: inherit
     with:
       command: compiler-gpu-build
 
   compiler-macos-tests:
     needs: file-change
-    if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-macos-workflow == 'true' || needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/tags/')
+    if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-macos-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
     uses: ./.github/workflows/compiler_macos_build_and_test.yml
     secrets: inherit
 
   compiler-publish-docker-images:
     needs: file-change
-    if: (needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-docker-images-workflow == 'true') && (needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/tags/'))
+    if: (needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-docker-images-workflow == 'true') && (needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/heads/force-docker-images'))
     uses: ./.github/workflows/start_slab.yml
     secrets: inherit
     with:
@@ -212,29 +215,29 @@ jobs:
 # Concrete Python jobs ##########################
   concrete-python:
     needs: file-change
-    if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.concrete-python-workflow == 'true' || needs.file-change.outputs.push-main || contains(github.ref, 'refs/tags/')
+    if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.concrete-python-workflow == 'true' || needs.file-change.outputs.push-main
     uses: ./.github/workflows/concrete_python_checks.yml
     secrets: inherit
 
   concrete-python-tests-linux:
     needs: file-change
-    if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main || contains(github.ref, 'refs/tags/')
+    if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main
     uses: ./.github/workflows/start_slab.yml
     secrets: inherit
     with:
       command: concrete-python-tests-linux
 
   concrete-python-tests-linux-gpu:
     needs: file-change
-    if: (needs.file-change.outputs.concrete-python == 'true' || contains(github.ref, 'refs/tags/')) && needs.file-change.outputs.push-main
+    if: needs.file-change.outputs.concrete-python == 'true' && needs.file-change.outputs.push-main
     uses: ./.github/workflows/start_slab.yml
     secrets: inherit
     with:
       command: concrete-python-tests-linux-gpu
 
   concrete-python-tests-macos:
     needs: file-change
-    if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main || contains(github.ref, 'refs/tags/')
+    if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main
     uses: ./.github/workflows/concrete_python_test_macos.yml
     secrets: inherit
 

.github/workflows/ml_benchmark_subset.yml

@@ -63,11 +63,8 @@ jobs:
         run: |
           echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
 
-      - name: Install rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
+      - name: Setup rust toolchain for concrete-cpu
+        uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
 
       - name: Build compiler and ML benchmarks
         run: |

.github/workflows/optimizer.yml

@@ -100,31 +100,3 @@ jobs:
         with:
           path: ./benchmark
           name: ${{ runner.os }}-benchmark
-
-  nightly-check:
-    runs-on: ubuntu-20.04
-    env:
-      RUSTFLAGS: -D warnings
-    steps:
-      - uses: actions/checkout@v3
-      - name: Rust install
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
-          components: rustfmt, clippy
-      # A SSH private key is required as some dependencies are from private repos
-      - name: Set ssh keys
-        uses: webfactory/ssh-agent@v0.7.0
-        with:
-          ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
-      - name: Formatting
-        run: |
-          cd compilers/concrete-optimizer
-          cargo fmt --check
-
-      - name: Lint
-        run: |
-          cd compilers/concrete-optimizer
-          cargo clippy --release --all-targets
-