Add xsuite-kernels to wfs

szymonlopaciuk · Apr 25, 2024 · d7450bf · d7450bf
1 parent db150ca
commit d7450bf
Show file tree

Hide file tree

Showing 12 changed files with 110 additions and 123 deletions.
diff --git a/.github/scripts/install_branches.sh b/.github/scripts/install_branches.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+# copyright ############################### #
+# This file is part of the Xsuite project.  #
+# Copyright (c) CERN, 2024.                 #
+# ######################################### #
+set -xe
+
+repos=(xobjects xdeps xpart xtrack xfields xmask xcoll xsuite-kernels)
+prefix="${prefix:-.}"
+
+# Expects the following environment variables:
+# - $prefix, where to clone the packages
+# - ${repo}_branch, where $repo is one of the repos above (replace - with _)
+# - $precompile_kernels set to "true" or "false"
+
+for project in "${repos[@]}"; do
+  branch_varname="${project//-/_}_branch"
+  project_branch=${!branch_varname}  # get value of the variable [project]_branch
+
+  IFS=':' read -r -a parts <<< "$project_branch"
+  user="${parts[0]}"
+  branch="${parts[1]}"
+
+  cd "$prefix"
+  git clone \
+    --recursive \
+    --single-branch -b "$branch" \
+    "https://github.com/${user}/${project}.git"
+
+   if [ "$project" == "xsuite-kernels" ]; then
+      if [ "${precompile_kernels:-false}" == "false" ]; then
+          echo export SKIP_KERNEL_BUILD=1
+      fi
+      echo pip install --no-deps -e "${prefix}/${project}"
+  else
+      echo pip install -e "${prefix}/${project}[tests]"
+  fi
+done
diff --git a/.github/workflows/cron_test_gh.yaml b/.github/workflows/cron_test_gh.yaml
@@ -12,23 +12,9 @@ jobs:
   run-tests-cron-gh:
     uses: ./.github/workflows/test_gh.yaml
     with:
-      xobjects_location: 'xsuite:main'
-      xdeps_location: 'xsuite:main'
-      xpart_location: 'xsuite:main'
-      xtrack_location: 'xsuite:main'
-      xfields_location: 'xsuite:main'
-      xmask_location: 'xsuite:main'
-      xcoll_location: 'xsuite:main'
       xobjects_test_contexts: "ContextCpu"
   run-tests-cron-gh-precompiled:
     uses: ./.github/workflows/test_gh.yaml
     with:
-      xobjects_location: 'xsuite:main'
-      xdeps_location: 'xsuite:main'
-      xpart_location: 'xsuite:main'
-      xtrack_location: 'xsuite:main'
-      xfields_location: 'xsuite:main'
-      xmask_location: 'xsuite:main'
-      xcoll_location: 'xsuite:main'
       precompile_kernels: true
       xobjects_test_contexts: "ContextCpu"
diff --git a/.github/workflows/cron_test_gh_omp.yaml b/.github/workflows/cron_test_gh_omp.yaml
@@ -12,11 +12,4 @@ jobs:
   run-tests-cron-gh-omp:
     uses: ./.github/workflows/test_gh.yaml
     with:
-      xobjects_location: 'xsuite:main'
-      xdeps_location: 'xsuite:main'
-      xpart_location: 'xsuite:main'
-      xtrack_location: 'xsuite:main'
-      xfields_location: 'xsuite:main'
-      xmask_location: 'xsuite:main'
-      xcoll_location: 'xsuite:main'
       xobjects_test_contexts: "ContextCpu:auto"
diff --git a/.github/workflows/cron_test_gpu_cl.yaml b/.github/workflows/cron_test_gpu_cl.yaml
@@ -13,12 +13,5 @@ jobs:
     if: github.repository == 'xsuite/xsuite'
     uses: ./.github/workflows/test_sh.yaml
     with:
-      xobjects_location: 'xsuite:main'
-      xdeps_location: 'xsuite:main'
-      xpart_location: 'xsuite:main'
-      xtrack_location: 'xsuite:main'
-      xfields_location: 'xsuite:main'
-      xmask_location: 'xsuite:main'
-      xcoll_location: 'xsuite:main'
       test_contexts: 'ContextPyopencl'
       platform: 'alma'
diff --git a/.github/workflows/cron_test_gpu_cuda.yaml b/.github/workflows/cron_test_gpu_cuda.yaml
@@ -13,12 +13,5 @@ jobs:
     if: github.repository == 'xsuite/xsuite'
     uses: ./.github/workflows/test_sh.yaml
     with:
-      xobjects_location: 'xsuite:main'
-      xdeps_location: 'xsuite:main'
-      xpart_location: 'xsuite:main'
-      xtrack_location: 'xsuite:main'
-      xfields_location: 'xsuite:main'
-      xmask_location: 'xsuite:main'
-      xcoll_location: 'xsuite:main'
       test_contexts: 'ContextCupy'
       platform: 'ubuntu'
diff --git a/.github/workflows/cron_test_sh_cpu.yaml b/.github/workflows/cron_test_sh_cpu.yaml
@@ -13,41 +13,20 @@ jobs:
     if: github.repository == 'xsuite/xsuite'
     uses: ./.github/workflows/test_sh.yaml
     with:
-      xobjects_location: 'xsuite:main'
-      xdeps_location: 'xsuite:main'
-      xpart_location: 'xsuite:main'
-      xtrack_location: 'xsuite:main'
-      xfields_location: 'xsuite:main'
-      xmask_location: 'xsuite:main'
-      xcoll_location: 'xsuite:main'
       test_contexts: 'ContextCpu'
       platform: 'alma-cpu'
       suites: '["xobjects", "xdeps", "xpart", "xtrack", "xfields", "xcoll"]'
   run-tests-cron-sh-cpu-no-xmask-openmp:
     if: github.repository == 'xsuite/xsuite'
     uses: ./.github/workflows/test_sh.yaml
     with:
-      xobjects_location: 'xsuite:main'
-      xdeps_location: 'xsuite:main'
-      xpart_location: 'xsuite:main'
-      xtrack_location: 'xsuite:main'
-      xfields_location: 'xsuite:main'
-      xmask_location: 'xsuite:main'
-      xcoll_location: 'xsuite:main'
       test_contexts: 'ContextCpu:auto'
       platform: 'alma-cpu'
       suites: '["xobjects", "xdeps", "xpart", "xtrack", "xfields", "xcoll"]'
   run-tests-cron-sh-cpu-xmask:
     if: github.repository == 'xsuite/xsuite'
     uses: ./.github/workflows/test_sh.yaml
     with:
-      xobjects_location: 'xsuite:main'
-      xdeps_location: 'xsuite:main'
-      xpart_location: 'xsuite:main'
-      xtrack_location: 'xsuite:main'
-      xfields_location: 'xsuite:main'
-      xmask_location: 'xsuite:main'
-      xcoll_location: 'xsuite:main'
       test_contexts: 'ContextCpu;ContextCpu:auto'
       platform: 'alma-cpu'
       suites: '["xmask"]'
diff --git a/.github/workflows/manual_test_gh.yaml b/.github/workflows/manual_test_gh.yaml
@@ -36,6 +36,10 @@ on:
         description: 'xcoll branch'
         default: 'xsuite:main'
         required: true
+      xsuite_kernels_location:
+        description: 'xsuite-kernels branch'
+        default: 'xsuite:main'
+        required: true
       precompile_kernels:
         description: 'precompile kernels?'
         type: boolean
@@ -59,5 +63,6 @@ jobs:
       xfields_location: ${{ inputs.xfields_location }}
       xmask_location: ${{ inputs.xmask_location }}
       xcoll_location: ${{ inputs.xcoll_location }}
+      xsuite_kernels_location: ${{ inputs.xsuite_kernels_location }}
       precompile_kernels: ${{ inputs.precompile_kernels }}
       xobjects_test_contexts: ${{ inputs.xobjects_test_contexts }}
diff --git a/.github/workflows/manual_test_sh.yaml b/.github/workflows/manual_test_sh.yaml
@@ -37,6 +37,10 @@ on:
         description: 'xcoll branch'
         default: 'xsuite:main'
         required: true
+      xsuite_kernels_location:
+        description: 'xsuite-kernels branch'
+        default: 'xsuite:main'
+        required: true
       test_contexts:
         description: test contexts
         default: 'ContextCpu;ContextCpu:auto;ContextCupy;ContextPyopencl'
@@ -65,6 +69,7 @@ jobs:
       xfields_location: ${{ inputs.xfields_location }}
       xmask_location: ${{ inputs.xmask_location }}
       xcoll_location: ${{ inputs.xcoll_location }}
+      xsuite_kernels_location: ${{ inputs.xsuite_kernels_location }}
       test_contexts: ${{ inputs.test_contexts }}
       platform: ${{ inputs.platform }}
       suites: ${{ inputs.suites }}
diff --git a/.github/workflows/test_gh.yaml b/.github/workflows/test_gh.yaml
@@ -7,25 +7,36 @@ on:
     # Inputs the workflow accepts.
     inputs:
       xobjects_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xdeps_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xpart_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xtrack_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xfields_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xmask_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xcoll_location:
-        required: true
+        required: false
+        default: "xsuite:main"
+        type: string
+      xsuite_kernels_location:
+        required: false
+        default: "xsuite:main"
         type: string
       precompile_kernels:
         required: false
@@ -88,36 +99,14 @@ jobs:
         xfields_branch: ${{ inputs.xfields_location }}
         xmask_branch: ${{ inputs.xmask_location }}
         xcoll_branch: ${{ inputs.xcoll_location }}
+        xsuite_kernels: ${{ inputs.xsuite_kernels }}
+        precompile_kernels: ${{ inputs.precompile_kernels }}
       run: |
-        for project in xobjects xdeps xpart xtrack xfields xmask xcoll; do
-          branch_varname="${project}_branch"
-          project_branch=${!branch_varname}  # get value of the variable [project]_branch
-          
-          IFS=':' read -r -a parts <<< $project_branch
-          user="${parts[0]}"
-          branch="${parts[1]}"
-          
-          echo git clone \
-            --recursive \
-            --single-branch -b "$branch" \
-            "https://github.com/${user}/${project}.git"
-          
-          cd $GITHUB_WORKSPACE
-          git clone \
-            --recursive \
-            --single-branch -b "$branch" \
-            "https://github.com/${user}/${project}.git"
-          pip install -e "${GITHUB_WORKSPACE}/${project}[tests]"
-        done
-    - name: Checkout xsuite (needed to precompile kernels)
-      if: ${{ inputs.precompile_kernels == true }}
-      uses: actions/checkout@v3
-      with:
-        path: 'xsuite'
+        export prefix="$GITHUB_WORKSPACE"
+        bash $GITHUB_WORKSPACE/.github/scripts/install_branches.sh
     - name: Precompile kernels
       if: ${{ inputs.precompile_kernels == true }}
       run: |
-        SKIP_KERNEL_BUILD=1 pip install --no-deps xsuite-kernels
         xsuite-kernels regenerate
     - name: Print versions
       run: conda list

diff --git a/.github/workflows/test_sh.yaml b/.github/workflows/test_sh.yaml
@@ -7,25 +7,36 @@ on:
     # Inputs the workflow accepts.
     inputs:
       xobjects_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xdeps_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xpart_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xtrack_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xfields_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xmask_location:
-        required: true
+        required: false
+        default: "xsuite:main"
         type: string
       xcoll_location:
-        required: true
+        required: false
+        default: "xsuite:main"
+        type: string
+      xsuite_kernels_location:
+        required: false
+        default: "xsuite:main"
         type: string
       test_contexts:
         required: false
@@ -66,6 +77,7 @@ jobs:
           xfields_branch: ${{ inputs.xfields_location }}
           xmask_branch: ${{ inputs.xmask_location }}
           xcoll_branch: ${{ inputs.xcoll_location }}
+          xsuite_kernels_branch: ${{ xsuite_kernels_location }}
         run: |
           IMAGE="xsuite-test-runner-$(cat /proc/sys/kernel/random/uuid)"
           echo "image_id=$IMAGE" >> $GITHUB_OUTPUT
@@ -79,6 +91,7 @@ jobs:
             --build-arg xfields_branch=${xfields_branch} \
             --build-arg xmask_branch=${xmask_branch} \
             --build-arg xcoll_branch=${xcoll_branch} \
+            --build_arg xsuite_kernels_branch=${xsuite_kernels_branch} \
             --build-arg with_gpu=${with_gpu} \
             -t $IMAGE .
 
@@ -129,7 +142,7 @@ jobs:
     if: success() || failure()
     steps:
       - name: Wait
-        run: sleep 1800
+        run: sleep 300
 
   # Cleanup after the tests by removing the image and making sure there are
   # no unused images and stopped containers

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-# This is a test runner Dockerfile. Specific branches used to
+# This is a test runner Dockerfile. The branches used to
 # build our image can be specified using the --build-arg's below.
 FROM cern/alma8-base:latest
 LABEL author="Szymon Lopaciuk <szymon@lopaciuk.eu>"
@@ -14,6 +14,7 @@ ARG xtrack_branch=xsuite:main
 ARG xfields_branch=xsuite:main
 ARG xmask_branch=xsuite:main
 ARG xcoll_branch=xsuite:main
+ARG xsuite_kernels_branch=xsuite:main
 ARG with_gpu
 
 # Use bash as the default shell
@@ -76,21 +77,8 @@ RUN if [[ "$with_gpu" == true ]]; then \
 
 # Install all the Xsuite packages in the required versions
 WORKDIR /opt/xsuite
-RUN for project in xobjects xdeps xpart xtrack xfields xmask xcoll; do \
-        branch_varname="${project}_branch" \
-        && project_branch=${!branch_varname} \
-        && IFS=':' read -r -a parts <<< $project_branch \
-        && user="${parts[0]}" \
-        && branch="${parts[1]}" \
-        && git clone \
-        --recursive \
-        --depth 1 \
-        --single-branch -b "$branch" \
-        "https://github.com/${user}/${project}.git" \
-        && pip install -e ${project}[tests] \
-        || break ; \
-    done; \
-    pip cache purge
+COPY .github/scripts/install_branches.sh /opt/xsuite/
+RUN bash /opt/xsuite/install_branches.sh && pip cache purge
 
 # Copy the test runner script into the image
 WORKDIR /opt