[CI, enhancement] add sklearnex CI job with nightly oneDAL DPCPP buil…

…d in github actions (#1845) * Create build_oneDAL.yml * Create test.yml * Update build_oneDAL.yml * Update build_oneDAL.yml * Update test.yml * Update test.yml * Update test.yml * Update build_oneDAL.yml * Update test.yml * Update test.yml * Update test.yml * Update test.yml * Update test.yml * Update test_examples_sklearnex.py * Update __init__.py * Update __init__.py * stage updates * remove on pull_request * add white spaces * small changes * make conformant name change * updates to reflect dpcpp deactivation * forgotten variable * fix loose pytest calls * update yaml * updates for numpy2 issues * Update _ridge.py * Update ridge.py * formatting * Update ridge.py * fix __init__ * cleanup * add missing copyright * Update ci.yml * Update oneDAL.yml * Update deselected_tests.yaml * Update requirements-test.txt * Update ci.yml * re-orient to oneDAL repo artifacts * forgotten change * Update ci.yml * Update ci.yml * Update setup.py * Update version.py * Update version.py * Update setup.py * Update activate_components.bat * Update activate_components.bat * Update ci.yml * Update ci.yml * Update ci.yml * Update __init__.py * remove unnecessary workflow introduced in #1844 * changes requested by reviewer * camelCase CI job names
intel · Aug 29, 2024 · ba7e146 · ba7e146
1 parent 765b9ac
commit ba7e146
Show file tree

Hide file tree

Showing 13 changed files with 349 additions and 104 deletions.
diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml
@@ -57,7 +57,7 @@ jobs:
       pip install isort black[jupyter]==24.1.1
       isort --profile black --check . && black --check .
     displayName: 'Linting'
-- job: Linux
+- job: LinuxCondaEnv
   dependsOn: Lint
   timeoutInMinutes: 120
   strategy:
@@ -84,7 +84,7 @@ jobs:
     vmImage: 'ubuntu-22.04'
   steps:
   - template: build-and-test-lnx.yml
-- job: Windows
+- job: WindowsCondaEnv
   dependsOn: Lint
   timeoutInMinutes: 120
   strategy:

diff --git a/.ci/scripts/get_compatible_scipy_version.py b/.ci/scripts/get_compatible_scipy_version.py
@@ -14,10 +14,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
-from sys import exit, stderr
+from sys import argv, exit, stderr
 from sys import version_info as python_version
 
-from daal4py.sklearn._utils import sklearn_check_version
+if len(argv) == 1:
+    from daal4py.sklearn._utils import sklearn_check_version
+else:
+    # This is necessary to prevent import errors from sklearnex and by extension
+    # sklearn caused by initial scipy and/or numpy versions that are installed.
+    # It is written in a way to be `packaging` independent. This branch occurs
+    # when a sklearn version is given to the script externally.
+    def sklearn_check_version(ver):
+        ver = [int(i) if i != "" else 0 for i in ver.split(".")[:2]]
+        sk_ver = [int(i) if i != "" else 0 for i in str(argv[1]).split(".")[:2]]
+        return sk_ver[0] > ver[0] or (sk_ver[0] == ver[0] and sk_ver[1] >= ver[1])
+
 
 if sklearn_check_version("1.4"):
     print("Scipy version is not specified for this sklearn/python version.", file=stderr)

diff --git a/.ci/scripts/run_sklearn_tests.py b/.ci/scripts/run_sklearn_tests.py
@@ -43,7 +43,8 @@
         os.environ["SELECTED_TESTS"] = ""
 
     pytest_args = (
-        "--verbose --pyargs --durations=100 --durations-min=0.01 "
+        "--verbose --durations=100 --durations-min=0.01 "
+        f"--rootdir={os.path.dirname(sklearn.__file__)} "
         f'{os.environ["DESELECTED_TESTS"]} {os.environ["SELECTED_TESTS"]}'.split(" ")
     )
     while "" in pytest_args:

diff --git a/.circleci/deselect_tests.py b/.circleci/deselect_tests.py
@@ -123,7 +123,7 @@ def create_pytest_switches(
         pytest_switches = []
         for test_name in filtered_deselection:
             if test_name:
-                pytest_switches.extend(["--deselect", base_dir + test_name])
+                pytest_switches.extend(["--deselect=" + base_dir + test_name])
     return pytest_switches
 
 

diff --git a/.github/scripts/activate_components.bat b/.github/scripts/activate_components.bat
@@ -0,0 +1,29 @@
+@echo off
+rem ============================================================================
+rem Copyright 2024 Intel Corporation
+rem
+rem Licensed under the Apache License, Version 2.0 (the "License");
+rem you may not use this file except in compliance with the License.
+rem You may obtain a copy of the License at
+rem
+rem     http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+rem ============================================================================
+
+rem %1 - dpcpp compiler version
+rem %2 - tbb version
+rem %3 - dpcpp activate flag
+
+rem prepare vc
+call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
+rem prepare icx only if no parameter is given.
+if "%3"=="" call .\dpcpp\compiler\%1\env\vars.bat
+rem prepare tbb
+call .\dpcpp\tbb\%2\env\vars.bat
+rem prepare oneDAL
+call .\__release_win\daal\latest\env\vars.bat
diff --git a/.github/scripts/activate_components.sh b/.github/scripts/activate_components.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+#===============================================================================
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+# if any parameter is given, then do not source the compiler
+if [ $# -eq 0 ]; then source /opt/intel/oneapi/compiler/latest/env/vars.sh; fi # prepare icpx
+source /opt/intel/oneapi/tbb/latest/env/vars.sh # prepare tbb
+source ./__release_lnx/daal/latest/env/vars.sh # prepare oneDAL
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,260 @@
+#===============================================================================
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+
+name: CI
+on:
+  pull_request:
+    branches: [ "main" ]
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  TBB_VERSION: 2021.13
+  DPCPP_VERSION: 2024.2
+  DPCTL_VERSION: 0.17.0
+  DPNP_VERSION: 0.15.0
+  DPCTL_PY_VERSIONS: '3.9\|3.11'
+
+jobs:
+  sklearn_lnx:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - PYTHON_VERSION: "3.9"
+            SKLEARN_VERSION: "1.1"
+          - PYTHON_VERSION: "3.10"
+            SKLEARN_VERSION: "1.2"
+          - PYTHON_VERSION: "3.11"
+            SKLEARN_VERSION: "1.3"
+    name: LinuxNightly/pip Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.PYTHON_VERSION }}  
+      - name: Get run ID of "Nightly-build" workflow
+        id: get-run-id
+        run: |
+          OTHER_REPO="oneapi-src/oneDAL"
+          WF_NAME="Nightly-build"
+          RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId --status completed --jq .[0].databaseId`
+          echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}"
+          echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Download oneDAL build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: __release_lnx
+          github-token: ${{ github.token }}
+          repository: oneapi-src/oneDAL
+          run-id: ${{ steps.get-run-id.outputs.run-id }}
+          path: ./__release_lnx
+      - name: Set Environment Variables
+        id: set-env
+        run: |
+          # Disable SPMD testing
+          echo "NO_DIST=1" >> "$GITHUB_ENV"
+          # Determine dpctl support
+          DPCFLAG="0"
+          if [ $(echo ${{ matrix.PYTHON_VERSION }} | grep '${{ env.DPCTL_PY_VERSIONS }}') ]; then DPCFLAG=''; fi
+          echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT"
+      - name: apt-get
+        run: sudo apt-get update && sudo apt-get install -y clang-format
+      - name: dpcpp installation
+        run: bash .ci/scripts/install_dpcpp.sh
+      - name: describe system
+        run: bash .ci/scripts/describe_system.sh
+      - name: Install develop requirements
+        run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install -r dependencies-dev
+          pip list
+      - name: Build daal4py/sklearnex
+        run: |
+          source venv/bin/activate
+          source .github/scripts/activate_components.sh ${{ steps.set-env.outputs.DPCFLAG }}
+          python setup.py install --single-version-externally-managed --record=record.txt
+      - name: Install testing requirements
+        run: |
+          source venv/bin/activate
+          source .github/scripts/activate_components.sh
+          bash .ci/scripts/setup_sklearn.sh ${{ matrix.SKLEARN_VERSION }}
+          pip install --upgrade -r requirements-test.txt
+          pip install $(python .ci/scripts/get_compatible_scipy_version.py ${{ matrix.SKLEARN_VERSION }}) pyyaml
+          if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then pip install dpctl==${{ env.DPCTL_VERSION }} dpnp==${{ env.DPNP_VERSION }}; fi
+          pip list
+      - name: Sklearnex testing
+        run: |
+          source venv/bin/activate
+          source .github/scripts/activate_components.sh
+          cd .ci
+          ../conda-recipe/run_test.sh
+      - name: Sklearn testing
+        run: |
+          source venv/bin/activate
+          source .github/scripts/activate_components.sh
+          bash .ci/scripts/run_sklearn_tests.sh
+      - name: Sklearn testing [preview]
+        run: |
+          source venv/bin/activate
+          source .github/scripts/activate_components.sh
+          export SKLEARNEX_PREVIEW='YES'
+          bash .ci/scripts/run_sklearn_tests.sh
+
+  sklearn_win:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - PYTHON_VERSION: "3.9"
+            SKLEARN_VERSION: "1.1"
+          - PYTHON_VERSION: "3.10"
+            SKLEARN_VERSION: "1.2"
+          - PYTHON_VERSION: "3.11"
+            SKLEARN_VERSION: "1.3"
+    name: WindowsNightly/pip Python${{ matrix.PYTHON_VERSION }}_Sklearn${{ matrix.SKLEARN_VERSION }}
+    runs-on: windows-2022
+    timeout-minutes: 120
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.PYTHON_VERSION }}
+      - name: Get run ID of "Nightly-build" workflow
+        id: get-run-id
+        shell: bash
+        run: |
+          OTHER_REPO="oneapi-src/oneDAL"
+          WF_NAME="Nightly-build"
+          RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId --status completed --jq .[0].databaseId`
+          echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}"
+          echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Download oneDAL build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: __release_win
+          github-token: ${{ github.token }}
+          repository: oneapi-src/oneDAL
+          run-id: ${{ steps.get-run-id.outputs.run-id }}
+          path: .\__release_win
+      - name: Download DPCPP compiler artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: icx_compiler
+          github-token: ${{ github.token }}
+          repository: oneapi-src/oneDAL
+          run-id: ${{ steps.get-run-id.outputs.run-id }}
+          path: .
+      - name: Unzip Compiler
+        shell: cmd
+        run: |
+          tar -xvzf icx.zip
+          echo "Unzip complete"
+      - name: Set Environment Variables
+        id: set-env
+        shell: bash
+        run: |
+          echo "C:\msys64\usr\bin;C:\Program Files (x86)\Common Files\Intel\Shared Libraries\bin;" >> $GITHUB_PATH
+          # Disable SPMD testing
+          echo "NO_DIST=1" >> "$GITHUB_ENV"
+          # Determine dpctl support
+          DPCFLAG="0"
+          if [ $(echo ${{ matrix.PYTHON_VERSION }} | grep '${{ env.DPCTL_PY_VERSIONS }}') ]; then DPCFLAG=''; fi
+          echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT"
+      - name: Download Intel OpenCL CPU Runtime artifact
+        if: ${{ steps.set-env.outputs.DPCFLAG == '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: opencl_rt_installer
+          github-token: ${{ github.token }}
+          repository: oneapi-src/oneDAL
+          run-id: ${{ steps.get-run-id.outputs.run-id }}
+          path: .
+      - name: Install Intel OpenCL CPU Runtime
+        if: ${{ steps.set-env.outputs.DPCFLAG == '' }}
+        run: |
+          Start-Process msiexec -ArgumentList "/i opencl_rt.msi /qn" -Wait
+          echo "Installed Intel OpenCL CPU Runtime"
+      - name: Install dependencies
+        shell: cmd
+        run: |
+          python -m venv venv
+          call .\venv\Scripts\activate.bat
+          pip install --upgrade setuptools
+          pip install cpufeature clang-format pyyaml
+          pip install -r dependencies-dev
+      - name: System info 
+        shell: cmd
+        run: |
+          call .\venv\Scripts\activate.bat
+          call .\dpcpp\compiler\${{ env.DPCPP_VERSION }}\env\vars.bat
+          call .\dpcpp\compiler\${{ env.DPCPP_VERSION }}\bin\sycl-ls.exe
+          bash .ci/scripts/describe_system.sh
+      - name: Build daal4py/sklearnex
+        shell: cmd
+        run: |
+          call .\venv\Scripts\activate.bat
+          call .\.github\scripts\activate_components.bat ${{ env.DPCPP_VERSION }} ${{ env.TBB_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }}
+          set PREFIX=.
+          set PYTHON=python
+          call .\conda-recipe\bld.bat
+      - name: Install testing requirements
+        shell: cmd
+        run: |
+          call .\venv\Scripts\activate.bat
+          call .\.github\scripts\activate_components.bat ${{ env.DPCPP_VERSION }} ${{ env.TBB_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }}
+          bash .ci/scripts/setup_sklearn.sh ${{ matrix.SKLEARN_VERSION }}
+          pip install --upgrade -r requirements-test.txt
+          for /f "delims=" %%c in ('python .ci\scripts\get_compatible_scipy_version.py ${{ matrix.SKLEARN_VERSION }}') do set SCIPY_VERSION=%%c
+          pip install %SCIPY_VERSION%
+          if "${{ steps.set-env.outputs.DPCFLAG }}"=="" pip install dpctl==${{ env.DPCTL_VERSION }} dpnp==${{ env.DPNP_VERSION }}
+          pip list
+      - name: Sklearnex testing
+        shell: cmd
+        run: |
+          call .\venv\Scripts\activate.bat
+          call .\.github\scripts\activate_components.bat ${{ env.DPCPP_VERSION }} ${{ env.TBB_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }}
+          set PYTHON=python
+          cd ..
+          call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex
+      - name: Sklearn testing
+        shell: cmd
+        run: |
+          call .\venv\Scripts\activate.bat
+          call .\.github\scripts\activate_components.bat ${{ env.DPCPP_VERSION }} ${{ env.TBB_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }}
+          bash .ci/scripts/run_sklearn_tests.sh
+      - name: Sklearn testing [preview]
+        shell: cmd
+        run: |
+          call .\venv\Scripts\activate.bat
+          call .\.github\scripts\activate_components.bat ${{ env.DPCPP_VERSION }} ${{ env.TBB_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }}
+          set SKLEARNEX_PREVIEW=YES
+          bash .ci/scripts/run_sklearn_tests.sh