Add changes for AVX-512 support in k-NN. (#2110) #4304
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backwards Compatibility Tests k-NN | |
| on: | |
| push: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| paths: | |
| - 'build.gradle' | |
| - 'settings.gradle' | |
| - 'src/**' | |
| - 'build-tools/**' | |
| - 'buildSrc/**' | |
| - 'gradle/**' | |
| - 'jni/**' | |
| - 'qa/**' | |
| - '.github/workflows/backwards_compatibility_tests_workflow.yml' | |
| pull_request: | |
| branches: | |
| - "*" | |
| - "feature/**" | |
| paths: | |
| - 'build.gradle' | |
| - 'settings.gradle' | |
| - 'src/**' | |
| - 'build-tools/**' | |
| - 'buildSrc/**' | |
| - 'gradle/**' | |
| - 'jni/**' | |
| - 'qa/**' | |
| - '.github/workflows/backwards_compatibility_tests_workflow.yml' | |
| jobs: | |
| Restart-Upgrade-BWCTests-k-NN: | |
| strategy: | |
| matrix: | |
| java: [ 21 ] | |
| os: [ubuntu-latest] | |
| bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0", "2.18.0-SNAPSHOT"] | |
| opensearch_version : [ "3.0.0-SNAPSHOT" ] | |
| exclude: | |
| - os: windows-latest | |
| bwc_version: "2.0.1" | |
| - os: windows-latest | |
| bwc_version: "2.1.0" | |
| - os: windows-latest | |
| bwc_version: "2.2.1" | |
| - os: windows-latest | |
| bwc_version: "2.3.0" | |
| name: k-NN Restart-Upgrade BWC Tests | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| BWC_VERSION_RESTART_UPGRADE: ${{ matrix.bwc_version }} | |
| steps: | |
| - name: Checkout k-NN | |
| uses: actions/checkout@v1 | |
| # Setup git user so that patches for native libraries can be applied and committed | |
| - name: Setup git user | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - if: startsWith(matrix.os,'ubuntu') | |
| name: Install dependencies on ubuntu | |
| run: | | |
| sudo apt-get install libopenblas-dev gfortran -y | |
| - if: startsWith(matrix.os,'windows') | |
| name: Install MinGW Using Scoop on Windows | |
| run: | | |
| iex "& {$(irm get.scoop.sh)} -RunAsAdmin" | |
| scoop bucket add main | |
| scoop install mingw | |
| - if: startsWith(matrix.os,'windows') | |
| name: Add MinGW to PATH on Windows | |
| run: | | |
| echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH | |
| refreshenv | |
| - if: startsWith(matrix.os,'windows') | |
| name: Download OpenBLAS on Windows | |
| run: | | |
| curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip | |
| mkdir OpenBLAS | |
| Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ | |
| mkdir ./src/main/resources/windowsDependencies | |
| cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ | |
| rm .\OpenBLAS-0.3.21-x64.zip | |
| rm -r .\OpenBLAS\ | |
| - if: startsWith(matrix.os,'windows') | |
| name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows | |
| run: | | |
| echo "Running restart-upgrade backwards compatibility tests ..." | |
| ./gradlew :qa:restart-upgrade:testRestartUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' | |
| - if: startsWith(matrix.os,'ubuntu') | |
| name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu | |
| run: | | |
| echo "Running restart-upgrade backwards compatibility tests ..." | |
| if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw | |
| then | |
| echo "avx512 available on system" | |
| ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Dnproc.count=`nproc` | |
| elif lscpu | grep -i avx2 | |
| then | |
| echo "avx2 available on system" | |
| ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Dnproc.count=`nproc` -Davx512.enabled=false | |
| else | |
| echo "avx512 and avx2 not available on system" | |
| ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Davx2.enabled=false -Davx512.enabled=false -Dsimd.enabled=false -Dnproc.count=`nproc` | |
| fi | |
| Rolling-Upgrade-BWCTests-k-NN: | |
| strategy: | |
| matrix: | |
| java: [ 21 ] | |
| os: [ubuntu-latest] | |
| bwc_version: [ "2.18.0-SNAPSHOT" ] | |
| opensearch_version: [ "3.0.0-SNAPSHOT" ] | |
| name: k-NN Rolling-Upgrade BWC Tests | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }} | |
| steps: | |
| - name: Checkout k-NN | |
| uses: actions/checkout@v1 | |
| # Setup git user so that patches for native libraries can be applied and committed | |
| - name: Setup git user | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| - name: Setup Java ${{ matrix.java }} | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: ${{ matrix.java }} | |
| - if: startsWith(matrix.os,'ubuntu') | |
| name: Install dependencies on ubuntu | |
| run: | | |
| sudo apt-get install libopenblas-dev gfortran -y | |
| - if: startsWith(matrix.os,'windows') | |
| name: Install MinGW Using Scoop on Windows | |
| run: | | |
| iex "& {$(irm get.scoop.sh)} -RunAsAdmin" | |
| scoop bucket add main | |
| scoop install mingw | |
| - if: startsWith(matrix.os,'windows') | |
| name: Add MinGW to PATH on Windows | |
| run: | | |
| echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH | |
| refreshenv | |
| - if: startsWith(matrix.os,'windows') | |
| name: Download OpenBLAS on Windows | |
| run: | | |
| curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip | |
| mkdir OpenBLAS | |
| Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\ | |
| mkdir ./src/main/resources/windowsDependencies | |
| cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/ | |
| rm .\OpenBLAS-0.3.21-x64.zip | |
| rm -r .\OpenBLAS\ | |
| - if: startsWith(matrix.os,'windows') | |
| name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows | |
| run: | | |
| echo "Running rolling-upgrade backwards compatibility tests ..." | |
| ./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}' | |
| - if: startsWith(matrix.os,'ubuntu') | |
| name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu | |
| run: | | |
| echo "Running rolling-upgrade backwards compatibility tests ..." | |
| if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw | |
| then | |
| echo "avx512 available on system" | |
| ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Dnproc.count=`nproc` | |
| elif lscpu | grep -i avx2 | |
| then | |
| echo "avx2 available on system" | |
| ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Dnproc.count=`nproc` -Davx512.enabled=false | |
| else | |
| echo "avx512 and avx2 not available on system" | |
| ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Davx2.enabled=false -Davx512.enabled=false -Dsimd.enabled=false -Dnproc.count=`nproc` | |
| fi |