From f61e2d13e303c828abe92419bd522d6e50672fbb Mon Sep 17 00:00:00 2001 From: nihui Date: Sun, 7 Apr 2024 11:37:48 +0800 Subject: [PATCH] unified python ci (#5407) --- .../workflows/linux-x64-cpu-clang-python.yml | 68 --------- .github/workflows/macos-x64-cpu-python.yml | 133 ------------------ ...ux-x64-gpu-clang-python.yml => python.yml} | 71 +++++++--- .../windows-x64-cpu-vs2019-python.yml | 67 --------- 4 files changed, 48 insertions(+), 291 deletions(-) delete mode 100644 .github/workflows/linux-x64-cpu-clang-python.yml delete mode 100644 .github/workflows/macos-x64-cpu-python.yml rename .github/workflows/{linux-x64-gpu-clang-python.yml => python.yml} (55%) delete mode 100644 .github/workflows/windows-x64-cpu-vs2019-python.yml diff --git a/.github/workflows/linux-x64-cpu-clang-python.yml b/.github/workflows/linux-x64-cpu-clang-python.yml deleted file mode 100644 index 8e6f6718f2f..00000000000 --- a/.github/workflows/linux-x64-cpu-clang-python.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: linux-x64-cpu-clang-python -on: - push: - branches: [master] - paths: - - '.github/workflows/linux-x64-cpu-clang-python.yml' - - 'CMakeLists.txt' - - 'cmake/**' - - 'src/*' - - 'src/layer/*' - - 'src/layer/x86/**' - - 'python/**' - pull_request: - branches: [master] - paths: - - '.github/workflows/linux-x64-cpu-clang-python.yml' - - 'CMakeLists.txt' - - 'cmake/**' - - 'src/*' - - 'src/layer/*' - - 'src/layer/x86/**' - - 'python/**' -concurrency: - group: linux-x64-cpu-clang-python-${{ github.ref }} - cancel-in-progress: true -permissions: - contents: read - -jobs: - linux-clang-python: - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.7, 3.9, 3.11] - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: set up python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest setuptools wheel twine - - name: configure - env: - CC: clang - CXX: clang++ - run: mkdir build && cd build && cmake -DNCNN_PYTHON=ON -DNCNN_DISABLE_RTTI=OFF -DNCNN_DISABLE_EXCEPTION=OFF -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF .. - - name: build - run: cmake --build build -j $(nproc) - - name: install python - run: cd python && pip install . - - name: test - run: cd python && pytest tests - - name: build and publish - if: startsWith(github.ref, 'refs/tags') - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/" - run: | - cd python - python setup.py bdist_wheel - twine upload dist/* diff --git a/.github/workflows/macos-x64-cpu-python.yml b/.github/workflows/macos-x64-cpu-python.yml deleted file mode 100644 index 6d048826064..00000000000 --- a/.github/workflows/macos-x64-cpu-python.yml +++ /dev/null @@ -1,133 +0,0 @@ -name: macos-x64-cpu-python -on: - push: - branches: [master] - paths: - - '.github/workflows/macos-x64-cpu-python.yml' - - 'CMakeLists.txt' - - 'cmake/**' - - 'src/*' - - 'src/layer/*' - - 'src/layer/x86/**' - - 'python/**' - pull_request: - branches: [master] - paths: - - '.github/workflows/macos-x64-cpu-python.yml' - - 'CMakeLists.txt' - - 'cmake/**' - - 'src/*' - - 'src/layer/*' - - 'src/layer/x86/**' - - 'python/**' -concurrency: - group: macos-x64-cpu-python-${{ github.ref }} - cancel-in-progress: true -env: - DEVELOPER_DIR: /Applications/Xcode_13.4.1.app/Contents/Developer - MAC_DEPLOYMENT_TARGET: '10.9' - MAC_ARM64_DEPLOYMENT_TARGET: '11.0' - ENABLE_BITCODE: OFF - ENABLE_ARC: OFF - ENABLE_VISIBILITY: OFF -permissions: - contents: read - -jobs: - macos-clang: - runs-on: macos-12 - strategy: - matrix: - python-version: [3.7, 3.9, 3.11] - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: set up python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest setuptools wheel twine - - - name: cache-openmp - id: cache-openmp - uses: actions/cache@v4 - with: - path: openmp-install - key: openmp-macos-install-20230504 - - name: openmp - if: steps.cache-openmp.outputs.cache-hit != 'true' - run: | - wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/openmp-11.0.0.src.tar.xz - tar -xf openmp-11.0.0.src.tar.xz - cd openmp-11.0.0.src - sed -i'' -e '/.size __kmp_unnamed_critical_addr/d' runtime/src/z_Linux_asm.S - sed -i'' -e 's/__kmp_unnamed_critical_addr/___kmp_unnamed_critical_addr/g' runtime/src/z_Linux_asm.S - - name: openmp-build-x86_64 - if: steps.cache-openmp.outputs.cache-hit != 'true' - run: | - cd openmp-11.0.0.src - mkdir -p build-x86_64 && cd build-x86_64 - cmake -DCMAKE_TOOLCHAIN_FILE=../../toolchains/ios.toolchain.cmake -DPLATFORM=MAC -DARCHS="x86_64" \ - -DDEPLOYMENT_TARGET=$MAC_DEPLOYMENT_TARGET -DENABLE_BITCODE=$ENABLE_BITCODE -DENABLE_ARC=$ENABLE_ARC -DENABLE_VISIBILITY=$ENABLE_VISIBILITY \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install \ - -DPERL_EXECUTABLE=/usr/local/bin/perl \ - -DLIBOMP_ENABLE_SHARED=OFF -DLIBOMP_OMPT_SUPPORT=OFF -DLIBOMP_USE_HWLOC=OFF .. - cmake --build . -j 3 - cmake --build . --target install - - name: openmp-build-arm64 - if: steps.cache-openmp.outputs.cache-hit != 'true' - run: | - cd openmp-11.0.0.src - mkdir -p build-arm64 && cd build-arm64 - cmake -DCMAKE_TOOLCHAIN_FILE=../../toolchains/ios.toolchain.cmake -DPLATFORM=MAC_ARM64 -DARCHS="arm64" \ - -DDEPLOYMENT_TARGET=$MAC_ARM64_DEPLOYMENT_TARGET -DENABLE_BITCODE=$ENABLE_BITCODE -DENABLE_ARC=$ENABLE_ARC -DENABLE_VISIBILITY=$ENABLE_VISIBILITY \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install \ - -DPERL_EXECUTABLE=/usr/local/bin/perl \ - -DLIBOMP_ENABLE_SHARED=OFF -DLIBOMP_OMPT_SUPPORT=OFF -DLIBOMP_USE_HWLOC=OFF .. - cmake --build . -j 3 - cmake --build . --target install - - name: openmp-merge-fat-library - if: steps.cache-openmp.outputs.cache-hit != 'true' - run: | - mkdir -p $GITHUB_WORKSPACE/openmp-install - cp -a openmp-11.0.0.src/build-x86_64/install/include $GITHUB_WORKSPACE/openmp-install - mkdir -p $GITHUB_WORKSPACE/openmp-install/lib - lipo -create \ - openmp-11.0.0.src/build-x86_64/install/lib/libomp.a \ - openmp-11.0.0.src/build-arm64/install/lib/libomp.a \ - -o $GITHUB_WORKSPACE/openmp-install/lib/libomp.a - - - name: install-openmp - run: | - sudo cp $GITHUB_WORKSPACE/openmp-install/include/* $DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include - sudo cp $GITHUB_WORKSPACE/openmp-install/lib/libomp.a $DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/lib - - - name: build - run: | - mkdir build && cd build - cmake -DCMAKE_TOOLCHAIN_FILE=../toolchains/ios.toolchain.cmake -DPLATFORM=MAC -DARCHS="x86_64" \ - -DDEPLOYMENT_TARGET=$MAC_DEPLOYMENT_TARGET -DENABLE_BITCODE=$ENABLE_BITCODE -DENABLE_ARC=$ENABLE_ARC -DENABLE_VISIBILITY=$ENABLE_VISIBILITY \ - -DOpenMP_C_FLAGS="-Xclang -fopenmp" -DOpenMP_CXX_FLAGS="-Xclang -fopenmp" \ - -DOpenMP_C_LIB_NAMES="libomp" -DOpenMP_CXX_LIB_NAMES="libomp" \ - -DOpenMP_libomp_LIBRARY="$DEVELOPER_DIR/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/lib/libomp.a" \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=install \ - -DNCNN_PYTHON=ON -DNCNN_DISABLE_RTTI=OFF -DNCNN_DISABLE_EXCEPTION=OFF -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF .. - cmake --build . -j 3 - - name: install python - run: cd python && pip install . - - name: test - run: cd python && pytest tests - - name: build and publish - if: startsWith(github.ref, 'refs/tags') - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/" - run: | - cd python - python setup.py bdist_wheel - twine upload dist/* diff --git a/.github/workflows/linux-x64-gpu-clang-python.yml b/.github/workflows/python.yml similarity index 55% rename from .github/workflows/linux-x64-gpu-clang-python.yml rename to .github/workflows/python.yml index ea9232bcfc4..3e793f618cf 100644 --- a/.github/workflows/linux-x64-gpu-clang-python.yml +++ b/.github/workflows/python.yml @@ -1,9 +1,9 @@ -name: linux-x64-gpu-clang-python +name: python on: push: branches: [master] paths: - - '.github/workflows/linux-x64-gpu-clang-python.yml' + - '.github/workflows/python.yml' - 'CMakeLists.txt' - 'cmake/**' - 'src/*' @@ -14,7 +14,7 @@ on: pull_request: branches: [master] paths: - - '.github/workflows/linux-x64-gpu-clang-python.yml' + - '.github/workflows/python.yml' - 'CMakeLists.txt' - 'cmake/**' - 'src/*' @@ -23,41 +23,54 @@ on: - 'src/layer/vulkan/**' - 'python/**' concurrency: - group: linux-x64-gpu-clang-python-${{ github.ref }} + group: python-${{ github.ref }} cancel-in-progress: true +env: + DEVELOPER_DIR: /Applications/Xcode_15.2.app/Contents/Developer + MAC_DEPLOYMENT_TARGET: '11.0' + ENABLE_BITCODE: OFF + ENABLE_ARC: OFF + ENABLE_VISIBILITY: OFF + CMAKE_BUILD_PARALLEL_LEVEL: 4 + UseMultiToolTask: true permissions: contents: read jobs: - linux-clang-gpu: - runs-on: ubuntu-20.04 + build: strategy: matrix: - python-version: [3.7, 3.9, 3.11] + os: [ubuntu-latest, macos-13, windows-latest] + python-version: [3.9, 3.12] + + runs-on: ${{ matrix.os }} + steps: - uses: actions/checkout@v4 with: submodules: true + - name: cache-swiftshader + if: matrix.os == 'ubuntu-latest' id: cache-swiftshader uses: actions/cache@v4 with: path: swiftshader-install key: swiftshader-linux-install-20230420 - name: checkout-swiftshader - if: steps.cache-swiftshader.outputs.cache-hit != 'true' + if: matrix.os == 'ubuntu-latest' && steps.cache-swiftshader.outputs.cache-hit != 'true' uses: actions/checkout@v4 with: repository: google/swiftshader path: swiftshader ref: dd55e592406dc0bae219df11adec6363840aff4a - name: checkout-swiftshader-submodules - if: steps.cache-swiftshader.outputs.cache-hit != 'true' + if: matrix.os == 'ubuntu-latest' && steps.cache-swiftshader.outputs.cache-hit != 'true' run: | cd swiftshader git -c submodule."third_party/git-hooks".update=none submodule update --init --recursive - name: swiftshader - if: steps.cache-swiftshader.outputs.cache-hit != 'true' + if: matrix.os == 'ubuntu-latest' && steps.cache-swiftshader.outputs.cache-hit != 'true' run: | cd swiftshader mkdir -p build; cd build @@ -65,35 +78,47 @@ jobs: cmake --build . -j $(nproc) mkdir $GITHUB_WORKSPACE/swiftshader-install cp Linux/* $GITHUB_WORKSPACE/swiftshader-install - - name: set up python ${{ matrix.python-version }} + + - name: setup-python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: install-deps run: | python -m pip install --upgrade pip pip install pytest setuptools wheel twine + - name: build + if: matrix.os == 'ubuntu-latest' env: CC: clang CXX: clang++ run: | mkdir build && cd build - cmake -DNCNN_VULKAN=ON -DNCNN_PYTHON=ON -DNCNN_DISABLE_RTTI=OFF -DNCNN_DISABLE_EXCEPTION=OFF -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF .. + cmake -DNCNN_VULKAN=ON -DNCNN_PYTHON=ON -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF .. cmake --build . -j $(nproc) - - name: install python + - name: build + if: matrix.os == 'macos-13' + run: | + mkdir build && cd build + cmake -DCMAKE_TOOLCHAIN_FILE=../toolchains/ios.toolchain.cmake -DPLATFORM=MAC -DARCHS="x86_64" \ + -DDEPLOYMENT_TARGET=$MAC_DEPLOYMENT_TARGET -DENABLE_BITCODE=$ENABLE_BITCODE -DENABLE_ARC=$ENABLE_ARC -DENABLE_VISIBILITY=$ENABLE_VISIBILITY \ + -DNCNN_VULKAN=OFF -DNCNN_PYTHON=ON -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF .. + cmake --build . -j 4 + - name: build + if: matrix.os == 'windows-latest' + run: | + mkdir build; cd build + cmake -T v142,host=x64 -A x64 -DNCNN_VULKAN=OFF -DNCNN_PYTHON=ON -DNCNN_BUILD_TOOLS=OFF -DNCNN_BUILD_EXAMPLES=OFF .. + cmake --build . --config Release -j 4 + - name: build-python run: cd python && pip install . - name: test + if: matrix.os == 'ubuntu-latest' run: | export VK_ICD_FILENAMES="$GITHUB_WORKSPACE/swiftshader-install/vk_swiftshader_icd.json" cd python && pytest tests - - name: build and publish - if: startsWith(github.ref, 'refs/tags') - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/" + - name: test + if: matrix.os != 'ubuntu-latest' run: | - cd python - python setup.py bdist_wheel - twine upload dist/* + cd python && pytest tests diff --git a/.github/workflows/windows-x64-cpu-vs2019-python.yml b/.github/workflows/windows-x64-cpu-vs2019-python.yml deleted file mode 100644 index 3df91ab878b..00000000000 --- a/.github/workflows/windows-x64-cpu-vs2019-python.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: windows-x64-cpu-vs2019-python -on: - push: - branches: [master] - paths: - - '.github/workflows/windows-x64-cpu-vs2019-python.yml' - - 'CMakeLists.txt' - - 'cmake/**' - - 'src/*' - - 'src/layer/*' - - 'src/layer/x86/**' - - 'python/**' - pull_request: - branches: [master] - paths: - - '.github/workflows/windows-x64-cpu-vs2019-python.yml' - - 'CMakeLists.txt' - - 'cmake/**' - - 'src/*' - - 'src/layer/*' - - 'src/layer/x86/**' - - 'python/**' -concurrency: - group: windows-x64-cpu-vs2019-python-${{ github.ref }} - cancel-in-progress: true -permissions: - contents: read - -jobs: - windows-vs2019-python: - runs-on: windows-latest - strategy: - matrix: - python-version: [3.7, 3.9, 3.11] - env: - UseMultiToolTask: true - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: set up python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest setuptools wheel twine - - name: build - run: | - mkdir build; cd build - cmake -T v142,host=x64 -A x64 -DNCNN_PYTHON=ON -DNCNN_DISABLE_RTTI=OFF -DNCNN_DISABLE_EXCEPTION=OFF -DNCNN_BUILD_BENCHMARK=OFF -DNCNN_BUILD_EXAMPLES=OFF -DNCNN_BUILD_TOOLS=OFF .. - cmake --build . --config Release -j 4 - - name: install python - run: cd python && pip install . - - name: test - run: cd python && pytest tests - - name: build and publish - if: startsWith(github.ref, 'refs/tags') - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - TWINE_REPOSITORY_URL: "https://test.pypi.org/legacy/" - run: | - cd python - python setup.py bdist_wheel - twine upload dist/*