diff --git a/.github/workflows/pycdePublish.yml b/.github/workflows/pycdePublish.yml index 84990daa6b3c..6cdc1de0b35f 100644 --- a/.github/workflows/pycdePublish.yml +++ b/.github/workflows/pycdePublish.yml @@ -1,60 +1,195 @@ name: PyCDE Test and Publish -# Build the wheels for a series of Python versions and OSes and publish them to -# PyPI. - on: + push: + tags: + - pycde-* workflow_dispatch: jobs: - # Build CIRCT and run its tests using a Docker container with all the - # integration testing prerequisite installed. - build-circt: + + # --------------------------------------------------------------------------- + # Build and test Linux wheels. Run the CIRCT tests also. + # --------------------------------------------------------------------------- + + build-linux: name: Build - runs-on: ubuntu-latest + # Run on an internal MSFT subscription. Please DO NOT use this for any other + # workflows without talking to John Demme (john.demme@microsoft.com, GH + # teqdruid) first. We may lose funding for this if it ends up costing too + # much. + # If individual jobs fail due to timeouts or disconnects, please report to + # John and re-run the job. + runs-on: + - self-hosted + - 1ES.Pool=1ES-CIRCT-builds + - 1ES.ImageOverride=CIRCT-StdUbuntu2204 strategy: - # Keep the 'matrix' strategy with one data point to make it obvious that - # this is one point in the overall matrix. matrix: python-env: - - cp38-win_amd64 + - cp38-manylinux_x86_64 + - cp39-manylinux_x86_64 + - cp310-manylinux_x86_64 + - cp311-manylinux_x86_64 + - cp312-manylinux_x86_64 steps: - # Clone the CIRCT repo and its submodules. Do shallow clone to save clone - # time. - - name: Get CIRCT - uses: actions/checkout@v3 + - name: Get CIRCT (no submodules) + uses: actions/checkout@v4 with: - submodules: true + fetch-depth: 0 + fetch-tags: true + submodules: false + + - name: Get shallow LLVM submodule + run: | + git submodule update --init --recursive --recommend-shallow --depth 1 - # -------- - # Build and test CIRCT - # -------- - name: Install dependencies run: | set -o errexit - sudo apt install -y python3-pip python3 -m pip install --upgrade pip python3 -m pip install cibuildwheel twine + + # Build CIRCT and run its tests using a Docker container with all the + # integration testing prerequisite installed. - name: Build wheel env: - CC: clang - CXX: clang++ CIBW_BUILD: ${{ matrix.python-env }} - CIBW_ENVIRONMENT: CMAKE_GENERATOR=Ninja + CMAKE_GENERATOR: Ninja SETUPTOOLS_SCM_DEBUG: True - BUILD_TYPE: RelWithAsserts + BUILD_TYPE: Release RUN_TESTS: True run: | + set -o errexit export PATH=$PATH:$HOME/.local/bin - cd $(Pipeline.Workspace)/circt - - echo "Fetching history and tags from CIRCT repo" - git fetch --depth=1000000 --tags --no-recurse-submodules echo "Building wheel" cibuildwheel --output-dir wheelhouse frontends/PyCDE - - name: Upload Binary (Non-Tag) - uses: actions/upload-artifact@v3 + + - name: Get wheel name + shell: bash + id: whl-name + run: | + cd wheelhouse + echo WHL=`ls *.whl` >> "$GITHUB_OUTPUT" + + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.whl-name.outputs.WHL }} + path: wheelhouse/${{ steps.whl-name.outputs.WHL }} + retention-days: 7 + if-no-files-found: error + + + # --------------------------------------------------------------------------- + # Build and test Windows wheels. + # --------------------------------------------------------------------------- + + build-windows: + name: Build + # Run on an internal MSFT subscription. Please DO NOT use this for any other + # workflows without talking to John Demme (john.demme@microsoft.com, GH + # teqdruid) first. We may lose funding for this if it ends up costing too + # much. + # If individual jobs fail due to timeouts or disconnects, please report to + # John and re-run the job. + runs-on: + - self-hosted + - 1ES.Pool=1ES-CIRCT-builds + - 1ES.ImageOverride=1esMMSWindows2022 + strategy: + matrix: + python-env: + - cp38-win_amd64 + - cp39-win_amd64 + - cp310-win_amd64 + - cp311-win_amd64 + - cp312-win_amd64 + steps: + # Since we don't use docker on Windows, we need to install the dependencies. + - name: Build additional c++ deps + shell: pwsh + run: | + & "${env:VCPKG_INSTALLATION_ROOT}/vcpkg" --triplet x64-windows install zlib capnproto + + - name: Get CIRCT + uses: actions/checkout@v4 with: - path: wheelhouse/*.whl + fetch-depth: 0 + fetch-tags: true + submodules: false + + - name: Get shallow LLVM submodule + run: | + git submodule update --init --recursive --recommend-shallow --depth 1 + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install cibuildwheel twine + choco install ninja + + - name: Build wheel + shell: cmd + env: + CIBW_BUILD: ${{ matrix.python-env }} + CMAKE_GENERATOR: Ninja + SETUPTOOLS_SCM_DEBUG: True + BUILD_TYPE: Release + # PyCDE integration tests are not yet implemented on Windows. + RUN_TESTS: False + run: | + echo "Building wheel" + "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=x64 -host_arch=x64 && python3 -m cibuildwheel --output-dir wheelhouse frontends/PyCDE + + - name: Get wheel name + shell: bash + id: whl-name + run: | + cd wheelhouse + echo WHL=`ls *.whl` >> "$GITHUB_OUTPUT" + + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.whl-name.outputs.WHL }} + path: wheelhouse/${{ steps.whl-name.outputs.WHL }} retention-days: 7 + if-no-files-found: error + + + # --------------------------------------------------------------------------- + # If both the Linux and Windows builds are successful, push the wheels to + # pypi. Only do this on the main branch or a tag. + # --------------------------------------------------------------------------- + + push_wheels: + name: Push wheels (Tag or Weekly) + runs-on: ubuntu-20.04 + if: github.repository == 'llvm/circt' && ( github.ref == 'refs/heads/main' || github.ref_type == 'tag' ) + needs: + - build-linux + - build-windows + environment: + name: pypi + url: https://pypi.org/p/esiaccel + permissions: + id-token: write + + steps: + - name: Download wheels + uses: actions/download-artifact@v4 + with: + path: ./wheelhouse/ + merge-multiple: true + + - name: List downloaded wheels + run: ls -laR + working-directory: ./wheelhouse/ + + - name: Upload wheels to pypi + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: wheelhouse/ + verify-metadata: false diff --git a/frontends/PyCDE/pyproject.toml b/frontends/PyCDE/pyproject.toml index 94874cacf1e7..5203af06035c 100644 --- a/frontends/PyCDE/pyproject.toml +++ b/frontends/PyCDE/pyproject.toml @@ -31,6 +31,16 @@ manylinux-x86_64-image = "ghcr.io/circt/images/pycde-build" [tool.cibuildwheel.linux] # Use our internal auditwheel script so as to not mess up the collateral. repair-wheel-command = "frontends/PyCDE/auditwheel.sh {dest_dir} {wheel}" +environment-pass = [ + "SCCACHE_GHA_ENABLED", + "ACTIONS_CACHE_URL", + "ACTIONS_RUNTIME_TOKEN", + "CMAKE_GENERATOR", + "SETUPTOOLS_SCM_DEBUG", + "BUILD_TYPE", + "RUN_TESTS", + "COMPILER_LAUNCHER" +] [project] name = "pycde" diff --git a/frontends/PyCDE/setup.py b/frontends/PyCDE/setup.py index 281e2f0537fa..871638f98a84 100644 --- a/frontends/PyCDE/setup.py +++ b/frontends/PyCDE/setup.py @@ -45,7 +45,6 @@ class CMakeBuild(build_py): def run(self): target_dir = self.build_lib - test = os.getenv("RUN_TESTS") cmake_build_dir = os.getenv("PYCDE_CMAKE_BUILD_DIR") if not cmake_build_dir: cmake_build_dir = os.path.join(target_dir, "..", "cmake_build") @@ -54,6 +53,8 @@ def run(self): os.environ.get("CIRCT_DIRECTORY", os.path.join(_thisdir, "..", ".."))) src_dir = os.path.abspath(os.path.join(circt_dir, "llvm", "llvm")) cfg = "Release" + if "BUILD_TYPE" in os.environ: + cfg = os.environ["BUILD_TYPE"] cmake_args = [ "-DCMAKE_INSTALL_PREFIX={}".format(os.path.abspath(cmake_install_dir)), "-DPython3_EXECUTABLE={}".format(sys.executable.replace("\\", "/")), @@ -66,9 +67,23 @@ def run(self): "-DCIRCT_ENABLE_FRONTENDS=PyCDE", "-DLLVM_EXTERNAL_PROJECTS=circt", "-DLLVM_EXTERNAL_CIRCT_SOURCE_DIR={}".format(circt_dir), + "-DESI_RUNTIME=ON", ] + if "COMPILER_LAUNCHER" in os.environ: + cmake_args += [ + f"-DCMAKE_C_COMPILER_LAUNCHER={os.environ['COMPILER_LAUNCHER']}", + f"-DCMAKE_CXX_COMPILER_LAUNCHER={os.environ['COMPILER_LAUNCHER']}" + ] + if "CC" in os.environ: + cmake_args += [f"-DCMAKE_C_COMPILER={os.environ['CC']}"] + if "CXX" in os.environ: + cmake_args += [f"-DCMAKE_CXX_COMPILER={os.environ['CXX']}"] if "CIRCT_EXTRA_CMAKE_ARGS" in os.environ: cmake_args += os.environ["CIRCT_EXTRA_CMAKE_ARGS"].split(" ") + if "VCPKG_INSTALLATION_ROOT" in os.environ: + cmake_args += [ + f"-DCMAKE_TOOLCHAIN_FILE={os.environ['VCPKG_INSTALLATION_ROOT']}/scripts/buildsystems/vcpkg.cmake" + ] build_args = [] build_parallelism = os.getenv("CMAKE_PARALLELISM") if build_parallelism: @@ -81,25 +96,22 @@ def run(self): cmake_cache_file = os.path.join(cmake_build_dir, "CMakeCache.txt") if os.path.exists(cmake_cache_file): os.remove(cmake_cache_file) + print(f"Running cmake with args: {cmake_args}", file=sys.stderr) + subprocess.check_call(["echo", "Running: cmake", src_dir] + cmake_args) subprocess.check_call(["cmake", src_dir] + cmake_args, cwd=cmake_build_dir) + targets = ["check-pycde"] + if "RUN_TESTS" in os.environ and os.environ["RUN_TESTS"] != "false": + # The pycde integration tests test both PyCDE and the ESIRuntime so + # failure shouldn't gate publishing PyCDE. + # targets.append("check-pycde-integration") + targets.append("check-circt") subprocess.check_call([ "cmake", "--build", ".", "--target", - "check-pycde", - ] + build_args, + ] + targets + build_args, cwd=cmake_build_dir) - if test.lower() == "true": - subprocess.check_call([ - "cmake", - "--build", - ".", - "--target", - "check-pycde-integration", - "check-circt", - ] + build_args, - cwd=cmake_build_dir) install_cmd = ["cmake", "--build", ".", "--target", "install-PyCDE"] subprocess.check_call(install_cmd + build_args, cwd=cmake_build_dir) shutil.copytree(os.path.join(cmake_install_dir, "python_packages"), diff --git a/lib/Dialect/ESI/runtime/CMakeLists.txt b/lib/Dialect/ESI/runtime/CMakeLists.txt index 9ed3b0153385..a4ca78e420bd 100644 --- a/lib/Dialect/ESI/runtime/CMakeLists.txt +++ b/lib/Dialect/ESI/runtime/CMakeLists.txt @@ -241,7 +241,9 @@ if (WHEEL_BUILD) endif() # Pybind11 is used to wrap the ESIRuntime APIs. -find_package(Python3 COMPONENTS Interpreter Development) +if(NOT DEFINED Python3_FOUND) + find_package(Python3 COMPONENTS Interpreter Development) +endif() if(Python3_FOUND) IF(MSVC) # Work around an issue with pybind11 and cmake incompatibility on Windows in debug mode.