fix linux PGO wheel build (#1557)

pydantic · Nov 26, 2024 · 49bcd51 · 49bcd51
1 parent 83ff1cf
commit 49bcd51
Show file tree

Hide file tree

Showing 5 changed files with 226 additions and 64 deletions.
diff --git a/.github/actions/build-pgo-wheel/action.yml b/.github/actions/build-pgo-wheel/action.yml
@@ -0,0 +1,74 @@
+name: Build PGO wheel
+description: Builds a PGO-optimized wheel
+inputs:
+  interpreter:
+    description: 'Interpreter to build the wheel for'
+    required: true
+  rust-toolchain:
+    description: 'Rust toolchain to use'
+    required: true
+outputs:
+  wheel:
+    description: 'Path to the built wheel'
+    value: ${{ steps.find_wheel.outputs.path }}
+runs:
+  using: "composite"
+  steps:
+    - name: prepare self schema
+      shell: bash
+      # generate up front so that we don't have to do this inside the docker container
+      run: uv run python generate_self_schema.py
+
+    - name: prepare profiling directory
+      shell: bash
+      # making this ahead of the compile ensures that the local user can write to this
+      # directory; the maturin action (on linux) runs in docker so would create as root
+      run: mkdir -p ${{ github.workspace }}/profdata
+
+    - name: build initial wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        manylinux: auto
+        args: >
+          --release
+          --out pgo-wheel
+          --interpreter ${{ inputs.interpreter }}
+        rust-toolchain: ${{ inputs.rust-toolchain }}
+        docker-options: -e CI
+      env:
+        RUSTFLAGS: '-Cprofile-generate=${{ github.workspace }}/profdata'
+
+    - name: detect rust host
+      run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
+      shell: bash
+
+    - name: generate pgo data
+      run: |
+        uv sync --group testing
+        uv pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall
+        uv run pytest tests/benchmarks
+        RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2)
+        rustup run ${{ inputs.rust-toolchain }} bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/$RUST_HOST/bin/llvm-profdata >> "$GITHUB_ENV"'
+      shell: bash
+
+    - name: merge pgo data
+      run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
+      shell: bash
+
+    - name: build pgo-optimized wheel
+      uses: PyO3/maturin-action@v1
+      with:
+        manylinux: auto
+        args: >
+          --release
+          --out dist
+          --interpreter ${{ inputs.interpreter }}
+        rust-toolchain: ${{inputs.rust-toolchain}}
+        docker-options: -e CI
+      env:
+        RUSTFLAGS: '-Cprofile-use=${{ github.workspace }}/merged.profdata'
+
+    - name: find built wheel
+      id: find_wheel
+      run: echo "path=$(ls dist/*.whl)" | tee -a "$GITHUB_OUTPUT"
+      shell: bash
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -556,50 +556,12 @@ jobs:
           # FIXME: Unpin when Python 3.8 support is dropped. (3.9 requires Windows 10)
           toolchain: ${{ (matrix.os == 'windows' && '1.77') || 'stable' }}
 
-      - run: pip install -U 'ruff==0.5.0' typing_extensions
-
-      # generate self-schema now, so we don't have to do so inside docker in maturin build
-      - run: python generate_self_schema.py
-
-      - name: build initial wheel
-        uses: PyO3/maturin-action@v1
+      - name: Build PGO wheel
+        id: pgo-wheel
+        uses: ./.github/actions/build-pgo-wheel
         with:
-          manylinux: auto
-          args: >
-            --release
-            --out pgo-wheel
-            --interpreter ${{ matrix.interpreter }}
+          interpreter: ${{ env.UV_PYTHON }}
           rust-toolchain: ${{ steps.rust-toolchain.outputs.name }}
-          docker-options: -e CI
-        env:
-          RUSTFLAGS: '-Cprofile-generate=${{ github.workspace }}/profdata'
-
-      - name: detect rust host
-        run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
-        shell: bash
-
-      - name: generate pgo data
-        run: |
-          uv sync --group testing
-          uv pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall
-          uv run pytest tests/benchmarks
-          rustup run ${{ steps.rust-toolchain.outputs.name }} bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"'
-
-      - name: merge pgo data
-        run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
-
-      - name: build pgo-optimized wheel
-        uses: PyO3/maturin-action@v1
-        with:
-          manylinux: auto
-          args: >
-            --release
-            --out dist
-            --interpreter ${{ matrix.interpreter }}
-          rust-toolchain: ${{steps.rust-toolchain.outputs.name}}
-          docker-options: -e CI
-        env:
-          RUSTFLAGS: '-Cprofile-use=${{ github.workspace }}/merged.profdata'
 
       - run: ${{ matrix.ls || 'ls -lh' }} dist/
 

diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -30,40 +30,36 @@ jobs:
         with:
           enable-cache: true
 
-      - name: install deps
+      - name: Install deps
         run: |
           uv sync --group testing
           uv pip uninstall pytest-speed
           uv pip install pytest-benchmark==4.0.0 pytest-codspeed
 
-      - name: install rust stable
+      - name: Install rust stable
         id: rust-toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
           components: llvm-tools
 
-      - name: cache rust
+      - name: Cache rust
         uses: Swatinem/rust-cache@v2
 
-      - name: Compile pydantic-core for profiling
-        run: make build-profiling
+      - name: Build PGO wheel
+        id: pgo-wheel
+        uses: ./.github/actions/build-pgo-wheel
+        with:
+          interpreter: ${{ env.UV_PYTHON }}
+          rust-toolchain: ${{ steps.rust-toolchain.outputs.name }}
         env:
-          CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
-          RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"
-
-      - name: Gather pgo data
-        run: uv run pytest tests/benchmarks
+          # make sure profiling information is present
+          CARGO_PROFILE_RELEASE_DEBUG: "line-tables-only"
+          CARGO_PROFILE_RELEASE_STRIP: false
 
-      - name: Prepare merged pgo data
-        run: rustup run stable bash -c '$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata'
-
-      - name: Compile pydantic-core for benchmarking
-        run: make build-profiling
-        env:
-          CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
-          RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"
+      - name: Install PGO wheel
+        run: uv pip install ${{ steps.pgo-wheel.outputs.wheel }} --force-reinstall
 
       - name: Run CodSpeed benchmarks
         uses: CodSpeedHQ/action@v3
         with:
-          run: uv run pytest tests/benchmarks/ --codspeed
+          run: uv run --group=codspeed pytest tests/benchmarks/ --codspeed
diff --git a/pyproject.toml b/pyproject.toml
@@ -82,6 +82,10 @@ wasm = [
     'maturin>=1,<2',
     'ruff',
 ]
+codspeed = [
+    # codspeed is only run on CI, with latest version of CPython
+    'pytest-codspeed; python_version == "3.13" and implementation_name == "cpython"',
+]
 
 all = [
   { include-group = 'testing' },