Merge pull request #2153 from infogulch/benchmark

Add benchmarks

.github/actions/setup-rust/action.yml

@@ -0,0 +1,52 @@
+name: 'Setup Rust'
+inputs:
+  rust-version:
+    required: true
+    type: string
+  targets:
+    required: true
+    type: string
+  components:
+    required: false
+    default:
+  cache-context:
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+  - uses: dtolnay/rust-toolchain@master
+    id: toolchain
+    with:
+      toolchain: ${{ inputs.rust-version }}
+      targets: ${{ inputs.targets }}
+      components: ${{ inputs.components }}
+
+  - name: Install i686 dependencies
+    if: "contains(inputs.targets,'i686')"
+    shell: bash
+    run: |
+      sudo dpkg --add-architecture i386
+      sudo apt-get update
+      sudo apt-get install libssl-dev:i386 gcc-multilib clang -y
+      echo "CC=clang" >> $GITHUB_ENV
+      echo "PKG_CONFIG_SYSROOT_DIR=/" >> $GITHUB_ENV
+
+  - uses: actions/cache@v3
+    with:
+      path: |
+        ~/.cargo/bin/
+        ~/.cargo/registry/index/
+        ~/.cargo/registry/cache/
+        ~/.cargo/git/db/
+        target/
+      key: ${{ inputs.cache-context }}_${{ inputs.targets }}_rustc-${{ steps.toolchain.outputs.cachekey }}_cargo-${{ hashFiles('**/Cargo.lock') }}
+
+  # Remove build artifacts for the current crate, since it will be rebuilt every
+  # run anyway, but keep dependency artifacts to cache them.
+  # Must be placed after actions/cache so its post step runs first.
+  - uses: pyTooling/Actions/with-post-step@v0.4.6
+    with:
+      main: bash ./.github/actions/setup-rust/cleanup.sh
+      post: bash ./.github/actions/setup-rust/cleanup.sh

.github/actions/setup-rust/cleanup.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+echo Cleanup workspace build artifacts and extra target output
+
+# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects
+cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'`
+
+# remove directories in /target/ that are not named `debug` or `release`
+find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \;

.github/workflows/ci.yml

@@ -15,42 +15,18 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@master
-        id: toolchain
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
         with:
-          toolchain: nightly
+          rust-version: nightly
           targets: x86_64-unknown-linux-gnu
           components: clippy, rustfmt
-      - run: cargo install cargo2junit --force
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: style_rustc-${{ steps.toolchain.outputs.cachekey }}_cargo-${{ hashFiles('**/Cargo.lock') }}
+          cache-context: style
 
       - name: Check formatting
         run: cargo fmt --check
       - name: Check clippy
         run: cargo clippy --no-deps --all-targets
-      - name: Test and report
-        run: |
-          RUSTC_BOOTSTRAP=1 cargo test --all -- -Z unstable-options --format json --report-time | cargo2junit > cargo_test_results.xml
-      - name: Publish cargo test results artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: cargo-test-results
-          path: cargo_test_results.xml
-      - name: Publish cargo test summary
-        uses: EnricoMi/publish-unit-test-result-action/composite@master
-        with:
-          check_name: Cargo test summary
-          files: cargo_test_results.xml
-          fail_on: nothing
-          comment_mode: off
 
   build-test:
     runs-on: ${{ matrix.os }}
@@ -75,34 +51,19 @@ jobs:
         shell: bash
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@master
-        id: toolchain
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
         with:
-          toolchain: ${{ matrix.rust-version }}
+          rust-version: ${{ matrix.rust-version }}
           targets: ${{ matrix.target }}
-      - name: Install i686 dependencies
-        if: "contains(matrix.target,'i686')"
-        run: |
-          sudo dpkg --add-architecture i386
-          sudo apt-get update
-          sudo apt-get install libssl-dev:i386 gcc-multilib clang -y
-          echo "CC=clang" >> $GITHUB_ENV
-          echo "PKG_CONFIG_SYSROOT_DIR=/" >> $GITHUB_ENV
-      - uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ matrix.os }}_${{ matrix.target }}_rustc-${{ steps.toolchain.outputs.cachekey }}_cargo-${{ hashFiles('**/Cargo.lock') }}
+          cache-context: ${{ matrix.os }}
 
       # Build and test.
       - name: Build library
         run: cargo rustc --lib --target ${{ matrix.target }} ${{ matrix.args }} --verbose
       - name: Test
-        run: cargo test --target ${{ matrix.target }} ${{ matrix.args }} --all --verbose || echo "::warning ::Tests failed"
+        continue-on-error: ${{ contains(matrix.target,'wasm32') }} # allow wasm builds to fail tests for now
+        run: cargo test --target ${{ matrix.target }} ${{ matrix.args }} --all --verbose
 
       # On stable rust builds, build a binary and publish as a github actions
       # artifact. These binaries could be useful for testing the pipeline but
@@ -166,6 +127,53 @@ jobs:
           fail_on: nothing
           comment_mode: off
 
+  report:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Rust
+        uses: ./.github/actions/setup-rust
+        with:
+          rust-version: nightly
+          targets: x86_64-unknown-linux-gnu
+          cache-context: report
+      - run: |
+          cargo install cargo2junit --force
+          cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'`
+          sudo apt install valgrind -y
+
+      - name: Test and report
+        run: |
+          RUSTC_BOOTSTRAP=1 cargo test --all -- -Z unstable-options --format json --report-time | cargo2junit > cargo_test_results.xml
+      - name: Publish cargo test results artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: cargo-test-results
+          path: cargo_test_results.xml
+      - name: Publish cargo test summary
+        uses: EnricoMi/publish-unit-test-result-action/composite@master
+        with:
+          check_name: Cargo test summary
+          files: cargo_test_results.xml
+          fail_on: nothing
+          comment_mode: off
+
+      - run: cargo build --bench run_iai --release
+      - run: cargo bench --bench run_iai
+      - run: cargo bench --bench run_criterion
+
+      - name: Publish criterion benchmark results
+        uses: actions/upload-artifact@v3
+        with:
+          name: bench-criterion-results
+          path: target/criterion/**/*
+
+      - name: Publish iai benchmark results
+        uses: actions/upload-artifact@v3
+        with:
+          name: bench-iai-results
+          path: target/iai/scryer-prolog/run_iai/bench_group/*
+
   # Publish binaries when building for a tag
   release:
     runs-on: ubuntu-20.04