check-binary-size.yml

# This workflow checks if a PR commit has changed the size of a hello world Rust program.
# It downloads Rustc and compiles two versions of a stage0 compiler - one using the base commit
# of the PR, and one using the latest commit in the PR.
# If the size of the hello world program has changed, it posts a comment to the PR.
name: Check binary size

on: pull_request_target

env:
  OUTPUT_DIR: sizes

# Responsibility is divided between two jobs "measure" and "report", so that the
# job that builds (and potentnially runs) untrusted code does not have PR write
# permission, and vice-versa.
jobs:
  measure:
    name: Check binary size
    strategy:
      matrix:
        platform: [ubuntu-latest, windows-latest]
    runs-on: ${{ matrix.platform }}
    permissions:
      contents: read
    env:
      # This cannot be used as a context variable in the 'uses' key later. If it
      # changes, update those steps too.
      BACKTRACE_DIR: backtrace
      RUSTC_DIR: rustc
      TEST_MAIN_RS: foo.rs
      BASE_COMMIT: ${{ github.event.pull_request.base.sha }}
      HEAD_COMMIT: ${{ github.event.pull_request.head.sha }}
      OUTPUT_FILE: size-${{ strategy.job-index }}.json
    steps:
      - name: Print info
        shell: bash
        run: |
          echo "Current SHA: $HEAD_COMMIT"
          echo "Base SHA: $BASE_COMMIT"
      # Note: the backtrace source that's cloned here is NOT the version to be
      # patched in to std. It's cloned here to access the Github action for
      # building the test binary and measuring its size.
      - name: Clone backtrace to access Github action
        uses: actions/checkout@v3
        with:
          path: ${{ env.BACKTRACE_DIR }}
      - name: Clone Rustc
        uses: actions/checkout@v3
        with:
          repository: rust-lang/rust
          path: ${{ env.RUSTC_DIR }}
      - name: Set up std repository and backtrace submodule for size test
        shell: bash
        working-directory: ${{ env.RUSTC_DIR }}
        run: |
          # Bootstrap config
          cat <<EOF > config.toml
          [llvm]
          download-ci-llvm = true
          [rust]
          incremental = false
          EOF

          # Test program source
          cat <<EOF > $TEST_MAIN_RS
          fn main() {
            panic!();
          }
          EOF

          git submodule update --init library/backtrace

          cd library/backtrace
          git remote add head-pr https://github.com/${{ github.event.pull_request.head.repo.full_name }}
          git fetch --all

          # Directory for size data
          mkdir -p "$OUTPUT_DIR"
      - name: Build binary with base version of backtrace
        uses: ./backtrace/.github/actions/build-with-patched-std
        with:
          backtrace-commit: $BASE_COMMIT
          main-rs: ${{ env.TEST_MAIN_RS }}
          rustc-dir: ${{ env.RUSTC_DIR }}
        id: size-reference
      - name: Build binary with PR version of backtrace
        uses: ./backtrace/.github/actions/build-with-patched-std
        with:
          backtrace-commit: $HEAD_COMMIT
          main-rs: ${{ env.TEST_MAIN_RS }}
          rustc-dir: ${{ env.RUSTC_DIR }}
        id: size-updated
      # There is no built-in way to "collect" all the outputs of a set of jobs
      # run with a matrix strategy. Subsequent jobs that have a "needs"
      # dependency on this one will be run once, when the last matrix job is
      # run. Appending data to a single file within a matrix is subject to race
      # conditions. So we write the size data to files with distinct names
      # generated from the job index.
      - name: Write sizes to file
        uses: actions/github-script@v6
        env:
          SIZE_REFERENCE: ${{ steps.size-reference.outputs.test-binary-size }}
          SIZE_UPDATED: ${{ steps.size-updated.outputs.test-binary-size }}
          PLATFORM: ${{ matrix.platform }}
        with:
          script: |
            const fs = require("fs");
            const path = require("path");

            const output_data = JSON.stringify({
              reference: process.env.SIZE_REFERENCE,
              updated: process.env.SIZE_UPDATED,
              platform: process.env.PLATFORM,
            });

            // The "wx" flag makes this fail if the file exists, which we want,
            // because there should be no collisions.
            fs.writeFileSync(
              path.join([process.env.OUTPUT_DIR, process.env.OUTPUT_FILE]),
              output_data,
              { flag: "wx" },
            );
      - name: Upload size data
        uses: actions/upload-artifact@v3
        with:
          name: size-files
          path: ${{ env.OUTPUT_DIR }}/${{ env.OUTPUT_FILE }}
          retention-days: 1
          if-no-files-found: error
  report:
    name: Report binary size changes
    runs-on: ubuntu-latest
    needs: measure
    permissions:
      pull-requests: write
    steps:
      - name: Download size data
        uses: actions/download-artifact@v3
        with:
          name: size-files
          path: ${{ env.OUTPUT_DIR }}
      - name: Post a PR comment if the size has changed
        uses: actions/github-script@v6
        with:
          script: |
            const {globSync} = require("glob");
            const fs = require("fs");

            const size_dir = process.env.OUTPUT_DIR;

            sizes = {};

            const sizes = globSync(`${size_dir}/*.json`).map(path => {
              const contents = fs.readFileSync(path);
              return JSON.parse(contents);
            });

            const size_reports = sizes.flatMap(size_data => {
              const platform = size_data["platform"];
              const reference = size_data["reference"];
              const updated = size_data["updated"];

              if (!(reference > 0)) {
                core.setFailed(`Reference size invalid: ${reference}`);
                return;
              }

              if (!(updated > 0)) {
                core.setFailed(`Updated size invalid: ${updated}`);
                return;
              }

              const formatter = Intl.NumberFormat("en", {
                useGrouping: "always"
              });

              const updated_str = formatter.format(updated);
              const reference_str = formatter.format(reference);

              const diff = updated - reference;
              const diff_pct = (updated / reference) - 1;

              const diff_str = Intl.NumberFormat("en", {
                useGrouping: "always",
                sign: "exceptZero"
              }).format(diff);

              const diff_pct_str = Intl.NumberFormat("en", {
                style: "percent",
                useGrouping: "always",
                sign: "exceptZero",
                maximumFractionDigits: 2
              }).format(diff_pct);

              if (diff !== 0) {
                // The body is created here and wrapped so "weirdly" to avoid whitespace at the start of the lines,
                // which is interpreted as a code block by Markdown.
                const report = `On platform \`${platform}\`:

            - Original binary size: **${reference_str} B**
            - Updated binary size: **${updated_str} B**
            - Difference: **${diff_str} B** (${diff_pct_str})

            `;

                return [report];
              } else {
                return [];
              }
            });

            if (size_reports.length > 0) {
              const comment_sizes = size_reports.join();
              const body = `Code size changes for a hello-world Rust program linked with libstd with backtrace:

              ${comment_body_sizes}`;

              github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body
              });
            }