benchmark.yml

name: Benchmark

on:
  workflow_dispatch:
    inputs:
      old:
        description: "Old version to compare"
        type: string
        default: "latest"
        required: true
      new:
        description: "New version to compare"
        type: string
        default: "trunk"
        required: true
      theme:
        description: "Theme to test"
        type: choice
        options:
        - twentytwentyone
        - twentytwentythree
        - twentytwentyfour

jobs:
  benchmarks:
    name: "Benchmarks"
    runs-on: ubuntu-latest
    strategy:
      matrix:
        run: [1, 2, 3, 4, 5]

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Checkout
        uses: actions/checkout@v3
        with:
          repository: GoogleChromeLabs/wpp-research
          path: wpp-research

      - name: Setup Node
        uses: actions/setup-node@v3
        with:
          node-version-file: ".nvmrc"
          cache: npm

      - name: Install dependencies
        run: npm ci

      - name: Install wp-env
        run: npm -g i @wordpress/env

      - name: Install wpp-research dependencies
        working-directory: wpp-research
        run: npm ci

      - name: Configure WordPress versions
        run: |
          echo "Old version: $OLD_VERSION"
          if [[ $OLD_VERSION != 'latest' ]]; then
            if [[ "$OLD_VERSION" == *".zip"* ]]; then
              echo "{\"core\":\"$OLD_VERSION\"}" >> old/.wp-env.override.json
            else
              echo "{\"core\":\"WordPress/WordPress#$OLD_VERSION\"}" >> old/.wp-env.override.json
            fi
          fi

          echo "New version: $NEW_VERSION"
          if [[ "$NEW_VERSION" == *".zip"* ]]; then
            echo "{\"core\":\"$NEW_VERSION\"}" >> new/.wp-env.override.json
          else
            echo "{\"core\":\"WordPress/WordPress#$NEW_VERSION\"}" >> new/.wp-env.override.json
          fi
        env:
          OLD_VERSION: ${{ inputs.old == 'trunk' && 'master' || inputs.old }}
          NEW_VERSION: ${{ inputs.new == 'trunk' && 'master' || inputs.new }}

      - name: Install WordPress
        run: |
          chmod -R 767 old/ # TODO: Possibly integrate in wp-env
          chmod -R 767 new/ # TODO: Possibly integrate in wp-env
          (cd old && wp-env start)
          (cd new && wp-env start)

      - name: Update permalink structure
        run: |
          (cd old && wp-env run tests-cli wp rewrite structure '/%postname%/' -- --hard)
          (cd new && wp-env run tests-cli wp rewrite structure '/%postname%/' -- --hard)

      - name: Import mock data
        run: |
          (cd old && wp-env run tests-cli curl https://raw.githubusercontent.com/WordPress/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml -- --output /tmp/themeunittestdata.wordpress.xml)
          (cd old && wp-env run tests-cli wp import /tmp/themeunittestdata.wordpress.xml -- --authors=create)
          (cd new && wp-env run tests-cli curl https://raw.githubusercontent.com/WordPress/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml -- --output /tmp/themeunittestdata.wordpress.xml)
          (cd new && wp-env run tests-cli wp import /tmp/themeunittestdata.wordpress.xml -- --authors=create)

      - name: Deactivate WordPress Importer
        run: |
          (cd old && wp-env run tests-cli wp plugin deactivate wordpress-importer)
          (cd new && wp-env run tests-cli wp plugin deactivate wordpress-importer)

      - name: Post Debugging Info
        run: |
          echo "Old version – Theme info"
          (cd old && npm run wp-env run tests-cli wp theme list)
          echo "New version – Theme info"
          (cd new && npm run wp-env run tests-cli wp theme list)

      - name: Install theme
        run: |
          (cd old && wp-env run tests-cli wp theme activate $THEME)
          (cd new && wp-env run tests-cli wp theme activate $THEME)
        env:
          THEME: ${{ inputs.theme }}

      - name: Benchmark Web Vitals
        working-directory: wpp-research
        run: |
          echo "## Run $RUN
          **Before:** $OLD_VERSION
          **After:** $NEW_VERSION" >> $GITHUB_STEP_SUMMARY
          npm run research --silent -- benchmark-web-vitals -u http://localhost:8881/ -n 100 -p -v -o csv > before.csv
          npm run research --silent -- benchmark-web-vitals -u http://localhost:8891/ -n 100 -p -v -o csv > after.csv
          node ../scripts/results.js "Web Vitals ($THEME)" before.csv after.csv > summary.md
          cat summary.md >> $GITHUB_STEP_SUMMARY
        env:
          THEME: ${{ inputs.theme }}
          RUN: ${{ matrix.run }}
          OLD_VERSION: ${{ inputs.old == 'trunk' && 'master' || inputs.old }}
          NEW_VERSION: ${{ inputs.new == 'trunk' && 'master' || inputs.new }}

      - name: Benchmark Server-Timing
        working-directory: wpp-research
        run: |
          npm run research --silent -- benchmark-server-timing -u http://localhost:8881/ -n 100 -p -v -o csv > before.csv
          npm run research --silent -- benchmark-server-timing -u http://localhost:8891/ -n 100 -p -v -o csv > after.csv
          node ../scripts/results.js "Server-Timing ($THEME)" before.csv after.csv > summary.md
          cat summary.md >> $GITHUB_STEP_SUMMARY
        env:
          THEME: ${{ inputs.theme }}