Benchmarks completely redone

.github/workflows/benchmark.yml

@@ -1,94 +1,234 @@
 name: Benchmark
 
 on:
+  push:
+    branches:
+      - master
   pull_request:
-    if: >-
-      (github.event.action == 'labeled' && github.event.label.name == 'performance') ||
-      (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'performance'))
-    branches: [master]
-    types: [labeled, synchronize]
 
-  workflow_dispatch:
-    branches: [master]
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
 
 jobs:
-  build:
+  results:
     runs-on: ubuntu-latest
+    needs: benchmark
 
     steps:
       - uses: actions/checkout@v2
-        with:
-          repository: yewstack/js-framework-benchmark
-
-      - name: Configure benchmark
-        run: |
-          replace="yew = { git = \"${REPO_HTML_URL}\", branch = \"${BRANCH}\" }"
-          input=$(cat frameworks/keyed/yew/Cargo.toml)
-          output=$(echo "$input" | sed -e "s@yew = .*}@$replace@g")
-          if [[ "$input" == "$output" ]]; then
-            echo "ERROR: failed to configure Cargo.toml"
-            exit 1
-          fi
-          echo "$output" > frameworks/keyed/yew/Cargo.toml
-          echo "$output"
-        env:
-          # REPO_HTML_URL: ${{ github.event.pull_request.head.repo.html_url }}
-          REPO_HTML_URL: https://github.com/voidpumpkin/yew.git
-          # HEAD_REF: ${{ github.event.pull_request.head.ref }}
-          BRANCH: master
 
-      - name: Setup ChromeDriver
-        uses: nanasess/setup-chromedriver@master
+      - run: |
+          touch results.json
+          echo '${{ needs.benchmark.outputs.results }}' >> results.json
 
-      - name: Setup Rust
-        uses: actions-rs/toolchain@v1
+      # gh-pages branch is updated and pushed automatically with extracted benchmark data
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
         with:
-          toolchain: stable
+          name: "Yew master branch benchmarks (Lower is better)"
+          tool: "customSmallerIsBetter"
+          output-file-path: results.json
+          gh-pages-branch: "gh-pages"
+          # Access token to deploy GitHub Pages branch
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # Push and deploy GitHub pages branch automatically
+          comment-always: true
+          alert-threshold: "150%"
+          comment-on-alert: true
+          alert-comment-cc-users: "@yewstack/yew"
+          # Don't push to gh-pages if its a pull request
+          auto-push: ${{ github.event_name != 'pull_request' }}
+          save-data-file: ${{ github.event_name != 'pull_request' }}
+
+  benchmark:
+    runs-on: ubuntu-latest
+    outputs:
+      results: ${{ steps.results.outputs.stdout }}
 
-      - uses: actions-rs/install@v0.1
+    steps:
+      - uses: actions/checkout@v2
         with:
-          crate: wasm-bindgen-cli
-          version: latest
-          use-tool-cache: true
+          path: "./yew"
 
-      - uses: actions-rs/install@v0.1
+      - uses: actions/checkout@v2
         with:
-          crate: wasm-pack
-          version: latest
-          use-tool-cache: true
+          repository: krausest/js-framework-benchmark
+          path: "./js-framework-benchmark"
 
-      - uses: actions-rs/install@v0.1
+      - uses: actions-rs/toolchain@v1
         with:
-          crate: https
-          version: latest
-          use-tool-cache: true
+          toolchain: stable
+          target: wasm32-unknown-unknown
+          override: true
+          profile: minimal
 
-      - name: Start Server
-        run: http -p 8080 &
+      - uses: jetli/wasm-pack-action@v0.3.0
+        with:
+          version: "latest"
 
       - name: Setup Node
         uses: actions/setup-node@v1
         with:
-          node-version: 12
+          node-version: 16
+
+      - uses: Swatinem/rust-cache@v1
+        with:
+          working-directory: yew
 
-      - name: npm Install
+      - uses: actions/cache@v2
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-benchmark-${{ hashFiles('js-framework-benchmark/package-lock.json') }}-${{ hashFiles('js-framework-benchmark/webdriver-ts/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-benchmark-
+            ${{ runner.os }}
+
+      - name: save yew-struct setup
+        shell: bash
+        run: |
+          mkdir yew-struct-setup
+          cp -r js-framework-benchmark/frameworks/keyed/yew/* yew-struct-setup/
+          cd yew-struct-setup/bundled-dist
+          rm -rf ./*
+      # Will be enabled after https://github.com/krausest/js-framework-benchmark/pull/985 gets merged
+      # - name: save yew-hooks setup
+      #   shell: bash
+      #   run: |
+      #     mkdir yew-hooks-setup
+      #     cp -r js-framework-benchmark/frameworks/keyed/yew-hooks/* yew-hooks-setup/
+      #     cd yew-hooks-setup/bundled-dist
+      #     rm -rf ./*
+
+      - name: replace framework version in yew
+        shell: bash
         run: |
-          npm install
-          (cd webdriver-ts && npm install)
-          (cd webdriver-ts-results && npm install)
+          replace="  \"frameworkVersion\": \"\""
+          input=$(cat yew-struct-setup/package.json)
+          output=$(echo "$input" | sed -e "s@\"frameworkVersion\": .*\"@$replace@g")
+          if [[ "$input" == "$output" ]]; then
+            echo "ERROR: failed to configure framework version"
+            exit 1
+          fi
+          echo "$output" > yew-struct-setup/package.json
+          echo "$output"
 
-      - name: Build
+      # Will be enabled after https://github.com/krausest/js-framework-benchmark/pull/985 gets merged
+      # - name: replace framework version in yew-hooks
+      #   shell: bash
+      #   run: |
+      #     replace="  \"frameworkVersion\": \"\""
+      #     input=$(cat yew-hooks-setup/package.json)
+      #     output=$(echo "$input" | sed -e "s@\"frameworkVersion\": .*\"@$replace@g")
+      #     if [[ "$input" == "$output" ]]; then
+      #       echo "ERROR: failed to configure framework version"
+      #       exit 1
+      #     fi
+      #     echo "$output" > yew-hooks-setup/package.json
+      #     echo "$output"
+
+      - name: delete all frameworks
+        shell: bash
         run: |
-          npm run build-prod
-          (cd webdriver-ts && npm run build-prod)
+          cd js-framework-benchmark/frameworks/keyed
+          rm -rf ./*
+          cd ../non-keyed
+          rm -rf ./*
 
-      - name: Benchmark
-        run: npm run bench -- --headless
+      - name: create framework folders
+        shell: bash
+        run: |
+          cd js-framework-benchmark/frameworks/keyed
+          mkdir yew-struct
+        # Will be enabled after https://github.com/krausest/js-framework-benchmark/pull/985 gets merged
+        #  mkdir yew-hooks
 
-      - name: Results
-        run: npm run results
+      - name: copy necessary framework files
+        shell: bash
+        run: |
+          cp -r yew-struct-setup/* js-framework-benchmark/frameworks/keyed/yew-struct/
+        # Will be enabled after https://github.com/krausest/js-framework-benchmark/pull/985 gets merged
+        #  cp -r yew-hooks-setup/* js-framework-benchmark/frameworks/keyed/yew-hooks/
+
+      - name: build benchmark-struct app
+        shell: bash
+        run: |
+          cd yew/tools/benchmark-struct
+          npm ci
+          npm run build-prod-without-tools-install
+
+      - name: build benchmark-hooks app
+        shell: bash
+        run: |
+          cd yew/tools/benchmark-hooks
+          npm ci
+          npm run build-prod-without-tools-install
+
+      - name: move dist files
+        shell: bash
+        run: |
+          mv yew/tools/benchmark-struct/bundled-dist js-framework-benchmark/frameworks/keyed/yew-struct/
+        # Will be enabled after https://github.com/krausest/js-framework-benchmark/pull/985 gets merged
+        #  mv yew/tools/benchmark-hooks/bundled-dist js-framework-benchmark/frameworks/keyed/yew-hooks/
+
+      - name: js-framework-benchmark npm ci
+        shell: bash
+        run: |
+          cd js-framework-benchmark
+          npm ci
 
-      - name: Log results
+      - name: js-framework-benchmark npm start
+        shell: bash
         run: |
-          msg=$(cd results_diff && cargo run)
-          echo "$msg"
+          cd js-framework-benchmark
+          npm start &
+
+      - name: js-framework-benchmark/webdriver-ts npm ci
+        shell: bash
+        run: |
+          cd js-framework-benchmark/webdriver-ts
+          npm ci
+          npm install chromedriver --chromedriver-force-download
+
+      - name: js-framework-benchmark/webdriver-ts npm run compile
+        shell: bash
+        run: |
+          cd js-framework-benchmark/webdriver-ts
+          npm run compile
+
+      - name: js-framework-benchmark npm run build-prod
+        shell: bash
+        run: |
+          cd js-framework-benchmark
+          npm run build-prod
+
+      - name: js-framework-benchmark/webdriver-ts npm run bench
+        shell: bash
+        run: |
+          cd js-framework-benchmark/webdriver-ts
+          npm run bench -- --headless
+
+      - name: transform results into json
+        shell: bash
+        run: |
+          cd js-framework-benchmark/webdriver-ts/results
+          touch temp.txt
+          echo "[" >> temp.txt
+          for filename in *.json; do cat ${filename} >> temp.txt; echo "," >> temp.txt; done
+          sed -i '$ s/.$//' temp.txt #remove trailing comma
+          echo "]" >> temp.txt
+          mv temp.txt results.json
+
+      - name: Build process-benchmark-results
+        shell: bash
+        run: |
+          cd yew
+          cargo build --release -p process-benchmark-results
+
+      - name: transform results to be fit for display benchmark-action/github-action-benchmark@v1
+        uses: mathiasvr/command-output@v1
+        id: results
+        with:
+          run: cat js-framework-benchmark/webdriver-ts/results/results.json | ./yew/target/release/process-benchmark-results

Cargo.toml

@@ -35,6 +35,9 @@ members = [
 
   # Tools
   "tools/changelog",
+  "tools/process-benchmark-results",
+  "tools/benchmark-struct",
+  "tools/benchmark-hooks",
 ]
 exclude = [
   # Tools

tools/benchmark-hooks/.gitignore

@@ -0,0 +1,4 @@
+node_modules
+/target
+/bundled-dist
+

tools/benchmark-hooks/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "js-framework-benchmark-yew-hooks"
+version = "1.0.0"
+authors = ["Julius Lungys <juliuslungys@gmail.com>"]
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+rand = { version = "0.8.4", features = ["small_rng"] }
+getrandom = { version = "0.2.1", features = ["js"] }
+wasm-bindgen = "0.2.78"
+web-sys = { version = "0.3.55", features = ["Window"]}
+yew = "0.19.3"
+
+[profile.release]
+lto = true
+codegen-units = 1
+panic = "abort"
+
+[package.metadata.wasm-pack.profile.release]
+wasm-opt = ['-O4']

tools/benchmark-hooks/README.md

@@ -0,0 +1,6 @@
+Copied from https://github.com/krausest/js-framework-benchmark
+
+It should ideally stay/be updated to always match what is on https://github.com/krausest/js-framework-benchmark
+Except the fixes required to tun using unreleased yew version.
+
+If you want to improve benchmarks, consider first opening a PR to https://github.com/krausest/js-framework-benchmark and only then sync this package with it.

tools/benchmark-hooks/index.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <title>Yew-Hooks</title>
+    <link href="/css/currentStyle.css" rel="stylesheet"/>
+    <base href="bundled-dist/"></base>
+  </head>
+  <body>
+    <div id='main'></div>
+    <script type="module">
+      import init from './js-framework-benchmark-yew-hooks.js';
+      init('./js-framework-benchmark-yew-hooks_bg.wasm');
+    </script>
+  </body>
+</html>