WIP: Average and best of 10 values

WIth this change we run the performance tests 10 times, and record the average and the best of 10 in the performance report. This will hopefully lead to more stable results. Contributed on behalf of STMicroelectronics.
eclipse-theia · Oct 1, 2023 · 2143933 · 2143933
1 parent b8173cd
commit 2143933
Show file tree

Hide file tree

Showing 3 changed files with 136 additions and 14 deletions.
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -2,35 +2,32 @@ name: Performance
 
 on:
   workflow_dispatch:
+  pull_request: # FIXME JUST FOR TESTING
+    branches:
+      - main
   schedule:
     - cron: "0 5 * * *" # Runs every day at 5am: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule
 
 jobs:
-  build:
+  prepare:
     name: Performance benchmark on ubuntu-latest with Node.js 16.x
-
     runs-on: ubuntu-latest
-    timeout-minutes: 60
-
+    timeout-minutes: 20
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-
       - name: Use Node.js "16.x"
         uses: actions/setup-node@v3
         with:
           node-version: "16.x"
           registry-url: "https://registry.npmjs.org"
-
       - name: Use Python 3.x
         uses: actions/setup-python@v4
         with:
           python-version: "3.x"
-
       - name: Build
         shell: bash
         run: yarn
-
       - name: Checkout Theia
         uses: actions/checkout@v3
         with:
@@ -57,29 +54,145 @@ jobs:
         env:
           NODE_OPTIONS: --max_old_space_size=4096
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
-      - name: Run Theia
+      - name: Cache Build Result
+        uses: actions/cache@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+
+  run-test:
+    name: Performance benchmark on ubuntu-latest with Node.js 16.x
+    runs-on: ubuntu-latest
+    needs: [prepare]
+    strategy:
+      matrix:
+        run: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    timeout-minutes: 40
+    steps:
+      - name: Restore Build Result (#${{ matrix.run }})
+        uses: actions/cache/restore@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+          fail-on-cache-miss: true
+      - name: Run Theia (#${{ matrix.run }})
         shell: bash
         working-directory: ./theia
         run: yarn browser start:debug &
-
-      - name: Run Performance Measurement
+      - name: Run Performance Measurement (#${{ matrix.run }})
         uses: GabrielBB/xvfb-action@v1
+        env:
+          RUN_NO: ${{ matrix.run }}
         with:
           run: yarn performance
+      # - name: Cache Performance Measurement (#${{ matrix.run }})
+      #   uses: actions/cache@v3
+      #   id: performance-measurement-${{ matrix.run }}
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-${{ matrix.run }}
 
+  generate-report:
+    name: Performance benchmark on ubuntu-latest with Node.js 16.x
+    runs-on: ubuntu-latest
+    needs: [run-test]
+    steps:
+      - name: Restore Build Result
+        uses: actions/cache/restore@v3
+        id: build-result
+        with:
+          path: ./*
+          key: ${{ github.run_number }}
+          fail-on-cache-miss: true
+      # ===============================================================================================
+      # START RESTORING PERFORMANCE MEASUREMENTS FROM CACHE
+      # we have to restore the performance measurement of each run individually
+      # so the following restore steps need to be aligned with the matrix.run variable in the job above
+      # - name: Restore Performance Measurement 0
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-0
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-0
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 1
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-1
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-1
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 2
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-2
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-2
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 3
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-3
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-3
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 4
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-4
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-4
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 5
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-5
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-5
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 6
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-6
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-6
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 7
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-7
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-7
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 8
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-8
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-8
+      #     fail-on-cache-miss: true
+      # - name: Restore Performance Measurement 9
+      #   uses: actions/cache/restore@v3
+      #   id: performance-measurement-9
+      #   with:
+      #     path: ./performance-metrics
+      #     key: ${{ github.run_number }}-9
+      #     fail-on-cache-miss: true
+      # END RESTORING PERFORMANCE MEASUREMENTS FROM CACHE
+      # ===============================================================================================
       - name: Get History
         uses: actions/checkout@v2
         if: always() && github.ref == 'refs/heads/main'
         continue-on-error: true
         with:
           ref: gh-pages
           path: gh-pages
-
       - name: Prepare Report
         if: always() && github.ref == 'refs/heads/main'
         shell: bash
         run: yarn performance-report
-
       - name: Publish Report
         if: always() && github.ref == 'refs/heads/main'
         uses: peaceiris/actions-gh-pages@v2

diff --git a/scripts/fetch-metrics.ts b/scripts/fetch-metrics.ts
@@ -42,7 +42,8 @@ export async function fetchPerformanceMetrics({
     const dateString = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`;
     const timeString = `${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`;
     const timestamp = `${dateString}T${timeString}`;
-    const fileName = outputFileNamePrefix + timestamp + outputFileNamePostfix;
+    const runNo = process.env.RUN_NO ? `_${process.env.RUN_NO}` : '';
+    const fileName = outputFileNamePrefix + timestamp + runNo + outputFileNamePostfix;
     const targetFile = `${outputFilePath}/${fileName}`;
 
     fs.ensureDirSync(outputFilePath);

diff --git a/scripts/performance-report.ts b/scripts/performance-report.ts
@@ -102,6 +102,14 @@ export async function generatePerformanceReport(path: string) {
         'process_cpu_seconds_total',
         'playwright_total_time'
     ]);
+
+    // TODO Post process values:
+    // if label ends with _X
+    //  take current as label
+    //  collect all values until X of _X is smaller than previous or doesn't exist
+    //  replace collected with one that has a computed average and best of 1ß value of collected
+    // else: set current value as average and best of 10
+
     const charts: string[] = [];
     for (const [valueLabel, valueHistory] of values) {
         const data = valueHistory.history.map(entry => ({ x: entry.entryLabel, y: entry.value }));