diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 1706398cd..05ed52e96 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -2,35 +2,32 @@ name: Performance on: workflow_dispatch: + pull_request: # FIXME JUST FOR TESTING + branches: + - main schedule: - cron: "0 5 * * *" # Runs every day at 5am: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events-schedule jobs: - build: + prepare: name: Performance benchmark on ubuntu-latest with Node.js 16.x - runs-on: ubuntu-latest - timeout-minutes: 60 - + timeout-minutes: 20 steps: - name: Checkout uses: actions/checkout@v3 - - name: Use Node.js "16.x" uses: actions/setup-node@v3 with: node-version: "16.x" registry-url: "https://registry.npmjs.org" - - name: Use Python 3.x uses: actions/setup-python@v4 with: python-version: "3.x" - - name: Build shell: bash run: yarn - - name: Checkout Theia uses: actions/checkout@v3 with: @@ -57,16 +54,134 @@ jobs: env: NODE_OPTIONS: --max_old_space_size=4096 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9 - - name: Run Theia + - name: Cache Build Result + uses: actions/cache@v3 + id: build-result + with: + path: ./* + key: ${{ github.run_number }} + + run-test: + name: Performance benchmark on ubuntu-latest with Node.js 16.x + runs-on: ubuntu-latest + needs: [prepare] + strategy: + matrix: + run: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + timeout-minutes: 40 + steps: + - name: Restore Build Result (#${{ matrix.run }}) + uses: actions/cache/restore@v3 + id: build-result + with: + path: ./* + key: ${{ github.run_number }} + fail-on-cache-miss: true + - name: Run Theia (#${{ matrix.run }}) shell: bash working-directory: ./theia run: yarn browser start:debug & - - - name: Run Performance Measurement + - name: Run Performance Measurement (#${{ matrix.run }}) uses: GabrielBB/xvfb-action@v1 + env: + RUN_NO: ${{ matrix.run }} with: run: yarn performance + # - name: Cache Performance Measurement (#${{ matrix.run }}) + # uses: actions/cache@v3 + # id: performance-measurement-${{ matrix.run }} + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-${{ matrix.run }} + generate-report: + name: Performance benchmark on ubuntu-latest with Node.js 16.x + runs-on: ubuntu-latest + needs: [run-test] + steps: + - name: Restore Build Result + uses: actions/cache/restore@v3 + id: build-result + with: + path: ./* + key: ${{ github.run_number }} + fail-on-cache-miss: true + # =============================================================================================== + # START RESTORING PERFORMANCE MEASUREMENTS FROM CACHE + # we have to restore the performance measurement of each run individually + # so the following restore steps need to be aligned with the matrix.run variable in the job above + # - name: Restore Performance Measurement 0 + # uses: actions/cache/restore@v3 + # id: performance-measurement-0 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-0 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 1 + # uses: actions/cache/restore@v3 + # id: performance-measurement-1 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-1 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 2 + # uses: actions/cache/restore@v3 + # id: performance-measurement-2 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-2 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 3 + # uses: actions/cache/restore@v3 + # id: performance-measurement-3 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-3 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 4 + # uses: actions/cache/restore@v3 + # id: performance-measurement-4 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-4 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 5 + # uses: actions/cache/restore@v3 + # id: performance-measurement-5 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-5 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 6 + # uses: actions/cache/restore@v3 + # id: performance-measurement-6 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-6 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 7 + # uses: actions/cache/restore@v3 + # id: performance-measurement-7 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-7 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 8 + # uses: actions/cache/restore@v3 + # id: performance-measurement-8 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-8 + # fail-on-cache-miss: true + # - name: Restore Performance Measurement 9 + # uses: actions/cache/restore@v3 + # id: performance-measurement-9 + # with: + # path: ./performance-metrics + # key: ${{ github.run_number }}-9 + # fail-on-cache-miss: true + # END RESTORING PERFORMANCE MEASUREMENTS FROM CACHE + # =============================================================================================== - name: Get History uses: actions/checkout@v2 if: always() && github.ref == 'refs/heads/main' @@ -74,12 +189,10 @@ jobs: with: ref: gh-pages path: gh-pages - - name: Prepare Report if: always() && github.ref == 'refs/heads/main' shell: bash run: yarn performance-report - - name: Publish Report if: always() && github.ref == 'refs/heads/main' uses: peaceiris/actions-gh-pages@v2 diff --git a/scripts/fetch-metrics.ts b/scripts/fetch-metrics.ts index cf4b02eed..74edfba6b 100644 --- a/scripts/fetch-metrics.ts +++ b/scripts/fetch-metrics.ts @@ -42,7 +42,8 @@ export async function fetchPerformanceMetrics({ const dateString = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`; const timeString = `${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`; const timestamp = `${dateString}T${timeString}`; - const fileName = outputFileNamePrefix + timestamp + outputFileNamePostfix; + const runNo = process.env.RUN_NO ? `_${process.env.RUN_NO}` : ''; + const fileName = outputFileNamePrefix + timestamp + runNo + outputFileNamePostfix; const targetFile = `${outputFilePath}/${fileName}`; fs.ensureDirSync(outputFilePath); diff --git a/scripts/performance-report.ts b/scripts/performance-report.ts index c494bd421..902f35cb5 100644 --- a/scripts/performance-report.ts +++ b/scripts/performance-report.ts @@ -102,6 +102,14 @@ export async function generatePerformanceReport(path: string) { 'process_cpu_seconds_total', 'playwright_total_time' ]); + + // TODO Post process values: + // if label ends with _X + // take current as label + // collect all values until X of _X is smaller than previous or doesn't exist + // replace collected with one that has a computed average and best of 1ß value of collected + // else: set current value as average and best of 10 + const charts: string[] = []; for (const [valueLabel, valueHistory] of values) { const data = valueHistory.history.map(entry => ({ x: entry.entryLabel, y: entry.value }));