Continous Benchmarking Action (#1137)

* ci: continous benchmarking action * fix pytorch install command * fixed action python version * added github token to benchmark action * test message worst performance * ci: test gh-pages results from cb action * ci: this should pass * fix: typo * remove cache action in favor of gh-pages * extra / * ci: action config should work now * extra benchmarks from linalg module * quicker linalg benchmarks * cluster benchmarks, removed some code duplication * corrected cb workflow command * ci: split cb workflow into a main and pr. pr workflow triggers with a 'run bench' comment * ci: bench pr runs after a review is requested * ci: bench now only triggers on pull requests with 'PR talk' tag * ci: reshape benchmark, removed cronjob from pytorch workflows, renamed 'old' benchmarking folder to '2020' * mend * fix: missing import in cb main * ci: changed benchmark python version to 3.10, added shield in readme pointing to the benchmarks, changed trigger tag * fix: incorrect python version param --------- Co-authored-by: Claudia Comito <39374113+ClaudiaComito@users.noreply.github.com>
helmholtz-analytics · Apr 27, 2023 · 02d2492 · 02d2492
1 parent 9f6b2eb
commit 02d2492
Show file tree

Hide file tree

Showing 56 changed files with 322 additions and 171 deletions.
diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml
@@ -5,40 +5,94 @@ categories:
   - title: '🐛 Bug Fixes'
     labels:
       - 'bug :bug:'
+  - title: 'Google Summer of Code 2022'
+    label: 'GSoC22'
+  - title: '💯 Benchmarking'
+    label: 'benchmarking'
+  - title: '📜 Documentation'
+    label: 'documentation :book:'
+  - title: '🧹 Maintenance'
+    label: 'chore'
+  - title: '🧪 Testing'
+    label: 'testing'
   - title: 'Arithmetic'
     label: 'arithmetic'
   - title: 'Array API'
     label: 'array API'
+  - title: 'Classification'
+    label: 'classification'
+  - title: 'Cluster'
+    label: 'cluster'
   - title: 'Communication'
     labels:
       - 'io'
       - 'I/O'
       - 'communication'
+  - title: 'Complex Math'
+    label: 'complex math'
+  - title: 'Constants'
+    label: 'constants'
+  - title: 'Datasets'
+    label: 'datasets'
+  - title: 'Devices'
+    label: 'devices'
   - title: 'DNDarray'
     label: 'dndarray'
+  - title: 'Exponential'
+    label: 'exponential'
+  - title: 'Factories'
+    label: 'factories'
+  - title: 'Graph'
+    label: 'graph'
+  - title: 'Indexing'
+    label: 'indexing'
   - title: 'Linear Algebra'
     label: 'linalg'
   - title: 'Logical'
     label: 'logical'
   - title: 'Manipulations'
     label: 'manipulation'
+  - title: 'Memory'
+    label: 'memory'
+  - title: 'Naive Bayes'
+    label: 'naive Bayes'
+  - title: 'Neural Networks'
+    labels:
+      - 'nn'
+      - 'neural net'
+  - title: 'Optimization'
+    label: 'optim'
+  - title: 'Printing'
+    label: 'printing'
   - title: 'Random'
     label: 'random'
+  - title: 'Regression'
+    label: 'regression'
+  - title: 'Relational'
+    label: 'relational'
+  - title: 'Rounding'
+    label: 'rounding'
+  - title: 'Sanitation'
+    label: 'sanitation'
+  - title: 'Signal'
+    label: 'signal'
   - title: 'Sparse'
     label: 'sparse'
-  - title: 'Google Summer of Code 2022'
-    label: 'GSoC22'
-  - title: '💯 Benchmarking'
-    label: 'benchmarking'
-  - title: '📜 Documentation'
-    label: 'documentation :book:'
-  - title: '🧹 Maintenance'
-    label: 'chore'
-  - title: '🧪 Testing'
-    label: 'testing'
+  - title: 'Spatial'
+    label: 'spatial'
+  - title: 'Statistics'
+    label: 'statistics'
+  - title: 'Stride Tricks'
+    label: 'stride tricks'
+  - title: 'Tiling'
+    label: 'tiling'
+  - title: 'Trigonometrics'
+    label: 'trigonometrics'
+  - title: 'Types'
+    label: 'types'
 
 change-template: '- #$NUMBER $TITLE (by @$AUTHOR)'
-categorie-template: '### $TITLE'
+category-template: '### $TITLE'
 exclude-labels:
 - 'workflow'
 template: |

diff --git a/.github/workflows/benchmark_main.yml b/.github/workflows/benchmark_main.yml
@@ -0,0 +1,45 @@
+name: Benchmark main and save
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  benchmark-main:
+    name: Benchmark main and save
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup MPI
+        uses: mpi4py/setup-mpi@v1
+      - name: Use Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.11 # Perun only supports 3.8 and ahead
+          architecture: x64
+      - name: Test
+        run: |
+            pip install torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html
+            pip install .[cb]
+            PERUN_RUN_ID=N4 mpirun -n 4 python benchmarks/cb/main.py
+            jq -s flatten bench_data/*.json > bench_data/all_benchmarks.json
+      - name: Save benchmark result and update gh-pages-chart
+        if: ${{github.ref == 'refs/heads/main'}}
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          # Benchmark action input and output
+          tool: 'customSmallerIsBetter'
+          output-file-path: bench_data/all_benchmarks.json
+          # external-data-json-path: ./cache/benchmark-data.json
+          # Alert configuration
+          fail-on-alert: false  # Don't fail on main branch
+          comment-on-alert: true
+          # Save benchmarks from the main branch
+          save-data-file: true
+          # Pages configuration
+          auto-push: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: dev/bench
+      # Upload the updated cache file for the next job by actions/cache
diff --git a/.github/workflows/benchmark_pr.yml b/.github/workflows/benchmark_pr.yml
@@ -0,0 +1,45 @@
+name: Benchmark PR
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    branches: [main]
+
+jobs:
+  benchmark-pr:
+    name: Benchmark PR
+    if: contains(github.event.pull_request.labels.*.name, 'benchmark PR')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup MPI
+        uses: mpi4py/setup-mpi@v1
+      - name: Use Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.11 # Perun only supports 3.8 and ahead
+          architecture: x64
+      - name: Test
+        run: |
+            pip install torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html
+            pip install .[cb]
+            PERUN_RUN_ID=N4 mpirun -n 4 python benchmarks/cb/main.py
+            jq -s flatten bench_data/*.json > bench_data/all_benchmarks.json
+      - name: Compare benchmark result
+        if: ${{github.ref != 'refs/heads/main'}}
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          # Benchmark action input and output
+          tool: 'customSmallerIsBetter'
+          output-file-path: bench_data/all_benchmarks.json
+          # external-data-json-path: ./cache/benchmark-data.json
+          # Alert configuration
+          fail-on-alert: true
+          comment-on-alert: true
+          # Ignore results from non main branches.
+          save-data-file: false
+          # Pages configuration
+          auto-push: false
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: dev/bench
diff --git a/.github/workflows/pytorch-latest-main.yml b/.github/workflows/pytorch-latest-main.yml
@@ -1,8 +1,6 @@
 name: Get latest PyTorch version main branch
 on:
   workflow_dispatch:
-  schedule:
-    - cron:  '0 3 * * 1,4'
 env:
   working_branch: workflows/new-pytorch-main-branch
   base_branch: main

diff --git a/.github/workflows/pytorch-latest-release.yml b/.github/workflows/pytorch-latest-release.yml
@@ -1,8 +1,6 @@
 name: Get latest PyTorch version release branch
 on:
   workflow_dispatch:
-  schedule:
-    - cron:  '0 3 * * 1,4'
 env:
   working_branch: workflows/new-pytorch-release-branch
   base_branch: release/1.2.x

diff --git a/.gitignore b/.gitignore
@@ -305,3 +305,5 @@ MANIFEST
 travis_wait_*.log
 testing.py
 heat/datasets/MNISTDataset
+perun_results/
+bench_data/
diff --git a/.perun.ini b/.perun.ini
@@ -0,0 +1,8 @@
+[output]
+format = bench
+data_out = ./bench_data
+
+[benchmarking]
+bench_enable = True
+bench_rounds = 10
+bench_warmup_rounds = 1
diff --git a/README.md b/README.md
@@ -15,6 +15,7 @@ Heat is a distributed tensor framework for high performance data analytics.
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![license: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
 [![Downloads](https://pepy.tech/badge/heat)](https://pepy.tech/project/heat)
+[![Github-Pages - Benchmarks](https://img.shields.io/badge/Github--Pages-Benchmarks-2ea44f)](https://helmholtz-analytics.github.io/heat/dev/bench)
 
 # New
 - **NO GSoC WITH US THIS YEAR**. On Feb 21st we learned that our Organisation has not been selected this year. Congratulations to the [GSoC 2023 Mentoring Orgs](https://summerofcode.withgoogle.com/programs/2023/organizations)!

diff --git a/benchmarks/__init__.py → benchmarks/2020/__init__.py b/benchmarks/__init__.py → benchmarks/2020/__init__.py
diff --git a/benchmarks/distance_matrix/__init__.py → benchmarks/2020/distance_matrix/__init__.py b/benchmarks/distance_matrix/__init__.py → benchmarks/2020/distance_matrix/__init__.py
diff --git a/benchmarks/distance_matrix/config.json → benchmarks/2020/distance_matrix/config.json b/benchmarks/distance_matrix/config.json → benchmarks/2020/distance_matrix/config.json
diff --git a/benchmarks/distance_matrix/dask-auto-cpu.py → ...rks/2020/distance_matrix/dask-auto-cpu.py b/benchmarks/distance_matrix/dask-auto-cpu.py → ...rks/2020/distance_matrix/dask-auto-cpu.py
diff --git a/...hmarks/distance_matrix/dask-chunks-cpu.py → ...s/2020/distance_matrix/dask-chunks-cpu.py b/...hmarks/distance_matrix/dask-chunks-cpu.py → ...s/2020/distance_matrix/dask-chunks-cpu.py
diff --git a/benchmarks/distance_matrix/heat-cpu.py → benchmarks/2020/distance_matrix/heat-cpu.py b/benchmarks/distance_matrix/heat-cpu.py → benchmarks/2020/distance_matrix/heat-cpu.py
diff --git a/benchmarks/distance_matrix/heat-gpu.py → benchmarks/2020/distance_matrix/heat-gpu.py b/benchmarks/distance_matrix/heat-gpu.py → benchmarks/2020/distance_matrix/heat-gpu.py
diff --git a/benchmarks/distance_matrix/numpy-cpu.py → benchmarks/2020/distance_matrix/numpy-cpu.py b/benchmarks/distance_matrix/numpy-cpu.py → benchmarks/2020/distance_matrix/numpy-cpu.py
diff --git a/benchmarks/distance_matrix/torch-cpu.py → benchmarks/2020/distance_matrix/torch-cpu.py b/benchmarks/distance_matrix/torch-cpu.py → benchmarks/2020/distance_matrix/torch-cpu.py
diff --git a/benchmarks/distance_matrix/torch-gpu.py → benchmarks/2020/distance_matrix/torch-gpu.py b/benchmarks/distance_matrix/torch-gpu.py → benchmarks/2020/distance_matrix/torch-gpu.py
diff --git a/benchmarks/generate_jobscripts.py → benchmarks/2020/generate_jobscripts.py b/benchmarks/generate_jobscripts.py → benchmarks/2020/generate_jobscripts.py
diff --git a/benchmarks/kmeans/__init__.py → benchmarks/2020/kmeans/__init__.py b/benchmarks/kmeans/__init__.py → benchmarks/2020/kmeans/__init__.py
diff --git a/benchmarks/kmeans/config.json → benchmarks/2020/kmeans/config.json b/benchmarks/kmeans/config.json → benchmarks/2020/kmeans/config.json
diff --git a/benchmarks/kmeans/dask-auto-cpu.py → benchmarks/2020/kmeans/dask-auto-cpu.py b/benchmarks/kmeans/dask-auto-cpu.py → benchmarks/2020/kmeans/dask-auto-cpu.py
diff --git a/benchmarks/kmeans/dask-chunks-cpu.py → benchmarks/2020/kmeans/dask-chunks-cpu.py b/benchmarks/kmeans/dask-chunks-cpu.py → benchmarks/2020/kmeans/dask-chunks-cpu.py
diff --git a/benchmarks/kmeans/heat-cpu.py → benchmarks/2020/kmeans/heat-cpu.py b/benchmarks/kmeans/heat-cpu.py → benchmarks/2020/kmeans/heat-cpu.py
diff --git a/benchmarks/kmeans/heat-gpu.py → benchmarks/2020/kmeans/heat-gpu.py b/benchmarks/kmeans/heat-gpu.py → benchmarks/2020/kmeans/heat-gpu.py
diff --git a/benchmarks/kmeans/numpy-cpu.py → benchmarks/2020/kmeans/numpy-cpu.py b/benchmarks/kmeans/numpy-cpu.py → benchmarks/2020/kmeans/numpy-cpu.py
diff --git a/benchmarks/kmeans/torch-cpu.py → benchmarks/2020/kmeans/torch-cpu.py b/benchmarks/kmeans/torch-cpu.py → benchmarks/2020/kmeans/torch-cpu.py
diff --git a/benchmarks/kmeans/torch-gpu.py → benchmarks/2020/kmeans/torch-gpu.py b/benchmarks/kmeans/torch-gpu.py → benchmarks/2020/kmeans/torch-gpu.py
diff --git a/benchmarks/lasso/__init__.py → benchmarks/2020/lasso/__init__.py b/benchmarks/lasso/__init__.py → benchmarks/2020/lasso/__init__.py
diff --git a/benchmarks/lasso/config.json → benchmarks/2020/lasso/config.json b/benchmarks/lasso/config.json → benchmarks/2020/lasso/config.json
diff --git a/benchmarks/lasso/dask-auto-cpu.py → benchmarks/2020/lasso/dask-auto-cpu.py b/benchmarks/lasso/dask-auto-cpu.py → benchmarks/2020/lasso/dask-auto-cpu.py
diff --git a/benchmarks/lasso/dask-chunks-cpu.py → benchmarks/2020/lasso/dask-chunks-cpu.py b/benchmarks/lasso/dask-chunks-cpu.py → benchmarks/2020/lasso/dask-chunks-cpu.py
diff --git a/benchmarks/lasso/heat-cpu.py → benchmarks/2020/lasso/heat-cpu.py b/benchmarks/lasso/heat-cpu.py → benchmarks/2020/lasso/heat-cpu.py
diff --git a/benchmarks/lasso/heat-gpu.py → benchmarks/2020/lasso/heat-gpu.py b/benchmarks/lasso/heat-gpu.py → benchmarks/2020/lasso/heat-gpu.py
diff --git a/benchmarks/lasso/numpy-cpu.py → benchmarks/2020/lasso/numpy-cpu.py b/benchmarks/lasso/numpy-cpu.py → benchmarks/2020/lasso/numpy-cpu.py
diff --git a/benchmarks/lasso/torch-cpu.py → benchmarks/2020/lasso/torch-cpu.py b/benchmarks/lasso/torch-cpu.py → benchmarks/2020/lasso/torch-cpu.py
diff --git a/benchmarks/lasso/torch-gpu.py → benchmarks/2020/lasso/torch-gpu.py b/benchmarks/lasso/torch-gpu.py → benchmarks/2020/lasso/torch-gpu.py
diff --git a/benchmarks/statistical_moments/__init__.py → ...arks/2020/statistical_moments/__init__.py b/benchmarks/statistical_moments/__init__.py → ...arks/2020/statistical_moments/__init__.py
diff --git a/benchmarks/statistical_moments/config.json → ...arks/2020/statistical_moments/config.json b/benchmarks/statistical_moments/config.json → ...arks/2020/statistical_moments/config.json
diff --git a/...arks/statistical_moments/dask-auto-cpu.py → ...2020/statistical_moments/dask-auto-cpu.py b/...arks/statistical_moments/dask-auto-cpu.py → ...2020/statistical_moments/dask-auto-cpu.py
diff --git a/...ks/statistical_moments/dask-chunks-cpu.py → ...20/statistical_moments/dask-chunks-cpu.py b/...ks/statistical_moments/dask-chunks-cpu.py → ...20/statistical_moments/dask-chunks-cpu.py
diff --git a/benchmarks/statistical_moments/heat-cpu.py → ...arks/2020/statistical_moments/heat-cpu.py b/benchmarks/statistical_moments/heat-cpu.py → ...arks/2020/statistical_moments/heat-cpu.py
diff --git a/benchmarks/statistical_moments/heat-gpu.py → ...arks/2020/statistical_moments/heat-gpu.py b/benchmarks/statistical_moments/heat-gpu.py → ...arks/2020/statistical_moments/heat-gpu.py
diff --git a/benchmarks/statistical_moments/numpy-cpu.py → ...rks/2020/statistical_moments/numpy-cpu.py b/benchmarks/statistical_moments/numpy-cpu.py → ...rks/2020/statistical_moments/numpy-cpu.py
diff --git a/benchmarks/statistical_moments/torch-cpu.py → ...rks/2020/statistical_moments/torch-cpu.py b/benchmarks/statistical_moments/torch-cpu.py → ...rks/2020/statistical_moments/torch-cpu.py
diff --git a/benchmarks/statistical_moments/torch-gpu.py → ...rks/2020/statistical_moments/torch-gpu.py b/benchmarks/statistical_moments/torch-gpu.py → ...rks/2020/statistical_moments/torch-gpu.py
diff --git a/benchmarks/cb/cluster.py b/benchmarks/cb/cluster.py
@@ -0,0 +1,30 @@
+import heat as ht
+from perun.decorator import monitor
+
+
+@monitor()
+def kmeans_cpu(data):
+    kmeans = ht.cluster.KMeans(n_clusters=4, init="kmeans++")
+    kmeans.fit(data)
+
+
+@monitor()
+def kmedians_cpu(data):
+    kmeans = ht.cluster.KMedians(n_clusters=4, init="kmedians++")
+    kmeans.fit(data)
+
+
+@monitor()
+def kmedoids_cpu(data):
+    kmeans = ht.cluster.KMedoids(n_clusters=4, init="kmedoids++")
+    kmeans.fit(data)
+
+
+n = 5000
+seed = 1
+data = ht.utils.data.spherical.create_spherical_dataset(
+    num_samples_cluster=n, radius=1.0, offset=4.0, dtype=ht.float32, random_state=seed
+)
+kmeans_cpu(data)
+kmedians_cpu(data)
+kmedoids_cpu(data)
diff --git a/benchmarks/cb/linalg.py b/benchmarks/cb/linalg.py
@@ -0,0 +1,38 @@
+# flake8: noqa
+import heat as ht
+from perun.decorator import monitor
+
+
+@monitor()
+def matmul_cpu_split_0(n: int = 3000):
+    a = ht.random.random((n, n), split=0, device="cpu")
+    b = ht.random.random((n, n), split=0, device="cpu")
+    a @ b
+
+
+@monitor()
+def matmul_cpu_split_1(n: int = 3000):
+    a = ht.random.random((n, n), split=1, device="cpu")
+    b = ht.random.random((n, n), split=1, device="cpu")
+    a @ b
+
+
+@monitor()
+def qr_cpu(n: int = 2000):
+    for t in range(1, 3):
+        for sp in range(2):
+            a = ht.random.random((n, n), split=sp)
+            qr = a.qr(tiles_per_proc=t)
+
+
+@monitor()
+def lanczos_cpu(n: int = 50):
+    A = ht.random.random((n, n), dtype=ht.float64, split=0)
+    B = A @ A.T
+    V, T = ht.lanczos(B, m=n)
+
+
+matmul_cpu_split_0()
+matmul_cpu_split_1()
+qr_cpu()
+lanczos_cpu()
diff --git a/benchmarks/cb/main.py b/benchmarks/cb/main.py
@@ -0,0 +1,4 @@
+# flake8: noqa
+import linalg
+import cluster
+import manipulations
diff --git a/benchmarks/cb/manipulations.py b/benchmarks/cb/manipulations.py
@@ -0,0 +1,14 @@
+# flake8: noqa
+import heat as ht
+from perun.decorator import monitor
+
+
+@monitor()
+def reshape_cpu():
+    sizes = [10000, 20000, 40000]
+    for size in sizes:
+        st = ht.zeros((1000, size), split=1)
+        a = ht.reshape(st, (10000000, -1), new_split=1)
+
+
+reshape_cpu()