chore(bench): add throughput benchmarks to suite

All integer benchmarks make recipes can be run to ouput throughput results. Only CPU is supported for throughput benchmarks in GitHub CI.
zama-ai · Nov 19, 2024 · 9059dde · 9059dde
1 parent 904ffa7
commit 9059dde
Show file tree

Hide file tree

Showing 23 changed files with 1,928 additions and 741 deletions.
diff --git a/.github/workflows/benchmark_boolean.yml b/.github/workflows/benchmark_boolean.yml
@@ -80,8 +80,7 @@ jobs:
           --commit-date "${{ env.COMMIT_DATE }}" \
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
-          --name-suffix avx512 \
-          --throughput
+          --name-suffix avx512
 
       - name: Measure key sizes
         run: |

diff --git a/.github/workflows/benchmark_core_crypto.yml b/.github/workflows/benchmark_core_crypto.yml
@@ -78,8 +78,7 @@ jobs:
           --commit-date "${{ env.COMMIT_DATE }}" \
           --bench-date "${{ env.BENCH_DATE }}" \
           --name-suffix avx512 \
-          --walk-subdirs \
-          --throughput
+          --walk-subdirs
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_4090.yml b/.github/workflows/benchmark_gpu_4090.yml
@@ -79,8 +79,7 @@ jobs:
           --branch ${{ github.ref_name }} \
           --commit-date "${{ env.COMMIT_DATE }}" \
           --bench-date "${{ env.BENCH_DATE }}" \
-          --walk-subdirs \
-          --throughput
+          --walk-subdirs
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
@@ -154,7 +153,7 @@ jobs:
           --commit-date "${{ env.COMMIT_DATE }}" \
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
-          --throughput
+      
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_core_crypto.yml b/.github/workflows/benchmark_gpu_core_crypto.yml
@@ -64,7 +64,6 @@ jobs:
           make -j"$(nproc)"
           sudo make install
 
-
       - name: Checkout tfhe-rs repo with tags
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
         with:
@@ -126,8 +125,7 @@ jobs:
           --commit-date "${{ env.COMMIT_DATE }}" \
           --bench-date "${{ env.BENCH_DATE }}" \
           --name-suffix avx512 \
-          --walk-subdirs \
-          --throughput
+          --walk-subdirs
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_integer.yml b/.github/workflows/benchmark_gpu_integer.yml
@@ -3,6 +3,12 @@ name: Integer GPU benchmarks
 
 on:
   workflow_dispatch:
+    inputs:
+      run_throughput:
+        description: "Run throughput benchmarks"
+        type: boolean
+        default: false
+
   push:
     branches:
       - main
@@ -18,6 +24,7 @@ env:
   SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
   SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+  BENCH_TYPE: latency
 
 jobs:
   setup-instance:
@@ -115,6 +122,11 @@ jobs:
         if: ${{ !cancelled() }}
         run: nvidia-smi
 
+      - name: Should run throughput benchmarks
+        if: inputs.run_throughput
+        run: |
+          echo "BENCH_TYPE=throughput" >> "${GITHUB_ENV}"
+
       - name: Run benchmarks with AVX512
         run: |
           make FAST_BENCH=TRUE BENCH_OP_FLAVOR=default bench_integer_gpu
@@ -142,7 +154,7 @@ jobs:
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
           --name-suffix avx512 \
-          --throughput
+          --bench-type ${{ env.BENCH_TYPE }}
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_integer_2H100_full.yml b/.github/workflows/benchmark_gpu_integer_2H100_full.yml
@@ -3,6 +3,12 @@ name: Integer 2xH100 benchmarks
 
 on:
   workflow_dispatch:
+    inputs:
+      run_throughput:
+        description: "Run throughput benchmarks"
+        type: boolean
+        default: false
+
   schedule:
     # Weekly benchmarks will be triggered each Saturday at 1a.m.
     - cron: '0 1 * * 6'
@@ -17,6 +23,7 @@ env:
   SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
   SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+  BENCH_TYPE: latency
 
 jobs:
   setup-instance:
@@ -126,6 +133,11 @@ jobs:
         if: ${{ !cancelled() }}
         run: nvidia-smi
 
+      - name: Should run throughput benchmarks
+        if: inputs.run_throughput
+        run: |
+          echo "BENCH_TYPE=throughput" >> "${GITHUB_ENV}"
+
       - name: Run benchmarks with AVX512
         run: |
           make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} bench_${{ matrix.command }}_gpu
@@ -142,7 +154,7 @@ jobs:
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
           --name-suffix avx512 \
-          --throughput
+          --bench-type ${{ env.BENCH_TYPE }}
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_integer_full.yml b/.github/workflows/benchmark_gpu_integer_full.yml
@@ -3,6 +3,12 @@ name: Integer GPU full benchmarks
 
 on:
   workflow_dispatch:
+    inputs:
+      run_throughput:
+        description: "Run throughput benchmarks"
+        type: boolean
+        default: false
+
   schedule:
     # Weekly benchmarks will be triggered each Saturday at 1a.m.
     - cron: '0 1 * * 6'
@@ -17,6 +23,7 @@ env:
   SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
   SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+  BENCH_TYPE: latency
 
 jobs:
   setup-instance:
@@ -126,6 +133,11 @@ jobs:
         if: ${{ !cancelled() }}
         run: nvidia-smi
 
+      - name: Should run throughput benchmarks
+        if: inputs.run_throughput
+        run: |
+          echo "BENCH_TYPE=throughput" >> "${GITHUB_ENV}"
+
       - name: Run benchmarks with AVX512
         run: |
           make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} bench_${{ matrix.command }}_gpu
@@ -148,7 +160,7 @@ jobs:
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
           --name-suffix avx512 \
-          --throughput
+          --bench-type ${{ env.BENCH_TYPE }}
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_integer_multi_bit.yml b/.github/workflows/benchmark_gpu_integer_multi_bit.yml
@@ -12,6 +12,10 @@ on:
         description: "Run only deduplicated default operations without scalar variants"
         type: boolean
         default: false
+      run_throughput:
+        description: "Run throughput benchmarks"
+        type: boolean
+        default: false
 
   schedule:
     # Weekly benchmarks will be triggered each Saturday at 1a.m.
@@ -30,6 +34,7 @@ env:
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
   FAST_BENCH: TRUE
   BENCH_OP_FLAVOR: default
+  BENCH_TYPE: latency
 
 jobs:
   setup-instance:
@@ -138,6 +143,11 @@ jobs:
         if: ${{ !cancelled() }}
         run: nvidia-smi
 
+      - name: Should run throughput benchmarks
+        if: inputs.run_throughput
+        run: |
+          echo "BENCH_TYPE=throughput" >> "${GITHUB_ENV}"
+
       - name: Run multi-bit benchmarks with AVX512
         run: |
           make bench_unsigned_integer_multi_bit_gpu
@@ -165,7 +175,7 @@ jobs:
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
           --name-suffix avx512 \
-          --throughput
+          --bench-type ${{ env.BENCH_TYPE }}
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_integer_multi_bit_multi_gpu.yml b/.github/workflows/benchmark_gpu_integer_multi_bit_multi_gpu.yml
@@ -12,6 +12,10 @@ on:
         description: "Run only deduplicated default operations without scalar variants"
         type: boolean
         default: false
+      run_throughput:
+        description: "Run throughput benchmarks"
+        type: boolean
+        default: false
 
   schedule:
     # Weekly benchmarks will be triggered each Saturday at 1a.m.
@@ -29,6 +33,7 @@ env:
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
   FAST_BENCH: TRUE
   BENCH_OP_FLAVOR: default
+  BENCH_TYPE: latency
 
 jobs:
   setup-instance:
@@ -146,6 +151,11 @@ jobs:
         if: ${{ !cancelled() }}
         run: nvidia-smi
 
+      - name: Should run throughput benchmarks
+        if: inputs.run_throughput
+        run: |
+          echo "BENCH_TYPE=throughput" >> "${GITHUB_ENV}"
+
       - name: Run multi-bit benchmarks with AVX512
         run: |
           make bench_unsigned_integer_multi_bit_gpu
@@ -162,7 +172,7 @@ jobs:
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
           --name-suffix avx512 \
-          --throughput
+          --bench-type ${{ env.BENCH_TYPE }}
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_integer_multi_gpu_full.yml b/.github/workflows/benchmark_gpu_integer_multi_gpu_full.yml
@@ -3,6 +3,12 @@ name: Integer multi GPU full benchmarks
 
 on:
   workflow_dispatch:
+    inputs:
+      run_throughput:
+        description: "Run throughput benchmarks"
+        type: boolean
+        default: false
+
   schedule:
     # Weekly benchmarks will be triggered each Saturday at 1a.m.
     - cron: '0 1 * * 6'
@@ -17,6 +23,7 @@ env:
   SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
   SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
   SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+  BENCH_TYPE: latency
 
 jobs:
   setup-instance:
@@ -126,6 +133,11 @@ jobs:
         if: ${{ !cancelled() }}
         run: nvidia-smi
 
+      - name: Should run throughput benchmarks
+        if: inputs.run_throughput
+        run: |
+          echo "BENCH_TYPE=throughput" >> "${GITHUB_ENV}"
+
       - name: Run benchmarks with AVX512
         run: |
           make BENCH_OP_FLAVOR=${{ matrix.op_flavor }} bench_${{ matrix.command }}_gpu
@@ -142,7 +154,7 @@ jobs:
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
           --name-suffix avx512 \
-          --throughput
+          --bench-type ${{ env.BENCH_TYPE }}
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882

diff --git a/.github/workflows/benchmark_gpu_l40.yml b/.github/workflows/benchmark_gpu_l40.yml
@@ -153,8 +153,7 @@ jobs:
           --commit-date "${{ env.COMMIT_DATE }}" \
           --bench-date "${{ env.BENCH_DATE }}" \
           --walk-subdirs \
-          --name-suffix avx512 \
-          --throughput
+          --name-suffix avx512
 
       - name: Upload parsed results artifact
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882