Update 1.6 benchmark w.r.t. 2.1

.github/workflows/perf_benchmark.yaml

@@ -7,6 +7,9 @@ on:
         type: choice
         description: Model category to run benchmark
         options:
+          - speed
+          - balance
+          - accuracy
           - default # speed, balance, accuracy models only
           - all # default + other models
         default: all
@@ -54,7 +57,7 @@ on:
     inputs:
       model-category:
         type: string
-        description: Model category to run benchmark [default, all]
+        description: Model category to run benchmark [speed, balance, accuracy, default, all]
         default: default
       data-group:
         type: string
@@ -156,13 +159,10 @@ jobs:
         if: ${{ always() }}
 
   Perf-Benchmark-Summary:
+    if: ${{ always() }}
     needs: Perf-Benchmark-Run
     runs-on: ubuntu-latest
     steps:
-      - name: Download benchmark results
-        uses: actions/download-artifact@v4
-        with:
-          path: tests/perf/history/latest
       - name: Checkout repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: Install Python
@@ -171,6 +171,10 @@ jobs:
           python-version: "3.10"
       - name: Install dependencies
         run: python -m pip install --upgrade pip pandas matplotlib nbconvert ipython ipykernel openpyxl
+      - name: Download benchmark results
+        uses: actions/download-artifact@v4
+        with:
+          path: tests/perf/history/latest
       - name: Summarize benchamrk results
         run: |
           python tests/perf/history/summary.py tests/perf/history ./perf-benchmark-summary --pattern "*raw*.csv"

tests/perf/conftest.py

@@ -29,8 +29,8 @@ def pytest_addoption(parser):
         "--model-category",
         action="store",
         default="all",
-        choices=("default", "all"),
-        help="Choose default|all. Defaults to all.",
+        choices=("speed", "balance", "accuracy", "default", "other", "all"),
+        help="Choose speed|balcence|accuracy|default|other|all. Defaults to all.",
     )
     parser.addoption(
         "--data-group",

tests/perf/history/summary.py

@@ -239,19 +239,26 @@ def normalize(data: pd.DataFrame) -> pd.DataFrame:
     data.loc[tiling_indices, "task"] = data.loc[tiling_indices, "task"].str.replace("tiling_", "")
     data.loc[tiling_indices, "model"] = data.loc[tiling_indices, "model"] + "_tile"
     # Map anomaly metrics
+    anomaly_indices = data["task"] == "anomaly_classification"
     if "test/image_F1Score" in data:
-        anomaly_indices = data["task"] == "anomaly_classification"
         data.loc[anomaly_indices, "test/f1-score"] = data.loc[anomaly_indices, "test/image_F1Score"]
+    if "export/image_F1Score" in data:
         data.loc[anomaly_indices, "export/f1-score"] = data.loc[anomaly_indices, "export/image_F1Score"]
+    if "optimize/image_F1Score" in data:
         data.loc[anomaly_indices, "optimize/f1-score"] = data.loc[anomaly_indices, "optimize/image_F1Score"]
-        anomaly_indices = data["task"] == "anomaly_detection"
+    anomaly_indices = data["task"] == "anomaly_detection"
+    if "test/image_F1Score" in data:
         data.loc[anomaly_indices, "test/f1-score"] = data.loc[anomaly_indices, "test/image_F1Score"]
+    if "export/image_F1Score" in data:
         data.loc[anomaly_indices, "export/f1-score"] = data.loc[anomaly_indices, "export/image_F1Score"]
+    if "optimize/image_F1Score" in data:
         data.loc[anomaly_indices, "optimize/f1-score"] = data.loc[anomaly_indices, "optimize/image_F1Score"]
+    anomaly_indices = data["task"] == "anomaly_segmentation"
     if "test/pixel_F1Score" in data:
-        anomaly_indices = data["task"] == "anomaly_segmentation"
         data.loc[anomaly_indices, "test/f1-score"] = data.loc[anomaly_indices, "test/pixel_F1Score"]
+    if "export/pixel_F1Score" in data:
         data.loc[anomaly_indices, "export/f1-score"] = data.loc[anomaly_indices, "export/pixel_F1Score"]
+    if "optimize/pixel_F1Score" in data:
         data.loc[anomaly_indices, "optimize/f1-score"] = data.loc[anomaly_indices, "optimize/pixel_F1Score"]
     # Map other names
     data = data.rename(columns=V1_V2_NAME_MAP).replace(V1_V2_NAME_MAP)