Skip to content

Commit

Permalink
Added more transforms to benchmark (#5)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ternaus
ternaus authored Oct 28, 2024
1 parent dee7b49 commit b38632e
Show file tree
Hide file tree
Showing 16 changed files with 1,593 additions and 1,292 deletions.
55 changes: 28 additions & 27 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,36 +65,37 @@ This benchmark suite measures the throughput and performance characteristics of

### Performance Comparison

Images per second (higher is better)

| Transform | albumentations<br>1.4.20 | augly<br>1.0.0 | imgaug<br>0.4.0 | kornia<br>0.7.3 | torchvision<br>0.20.0 |
|:------------------|:---------------------------|:-----------------|:------------------|:------------------|:------------------------|
| HorizontalFlip | **8325 ± 955** | 4807 ± 818 | 5585 ± 1146 | 390 ± 106 | 875 ± 69 |
| VerticalFlip | **20493 ± 1134** | 9153 ± 1291 | 10390 ± 290 | 1212 ± 402 | 3131 ± 61 |
| Rotate | **1272 ± 12** | 1119 ± 41 | 1054 ± 96 | 143 ± 11 | 147 ± 6 |
| Affine | **967 ± 3** | - | 802 ± 55 | 147 ± 9 | 128 ± 6 |
| Equalize | **961 ± 4** | - | 540 ± 39 | 152 ± 19 | 414 ± 64 |
| RandomCrop80 | **118946 ± 741** | 25272 ± 1822 | 11009 ± 404 | 1510 ± 230 | 22499 ± 5532 |
| ShiftRGB | **1873 ± 252** | - | 1563 ± 195 | - | - |
| Resize | **2365 ± 153** | 611 ± 78 | 1699 ± 105 | 232 ± 24 | 166 ± 9 |
| RandomGamma | **8608 ± 220** | - | 2315 ± 151 | 108 ± 13 | - |
| Grayscale | **3050 ± 597** | 2720 ± 932 | 1670 ± 56 | 289 ± 75 | 1626 ± 156 |
| RandomPerspective | 410 ± 20 | - | **537 ± 41** | 86 ± 11 | 96 ± 4 |
| GaussianBlur | **1734 ± 204** | 242 ± 4 | 1047 ± 122 | 176 ± 18 | 73 ± 6 |
| MedianBlur | **862 ± 30** | - | 814 ± 71 | 5 ± 0 | - |
| MotionBlur | **2975 ± 52** | - | 583 ± 52 | 73 ± 2 | - |
| Posterize | **5214 ± 101** | - | 2112 ± 281 | 430 ± 49 | 3063 ± 116 |
| JpegCompression | **845 ± 61** | 778 ± 5 | 413 ± 29 | 71 ± 3 | 617 ± 24 |
| GaussianNoise | 147 ± 10 | 67 ± 2 | **203 ± 10** | 75 ± 1 | - |
| Elastic | 171 ± 15 | - | **227 ± 17** | 1 ± 0 | 2 ± 0 |
| ColorJitter | **536 ± 41** | 255 ± 13 | - | 55 ± 18 | 45 ± 2 |
| Brightness | **4443 ± 84** | 1163 ± 86 | - | 472 ± 101 | - |
| Contrast | **4398 ± 143** | 736 ± 79 | - | 425 ± 52 | - |
| Blur | **4816 ± 59** | 246 ± 3 | - | - | - |
| RandomResizedCrop | **2952 ± 24** | - | - | 287 ± 58 | 466 ± 30 |
| Normalize | **1016 ± 84** | - | - | 626 ± 40 | 422 ± 64 |
| HorizontalFlip | **8325 ± 955** | 4807 ± 818 | 6042 ± 788 | 390 ± 106 | 914 ± 67 |
| VerticalFlip | **20493 ± 1134** | 9153 ± 1291 | 10931 ± 1844 | 1212 ± 402 | 3198 ± 200 |
| Rotate | **1272 ± 12** | 1119 ± 41 | 1136 ± 218 | 143 ± 11 | 181 ± 11 |
| Affine | **967 ± 3** | - | 774 ± 97 | 147 ± 9 | 130 ± 12 |
| Equalize | **961 ± 4** | - | 581 ± 54 | 152 ± 19 | 479 ± 12 |
| RandomCrop80 | **118946 ± 741** | 25272 ± 1822 | 11503 ± 441 | 1510 ± 230 | 32109 ± 1241 |
| ShiftRGB | **1873 ± 252** | - | 1582 ± 65 | - | - |
| Resize | **2365 ± 153** | 611 ± 78 | 1806 ± 63 | 232 ± 24 | 195 ± 4 |
| RandomGamma | **8608 ± 220** | - | 2318 ± 269 | 108 ± 13 | - |
| Grayscale | **3050 ± 597** | 2720 ± 932 | 1681 ± 156 | 289 ± 75 | 1838 ± 130 |
| RandomPerspective | 410 ± 20 | - | **554 ± 22** | 86 ± 11 | 96 ± 5 |
| GaussianBlur | **1734 ± 204** | 242 ± 4 | 1090 ± 65 | 176 ± 18 | 79 ± 3 |
| MedianBlur | **862 ± 30** | - | 813 ± 30 | 5 ± 0 | - |
| MotionBlur | **2975 ± 52** | - | 612 ± 18 | 73 ± 2 | - |
| Posterize | **5214 ± 101** | - | 2097 ± 68 | 430 ± 49 | 3196 ± 185 |
| JpegCompression | **845 ± 61** | 778 ± 5 | 459 ± 35 | 71 ± 3 | 625 ± 17 |
| GaussianNoise | 147 ± 10 | 67 ± 2 | **206 ± 11** | 75 ± 1 | - |
| Elastic | 171 ± 15 | - | **235 ± 20** | 1 ± 0 | 2 ± 0 |
| Clahe | **423 ± 10** | - | 335 ± 43 | 94 ± 9 | - |
| CoarseDropout | **11288 ± 609** | - | 671 ± 38 | 536 ± 87 | - |
| Blur | **4816 ± 59** | 246 ± 3 | 3807 ± 325 | - | - |
| ColorJitter | **536 ± 41** | 255 ± 13 | - | 55 ± 18 | 46 ± 2 |
| Brightness | **4443 ± 84** | 1163 ± 86 | - | 472 ± 101 | 429 ± 20 |
| Contrast | **4398 ± 143** | 736 ± 79 | - | 425 ± 52 | 335 ± 35 |
| RandomResizedCrop | **2952 ± 24** | - | - | 287 ± 58 | 511 ± 10 |
| Normalize | **1016 ± 84** | - | - | 626 ± 40 | 519 ± 12 |
| PlankianJitter | **1844 ± 208** | - | - | 813 ± 211 | - |
| Clahe | **423 ± 10** | - | - | 94 ± 9 | - |
| CoarseDropout | **11288 ± 609** | - | - | 536 ± 87 | - |


## Requirements

Expand Down
Empty file added benchmark/__init__.py
View file
Empty file.
31 changes: 17 additions & 14 deletions benchmark/compare_results.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
from pathlib import Path
import json
from pathlib import Path

import pandas as pd
import numpy as np


def load_results(file_path: Path) -> tuple[str, dict[str, float], dict[str, float], str]:
"""Load results from a JSON file and extract mean and std throughputs and version"""
Expand All @@ -14,13 +15,14 @@ def load_results(file_path: Path) -> tuple[str, dict[str, float], dict[str, floa
version = data["metadata"]["library_versions"].get(library, "N/A")

for transform_name, results in data["results"].items():
transform_name = transform_name.split('(')[0].strip()
transform_name_stripped = transform_name.split("(")[0].strip()
if results["supported"]:
medians[transform_name] = results["median_throughput"]
stds[transform_name] = results["std_throughput"]
medians[transform_name_stripped] = results["median_throughput"]
stds[transform_name_stripped] = results["std_throughput"]

return library, medians, stds, version


def create_comparison_table(results_dir: Path) -> pd.DataFrame:
"""Create a comparison table from all result files in the directory"""
result_files = list(results_dir.glob("*_results.json"))
Expand Down Expand Up @@ -55,16 +57,14 @@ def create_comparison_table(results_dir: Path) -> pd.DataFrame:
median = df_medians.loc[idx, library]
std = df_stds.loc[idx, library]
# Bold if it's the maximum value
if median == max_values[idx]:
value = f"**{median:.0f} ± {std:.0f}**"
else:
value = f"{median:.0f} ± {std:.0f}"
value = f"**{median:.0f} ± {std:.0f}**" if median == max_values[idx] else f"{median:.0f} ± {std:.0f}"
column_values.append(value)

formatted_data[f"{library}<br>{versions[library]}"] = column_values

return pd.DataFrame(formatted_data)


def get_system_summary(results_dir: Path) -> str:
"""Extract and format system information from any result file"""
result_files = list(results_dir.glob("*_results.json"))
Expand Down Expand Up @@ -96,11 +96,13 @@ def get_system_summary(results_dir: Path) -> str:
]

# Add library versions
summary.extend([
"",
"### Library Versions",
"",
])
summary.extend(
[
"",
"### Library Versions",
"",
],
)

# Collect versions from all result files
versions = {}
Expand All @@ -116,6 +118,7 @@ def get_system_summary(results_dir: Path) -> str:

return "\n".join(summary)


def main() -> None:
import argparse

Expand Down
Loading

0 comments on commit b38632e

Please sign in to comment.