Skip to content

Commit

Permalink
[ci] Use %10 as default flagging threshold for x86_64 benchmarks (ire…
Browse files Browse the repository at this point in the history
…e-org#12915)

x86_64 benchmarks are not stable in general. Use a larger
threshold to avoid being noisy.
  • Loading branch information
antiagainst authored and NatashaKnk committed Jul 6, 2023
1 parent 0bb6045 commit 55703f2
Showing 1 changed file with 2 additions and 12 deletions.
14 changes: 2 additions & 12 deletions build_tools/benchmarks/common/benchmark_thresholds.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,18 +56,6 @@ def get_threshold_str(self):
BenchmarkThreshold(re.compile(r"^PoseNet.*big-core.*LLVM-CPU.* @ Pixel"),
15, ThresholdUnit.PERCENTAGE),

# Fluctuating benchmarks on X86_64 CPUs.
BenchmarkThreshold(re.compile(r"^BertForMaskedLMTF.*x86_64"), 10,
ThresholdUnit.PERCENTAGE),
BenchmarkThreshold(re.compile(r"^BertLargeTF.*x86_64"), 10,
ThresholdUnit.PERCENTAGE),
BenchmarkThreshold(re.compile(r"^MobileBertSquad_fp32.*x86_64"), 10,
ThresholdUnit.PERCENTAGE),
BenchmarkThreshold(re.compile(r"^MobileSSD_fp32.*x86_64"), 10,
ThresholdUnit.PERCENTAGE),
BenchmarkThreshold(re.compile(r"^Resnet50TF.*x86_64"), 10,
ThresholdUnit.PERCENTAGE),

# Benchmarks that complete <= 10ms on X86_64 CPUs; using percentage is not
# suitable anymore.
BenchmarkThreshold(re.compile(r"^DeepLabV3_fp32.*x86_64"), 1 * 10**6,
Expand Down Expand Up @@ -97,6 +85,8 @@ def get_threshold_str(self):
BenchmarkThreshold(re.compile(r"^MobileNet.*GPU"), 1 * 10**6,
ThresholdUnit.VALUE_NS),

# Default threshold for all x86_64 benchmarks: 10%.
BenchmarkThreshold(re.compile(r".*x86_64.*"), 10, ThresholdUnit.PERCENTAGE),
# Default threshold for all benchmarks: 5%.
BenchmarkThreshold(re.compile(r".*"), 5, ThresholdUnit.PERCENTAGE),
]
Expand Down

0 comments on commit 55703f2

Please sign in to comment.