Skip to content

Commit

Permalink
Benchmark: Use a 50% confidence interval
Browse files Browse the repository at this point in the history
  • Loading branch information
walles committed Jan 3, 2021
1 parent f6c9c31 commit 40f3e6d
Showing 1 changed file with 6 additions and 3 deletions.
9 changes: 6 additions & 3 deletions benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
# Number of benchmark iterations to run
ITERATIONS = 60

# Confidence interval to present
PRECISION_PERCENT = 50

# Number of throwaway iterations to run before starting the benchmark
WARMUP_RUNS = 10

Expand Down Expand Up @@ -96,9 +99,9 @@ def print_timings(binary: str, testdata_filename: str):
Run the indicated binary and print timings for it
"""

# Throw away the top and bottom 5%, giving us 90% coverage
assert ITERATIONS % 20 == 0
THROW_AWAY_AT_EACH_END = ITERATIONS // 20
# Ensure we throw away an integer number of iterations
assert ((100 - PRECISION_PERCENT) * ITERATIONS) % 200 == 0
THROW_AWAY_AT_EACH_END = ((100 - PRECISION_PERCENT) * ITERATIONS) // 200

# Do some warmup runs
for _ in range(WARMUP_RUNS):
Expand Down

0 comments on commit 40f3e6d

Please sign in to comment.