Skip to content

Commit

Permalink
update error message for rsd calculation
Browse files Browse the repository at this point in the history
Signed-off-by: Michael Oviedo <mikeovi@amazon.com>
  • Loading branch information
OVI3D0 committed Oct 23, 2024
1 parent 0467ae8 commit d2910b6
Showing 1 changed file with 5 additions and 5 deletions.
10 changes: 5 additions & 5 deletions osbenchmark/aggregator.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,19 +127,19 @@ def build_aggregated_results(self):
}
for metric in self.metrics:
op_metric[metric] = aggregated_task_metrics[metric]

# Handle standard metrics (like latency, service_time) which are stored as dictionaries
if isinstance(aggregated_task_metrics[metric], dict):
# Calculate RSD for the mean values across all test executions
# We use mean here as it's more sensitive to outliers, which is desirable for assessing variability
mean_values = [v['mean'] for v in task_metrics[metric]]
rsd = self.calculate_rsd(mean_values)
rsd = self.calculate_rsd(mean_values, f"{task}.{metric}.mean")
op_metric[metric]['mean_rsd'] = rsd

# Handle derived metrics (like error_rate, duration) which are stored as simple values
else:
# Calculate RSD directly from the metric values across all test executions
rsd = self.calculate_rsd(task_metrics[metric])
rsd = self.calculate_rsd(task_metrics[metric], f"{task}.{metric}")
op_metric[f"{metric}_rsd"] = rsd

aggregated_results["op_metrics"].append(op_metric)
Expand Down Expand Up @@ -214,9 +214,9 @@ def calculate_weighted_average(self, task_metrics: Dict[str, List[Any]], iterati

return weighted_metrics

def calculate_rsd(self, values):
def calculate_rsd(self, values, metric_name: str):
if not values:
raise ValueError("Cannot calculate RSD for an empty list of values")
raise ValueError(f"Cannot calculate RSD for metric '{metric_name}': empty list of values")
if len(values) == 1:
return "NA" # RSD is not applicable for a single value
mean = statistics.mean(values)
Expand Down

0 comments on commit d2910b6

Please sign in to comment.