Skip to content

Commit

Permalink
Added brier score for p-metrics in forecasting
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsinghargo
jsinghargo committed Apr 18, 2021
1 parent 538f3e6 commit 6f9f275
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 47 deletions.
17 changes: 3 additions & 14 deletions argoverse/evaluation/eval_forecasting.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,20 +115,9 @@ def get_displacement_errors_and_miss_rate(

if forecasted_probabilities is not None:
prob_n_misses.append(1.0 if curr_min_fde > miss_threshold else (1.0 - pruned_probabilities[min_idx]))
prob_min_ade.append(
min(
-np.log(pruned_probabilities[min_idx]),
-np.log(LOW_PROB_THRESHOLD_FOR_METRICS),
)
+ curr_min_ade
)
prob_min_fde.append(
min(
-np.log(pruned_probabilities[min_idx]),
-np.log(LOW_PROB_THRESHOLD_FOR_METRICS),
)
+ curr_min_fde
)
prob_min_ade.append((1 - pruned_probabilities[min_idx]) ** 2 + curr_min_ade)
prob_min_fde.append((1 - pruned_probabilities[min_idx]) ** 2 + curr_min_fde)

metric_results["minADE"] = sum(min_ade) / len(min_ade)
metric_results["minFDE"] = sum(min_fde) / len(min_fde)
metric_results["MR"] = sum(n_misses) / len(n_misses)
Expand Down
173 changes: 140 additions & 33 deletions integration_tests/test_eval_forecasting.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,27 @@

"""This module tests the motion forecasting metric evaluation."""

from typing import Dict, Mapping

import numpy as np
from numpy.testing import assert_almost_equal

from argoverse.evaluation.eval_forecasting import compute_forecasting_metrics


def assert_metrics_almost_equal(expected_metrics: Mapping[str, float], output_metrics: Mapping[str, float]) -> None:
"""Assert that the expected metrics match the computed metrics.
Args:
expected_metrics: Expected metrics for the test case.
output_metrics: Metrics computed by the evaluation code.
"""

assert set(expected_metrics.keys()).issubset(set(output_metrics.keys()))
for metric in expected_metrics:
assert_almost_equal(expected_metrics[metric], round(output_metrics[metric], 2), 2)


def test_compute_forecasting_metric() -> None:
"""Test computation of motion forecasting metrics."""
# Test Case:
Expand Down Expand Up @@ -150,14 +165,14 @@ def test_compute_forecasting_metric() -> None:
)

city_names = {1: "MIA"}
max_n_guesses = 2
horizon = 30
miss_threshold = 1.0

# Case 1
# Case 1: Top-2 error, 2 forecasts, probabilities provided
forecasted_trajectories = {1: [forecasted_1, forecasted_2]}
forecasted_probabilities = {1: [0.80, 0.20]}
ground_truth_trajectories = {1: ground_truth}
max_n_guesses = 2

metrics = compute_forecasting_metrics(
forecasted_trajectories,
Expand All @@ -169,25 +184,130 @@ def test_compute_forecasting_metric() -> None:
forecasted_probabilities,
)

expected_min_ade = 1.0
expected_min_fde = 1.0
expected_dac = 1.0
expected_miss_rate = 0.0
expected_p_min_ade = 1.22
expected_p_min_fde = 1.22
expected_p_miss_rate = 0.20
assert_almost_equal(expected_min_ade, round(metrics["minADE"], 2), 2)
assert_almost_equal(expected_min_fde, round(metrics["minFDE"], 2), 2)
assert_almost_equal(expected_dac, round(metrics["DAC"], 2), 2)
assert_almost_equal(expected_miss_rate, round(metrics["MR"], 2), 2)
assert_almost_equal(expected_p_min_ade, round(metrics["p-minADE"], 2), 2)
assert_almost_equal(expected_p_min_fde, round(metrics["p-minFDE"], 2), 2)
assert_almost_equal(expected_p_miss_rate, round(metrics["p-MR"], 2), 2)

# Case 2
expected_metrics: Dict[str, float] = {
"minADE": 1.0,
"minFDE": 1.0,
"DAC": 1.0,
"MR": 0.0,
"p-minADE": 1.04,
"p-minFDE": 1.04,
"p-MR": 0.20,
}
assert_metrics_almost_equal(expected_metrics, metrics)

# Case 2: Top-2, 1 forecast, probabilities provided
forecasted_trajectories = {1: [forecasted_2]}
forecasted_probabilities = {1: [1.0]}
ground_truth_trajectories = {1: ground_truth}
max_n_guesses = 2

metrics = compute_forecasting_metrics(
forecasted_trajectories,
ground_truth_trajectories,
city_names,
max_n_guesses,
horizon,
miss_threshold,
forecasted_probabilities,
)

expected_metrics: Dict[str, float] = {
"minADE": 3.23,
"minFDE": 11.31,
"DAC": 1.0,
"MR": 1.0,
"p-minADE": 3.23,
"p-minFDE": 11.31,
"p-MR": 1.0,
}
assert_metrics_almost_equal(expected_metrics, metrics)

# Case 3: Top-1, 2 forecast, probabilities provided
forecasted_trajectories = {1: [forecasted_1, forecasted_2]}
forecasted_probabilities = {1: [0.2, 0.8]}
ground_truth_trajectories = {1: ground_truth}
max_n_guesses = 1

metrics = compute_forecasting_metrics(
forecasted_trajectories,
ground_truth_trajectories,
city_names,
max_n_guesses,
horizon,
miss_threshold,
forecasted_probabilities,
)

expected_metrics: Dict[str, float] = {
"minADE": 3.23,
"minFDE": 11.31,
"DAC": 1.0,
"MR": 1.0,
"p-minADE": 3.23,
"p-minFDE": 11.31,
"p-MR": 1.0,
}
assert_metrics_almost_equal(expected_metrics, metrics)

# Case 4: Top-2, 2 forecast, probabilities provided (not normalized)
forecasted_trajectories = {1: [forecasted_1, forecasted_2]}
forecasted_probabilities = {1: [0.3, 0.2]}
ground_truth_trajectories = {1: ground_truth}
max_n_guesses = 2

metrics = compute_forecasting_metrics(
forecasted_trajectories,
ground_truth_trajectories,
city_names,
max_n_guesses,
horizon,
miss_threshold,
forecasted_probabilities,
)

expected_metrics: Dict[str, float] = {
"minADE": 1.0,
"minFDE": 1.0,
"DAC": 1.0,
"MR": 0.0,
"p-minADE": 1.16,
"p-minFDE": 1.16,
"p-MR": 0.4,
}
assert_metrics_almost_equal(expected_metrics, metrics)

# Case 5: Top-2, 2 forecast, uniform probabilities
forecasted_trajectories = {1: [forecasted_1, forecasted_2]}
forecasted_probabilities = {1: [0.5, 0.5]}
ground_truth_trajectories = {1: ground_truth}
max_n_guesses = 2

metrics = compute_forecasting_metrics(
forecasted_trajectories,
ground_truth_trajectories,
city_names,
max_n_guesses,
horizon,
miss_threshold,
forecasted_probabilities,
)

expected_metrics: Dict[str, float] = {
"minADE": 1.0,
"minFDE": 1.0,
"DAC": 1.0,
"MR": 0.0,
"p-minADE": 1.25,
"p-minFDE": 1.25,
"p-MR": 0.5,
}
assert_metrics_almost_equal(expected_metrics, metrics)

# Case 5: Top-2, 2 forecast, No probabilities
forecasted_trajectories = {1: [forecasted_2, forecasted_2, forecasted_1]}
forecasted_probabilities = None
ground_truth_trajectories = {1: ground_truth}
max_n_guesses = 2

metrics = compute_forecasting_metrics(
forecasted_trajectories,
Expand All @@ -199,18 +319,5 @@ def test_compute_forecasting_metric() -> None:
forecasted_probabilities,
)

expected_min_ade = 3.23
expected_min_fde = 11.31
expected_dac = 1.0
expected_miss_rate = 1.0
expected_p_min_ade = 3.23
expected_p_min_fde = 11.31
expected_p_miss_rate = 1.0

assert_almost_equal(expected_min_ade, round(metrics["minADE"], 2), 2)
assert_almost_equal(expected_min_fde, round(metrics["minFDE"], 2), 2)
assert_almost_equal(expected_dac, round(metrics["DAC"], 2), 2)
assert_almost_equal(expected_miss_rate, round(metrics["MR"], 2), 2)
assert_almost_equal(expected_p_min_ade, round(metrics["p-minADE"], 2), 2)
assert_almost_equal(expected_p_min_fde, round(metrics["p-minFDE"], 2), 2)
assert_almost_equal(expected_p_miss_rate, round(metrics["p-MR"], 2), 2)
expected_metrics: Dict[str, float] = {"minADE": 3.23, "minFDE": 11.31, "DAC": 1.0, "MR": 1.0}
assert_metrics_almost_equal(expected_metrics, metrics)

0 comments on commit 6f9f275

Please sign in to comment.