Skip to content

Commit

Permalink
Renamed confusion matrix components. (#825)
Browse files Browse the repository at this point in the history
  • Loading branch information
@czaloom
czaloom authored Dec 5, 2024
1 parent 300886e commit 6baaba2
Show file tree
Hide file tree
Showing 24 changed files with 295 additions and 281 deletions.
8 changes: 4 additions & 4 deletions lite/examples/object-detection.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -959,7 +959,7 @@
"id": "98edc4dd",
"metadata": {},
"source": [
"### Hallucinations"
"### Unmatched Predictions"
]
},
{
Expand Down Expand Up @@ -1764,7 +1764,7 @@
}
],
"source": [
"metric.value[\"hallucinations\"]"
"metric.value[\"unmatched_predictions\"]"
]
},
{
Expand All @@ -1773,7 +1773,7 @@
"id": "415335e4",
"metadata": {},
"source": [
"### Ground Truths Missing Predictions"
"### Unmatched Ground Truths"
]
},
{
Expand Down Expand Up @@ -2995,7 +2995,7 @@
}
],
"source": [
"metric.value[\"missing_predictions\"]"
"metric.value[\"unmatched_ground_truths\"]"
]
}
],
Expand Down
2 changes: 1 addition & 1 deletion lite/examples/tabular_classification.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -605,7 +605,7 @@
}
],
"source": [
"cm.value[\"missing_predictions\"]"
"cm.value[\"unmatched_ground_truths\"]"
]
}
],
Expand Down
42 changes: 21 additions & 21 deletions lite/tests/classification/test_confusion_matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def test_compute_confusion_matrix():

score_thresholds = np.array([0.25, 0.75], dtype=np.float64)

confusion_matrix, missing_predictions = compute_confusion_matrix(
confusion_matrix, unmatched_ground_truths = compute_confusion_matrix(
data=data,
label_metadata=label_metadata,
score_thresholds=score_thresholds,
Expand Down Expand Up @@ -74,15 +74,15 @@ def test_compute_confusion_matrix():
)
).all()

assert missing_predictions.shape == (2, 4, 1)
assert unmatched_ground_truths.shape == (2, 4, 1)
assert (
# score >= 0.25
missing_predictions[0, :, 0]
unmatched_ground_truths[0, :, 0]
== np.array([-1.0, -1.0, -1.0, -1.0])
).all()
assert (
# score >= 0.75
missing_predictions[1, :, 0]
unmatched_ground_truths[1, :, 0]
== np.array([-1.0, -1.0, -1.0, 1.0])
).all()

Expand Down Expand Up @@ -144,7 +144,7 @@ def test_confusion_matrix_basic(basic_classifications: list[Classification]):
}
},
},
"missing_predictions": {},
"unmatched_ground_truths": {},
},
"parameters": {
"score_threshold": 0.25,
Expand All @@ -166,7 +166,7 @@ def test_confusion_matrix_basic(basic_classifications: list[Classification]):
},
}
},
"missing_predictions": {
"unmatched_ground_truths": {
"3": {"count": 1, "examples": [{"datum": "uid2"}]}
},
},
Expand All @@ -179,7 +179,7 @@ def test_confusion_matrix_basic(basic_classifications: list[Classification]):
for m in actual_metrics:
_filter_elements_with_zero_count(
cm=m["value"]["confusion_matrix"],
mp=m["value"]["missing_predictions"],
mp=m["value"]["unmatched_ground_truths"],
)
assert m in expected_metrics
for m in expected_metrics:
Expand Down Expand Up @@ -212,7 +212,7 @@ def test_confusion_matrix_unit(
"1": {"1": {"count": 1, "examples": []}},
"2": {"1": {"count": 2, "examples": []}},
},
"missing_predictions": {},
"unmatched_ground_truths": {},
},
"parameters": {
"score_threshold": 0.5,
Expand All @@ -223,7 +223,7 @@ def test_confusion_matrix_unit(
for m in actual_metrics:
_filter_elements_with_zero_count(
cm=m["value"]["confusion_matrix"],
mp=m["value"]["missing_predictions"],
mp=m["value"]["unmatched_ground_truths"],
)
assert m in expected_metrics
for m in expected_metrics:
Expand Down Expand Up @@ -282,7 +282,7 @@ def test_confusion_matrix_with_animal_example(
}
},
},
"missing_predictions": {
"unmatched_ground_truths": {
"dog": {"count": 1, "examples": [{"datum": "uid5"}]}
},
},
Expand All @@ -295,7 +295,7 @@ def test_confusion_matrix_with_animal_example(
for m in actual_metrics:
_filter_elements_with_zero_count(
cm=m["value"]["confusion_matrix"],
mp=m["value"]["missing_predictions"],
mp=m["value"]["unmatched_ground_truths"],
)
assert m in expected_metrics
for m in expected_metrics:
Expand Down Expand Up @@ -356,7 +356,7 @@ def test_confusion_matrix_with_color_example(
}
},
},
"missing_predictions": {
"unmatched_ground_truths": {
"red": {"count": 1, "examples": [{"datum": "uid2"}]}
},
},
Expand All @@ -369,7 +369,7 @@ def test_confusion_matrix_with_color_example(
for m in actual_metrics:
_filter_elements_with_zero_count(
cm=m["value"]["confusion_matrix"],
mp=m["value"]["missing_predictions"],
mp=m["value"]["unmatched_ground_truths"],
)
assert m in expected_metrics
for m in expected_metrics:
Expand Down Expand Up @@ -438,7 +438,7 @@ def test_confusion_matrix_multiclass(
}
},
},
"missing_predictions": {},
"unmatched_ground_truths": {},
},
"parameters": {
"score_threshold": 0.05,
Expand Down Expand Up @@ -466,7 +466,7 @@ def test_confusion_matrix_multiclass(
}
},
},
"missing_predictions": {
"unmatched_ground_truths": {
"cat": {
"count": 2,
"examples": [{"datum": "uid0"}, {"datum": "uid2"}],
Expand All @@ -483,7 +483,7 @@ def test_confusion_matrix_multiclass(
"type": "ConfusionMatrix",
"value": {
"confusion_matrix": {},
"missing_predictions": {
"unmatched_ground_truths": {
"cat": {
"count": 2,
"examples": [{"datum": "uid0"}, {"datum": "uid2"}],
Expand All @@ -504,7 +504,7 @@ def test_confusion_matrix_multiclass(
for m in actual_metrics:
_filter_elements_with_zero_count(
cm=m["value"]["confusion_matrix"],
mp=m["value"]["missing_predictions"],
mp=m["value"]["unmatched_ground_truths"],
)
assert m in expected_metrics
for m in expected_metrics:
Expand Down Expand Up @@ -560,7 +560,7 @@ def test_confusion_matrix_without_hardmax_animal_example(
},
}
},
"missing_predictions": {},
"unmatched_ground_truths": {},
},
"parameters": {
"score_threshold": 0.05,
Expand All @@ -580,7 +580,7 @@ def test_confusion_matrix_without_hardmax_animal_example(
}
}
},
"missing_predictions": {},
"unmatched_ground_truths": {},
},
"parameters": {
"score_threshold": 0.4,
Expand All @@ -591,7 +591,7 @@ def test_confusion_matrix_without_hardmax_animal_example(
"type": "ConfusionMatrix",
"value": {
"confusion_matrix": {},
"missing_predictions": {
"unmatched_ground_truths": {
"ant": {
"count": 1,
"examples": [
Expand All @@ -611,7 +611,7 @@ def test_confusion_matrix_without_hardmax_animal_example(
for m in actual_metrics:
_filter_elements_with_zero_count(
cm=m["value"]["confusion_matrix"],
mp=m["value"]["missing_predictions"],
mp=m["value"]["unmatched_ground_truths"],
)
assert m in expected_metrics
for m in expected_metrics:
Expand Down
2 changes: 1 addition & 1 deletion lite/tests/classification/test_dataloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ def test_no_data():
loader.finalize()


def test_missing_predictions(
def test_unmatched_ground_truths(
classifications_no_predictions: list[Classification],
):
loader = DataLoader()
Expand Down
4 changes: 2 additions & 2 deletions lite/tests/object_detection/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -704,7 +704,7 @@ def false_negatives_two_images_one_only_with_different_class_high_confidence_of_


@pytest.fixture
def detections_fp_hallucination_edge_case() -> list[Detection]:
def detections_fp_unmatched_prediction_edge_case() -> list[Detection]:
return [
Detection(
uid="uid1",
Expand Down Expand Up @@ -1093,7 +1093,7 @@ def detections_for_detailed_counting(
xmax=rect4[1],
ymin=rect4[2],
ymax=rect4[3],
labels=["hallucination"],
labels=["no_overlap"],
scores=[0.1],
),
],
Expand Down
6 changes: 3 additions & 3 deletions lite/tests/object_detection/test_accuracy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,9 @@ def test_accuracy_metrics_first_class(
groundtruths
datum uid1
box 1 - label v1 - tp
box 3 - label v2 - fn missing prediction
box 3 - label v2 - fn unmatched ground truths
datum uid2
box 2 - label v1 - fn missing prediction
box 2 - label v1 - fn unmatched ground truths
predictions
datum uid1
Expand Down Expand Up @@ -176,7 +176,7 @@ def test_accuracy_metrics_second_class(
groundtruths
datum uid1
box 3 - label v2 - fn missing prediction
box 3 - label v2 - fn unmatched ground truths
datum uid2
none
predictions
Expand Down
4 changes: 2 additions & 2 deletions lite/tests/object_detection/test_average_precision.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def test_ap_metrics_first_class(
datum uid1
box 1 - label v1 - tp
datum uid2
box 2 - label v1 - fn missing prediction
box 2 - label v1 - fn unmatched ground truths
predictions
datum uid1
Expand Down Expand Up @@ -192,7 +192,7 @@ def test_ap_metrics_second_class(
groundtruths
datum uid1
box 3 - label v2 - fn missing prediction
box 3 - label v2 - fn unmatched ground truths
datum uid2
none
predictions
Expand Down
4 changes: 2 additions & 2 deletions lite/tests/object_detection/test_average_recall.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def test_ar_metrics_first_class(
datum uid1
box 1 - label v1 - tp
datum uid2
box 2 - label v1 - fn missing prediction
box 2 - label v1 - fn unmatched ground truths
predictions
datum uid1
Expand Down Expand Up @@ -198,7 +198,7 @@ def test_ar_metrics_second_class(
groundtruths
datum uid1
box 3 - label v2 - fn missing prediction
box 3 - label v2 - fn unmatched ground truths
datum uid2
none
predictions
Expand Down
Loading

0 comments on commit 6baaba2

Please sign in to comment.