bug: fix MRR and MAP calculations

haystack/components/evaluators/document_map.py

@@ -69,24 +69,26 @@ def run(
 
         for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents):
             score = 0.0
+            average_precision = 0.0
+            relevant_documents = 0
+            ground_truth_content = []
+
             for ground_document in ground_truth:
                 if ground_document.content is None:
                     continue
+                ground_truth_content.append(ground_document.content)
 
-                average_precision = 0.0
-                relevant_documents = 0
-
-                for rank, retrieved_document in enumerate(retrieved):
-                    if retrieved_document.content is None:
-                        continue
+            for rank, retrieved_document in enumerate(retrieved):
+                if retrieved_document.content is None:
+                    continue
 
-                    if ground_document.content in retrieved_document.content:
-                        relevant_documents += 1
-                        average_precision += relevant_documents / (rank + 1)
-                if relevant_documents > 0:
-                    score = average_precision / relevant_documents
+                if retrieved_document.content in ground_truth_content:
+                    relevant_documents += 1
+                    average_precision += relevant_documents / (rank + 1)
+            if relevant_documents > 0:
+                score = average_precision / relevant_documents
             individual_scores.append(score)
 
-        score = sum(individual_scores) / len(retrieved_documents)
+        score = sum(individual_scores) / len(ground_truth_documents)
 
         return {"score": score, "individual_scores": individual_scores}

haystack/components/evaluators/document_mrr.py

@@ -67,17 +67,23 @@ def run(
 
         for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents):
             score = 0.0
+            flag = False
+            ground_truth_content = []
+
             for ground_document in ground_truth:
                 if ground_document.content is None:
                     continue
+                ground_truth_content.append(ground_document.content)
 
-                for rank, retrieved_document in enumerate(retrieved):
-                    if retrieved_document.content is None:
-                        continue
-
-                    if ground_document.content in retrieved_document.content:
-                        score = 1 / (rank + 1)
-                        break
+            for rank, retrieved_document in enumerate(retrieved):
+                if flag:
+                    break
+                if retrieved_document.content is None:
+                    continue
+                if retrieved_document.content in ground_truth_content:
+                    score = 1 / (rank + 1)
+                    flag = True
+                    break
             individual_scores.append(score)
 
         score = sum(individual_scores) / len(retrieved_documents)

releasenotes/notes/fix-issue-7758-d35b687ca226a707.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Fixed the calculation for MRR and MAP scores.

test/components/evaluators/test_document_map.py

@@ -62,7 +62,17 @@ def test_run_with_complex_data():
             ],
         ],
     )
-    assert result == {"individual_scores": [1.0, 0.8333333333333333, 1.0, 0.5, 0.0, 1.0], "score": 0.7222222222222222}
+    assert result == {
+        "individual_scores": [
+            1.0,
+            pytest.approx(0.8333333333333333),
+            1.0,
+            pytest.approx(0.5833333333333333),
+            0.0,
+            pytest.approx(0.8055555555555555),
+        ],
+        "score": pytest.approx(0.7037037037037037),
+    }
 
 
 def test_run_with_different_lengths():