Fix NDCG for empty search results (#29267)

Fixes and edge case where DiscountedCumulativeGain can return NaN as result of the quality metric calculation. This can happen when the search result set is empty and normalization is used. We should return 0 in this case. Also adding related unit tests to the other two metrics.
elastic · Apr 3, 2018 · 2b07f63 · 2b07f63
1 parent 0028563
commit 2b07f63
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 3 deletions.
diff --git a/...es/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/...es/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java
@@ -140,9 +140,12 @@ public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
 
         if (normalize) {
             Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder()));
-            double idcg = computeDCG(
-                    allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size())));
-            dcg = dcg / idcg;
+            double idcg = computeDCG(allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size())));
+            if (idcg > 0) {
+                dcg = dcg / idcg;
+            } else {
+                dcg = 0;
+            }
         }
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg);
         evalQueryQuality.addHitsAndRatings(ratedHits);

diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java
@@ -228,6 +228,10 @@ public String getWriteableName() {
             return NAME;
         }
 
+        /**
+         * the ranking of the first relevant document, or -1 if no relevant document was
+         * found
+         */
         int getFirstRelevantRank() {
             return firstRelevantRank;
         }

diff --git a/...nk-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/...nk-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java
@@ -205,6 +205,32 @@ public void testDCGAtFourMoreRatings() {
         assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), DELTA);
     }
 
+    /**
+     * test that metric returns 0.0 when there are no search results
+     */
+    public void testNoResults() throws Exception {
+        Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1, null };
+        List<RatedDocument> ratedDocs = new ArrayList<>();
+        for (int i = 0; i < 6; i++) {
+            if (i < relevanceRatings.length) {
+                if (relevanceRatings[i] != null) {
+                    ratedDocs.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i]));
+                }
+            }
+        }
+        SearchHit[] hits = new SearchHit[0];
+        DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
+        EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs);
+        assertEquals(0.0d, result.getQualityLevel(), DELTA);
+        assertEquals(0, filterUnknownDocuments(result.getHitsAndRatings()).size());
+
+        // also check normalized
+        dcg = new DiscountedCumulativeGain(true, null, 10);
+        result = dcg.evaluate("id", hits, ratedDocs);
+        assertEquals(0.0d, result.getQualityLevel(), DELTA);
+        assertEquals(0, filterUnknownDocuments(result.getHitsAndRatings()).size());
+    }
+
     public void testParseFromXContent() throws IOException {
         assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"normalize\": true, \"k\" : 15 }", 2, true, 15);
         assertParsedCorrect("{ \"normalize\": false, \"k\" : 15 }", null, false, 15);

diff --git a/...les/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java b/...les/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java
@@ -158,6 +158,13 @@ public void testEvaluationNoRelevantInResults() {
         assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
     }
 
+    public void testNoResults() throws Exception {
+        SearchHit[] hits = new SearchHit[0];
+        EvalQueryQuality evaluated = (new MeanReciprocalRank()).evaluate("id", hits, Collections.emptyList());
+        assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
+        assertEquals(-1, ((MeanReciprocalRank.Breakdown) evaluated.getMetricDetails()).getFirstRelevantRank());
+    }
+
     public void testXContentRoundtrip() throws IOException {
         MeanReciprocalRank testItem = createTestItem();
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));

diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java
@@ -142,6 +142,14 @@ public void testNoRatedDocs() throws Exception {
         assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
     }
 
+    public void testNoResults() throws Exception {
+        SearchHit[] hits = new SearchHit[0];
+        EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList());
+        assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
+        assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
+        assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
+    }
+
     public void testParseFromXContent() throws IOException {
         String xContent = " {\n" + "   \"relevant_rating_threshold\" : 2" + "}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {