[ issue #5 ] composite / compound metrics

SeaseLtd · May 30, 2018 · df44859 · df44859
1 parent c258d2e
commit df44859
Show file tree

Hide file tree

Showing 16 changed files with 293 additions and 30 deletions.
diff --git a/rre-core/src/main/java/io/sease/rre/Calculator.java b/rre-core/src/main/java/io/sease/rre/Calculator.java
@@ -22,4 +22,10 @@ public static BigDecimal subtract(final BigDecimal minuend, final BigDecimal sub
     public static BigDecimal divide(final BigDecimal dividend, final BigDecimal divisor) {
         return dividend.divide(divisor, 4, RoundingMode.CEILING);
     }
+
+    public static BigDecimal divide(final BigDecimal dividend, final int divisor) {
+        return dividend.divide(new BigDecimal(divisor), 4, RoundingMode.CEILING);
+    }
+
+
 }
diff --git a/rre-core/src/main/java/io/sease/rre/a.json b/rre-core/src/main/java/io/sease/rre/a.json
@@ -0,0 +1,15 @@
+Corpus :
+  NDCG: {
+    Delta: +0.04
+    V1: 0.94
+    V2: 0.98
+    V3: 0.72
+    Topics: [
+      {
+        Delta: +0.04
+        V1: 0.94
+        V2: 0.98
+        V3: 0.72
+      }
+    ]
+}
diff --git a/rre-core/src/main/java/io/sease/rre/core/Engine.java b/rre-core/src/main/java/io/sease/rre/core/Engine.java
@@ -3,16 +3,16 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.sease.rre.core.domain.*;
+import io.sease.rre.core.domain.metrics.CompoundMetric;
 import io.sease.rre.core.domain.metrics.Metric;
+import io.sease.rre.core.event.MetricEvent;
+import io.sease.rre.core.event.MetricEventListener;
 import io.sease.rre.search.api.QueryOrSearchResponse;
 import io.sease.rre.search.api.SearchPlatform;
 
 import java.io.File;
 import java.nio.file.Files;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
@@ -33,7 +33,11 @@ public class Engine {
     private final File ratingsFolder;
     private final File templatesFolder;
 
+    private final List<Class<? extends Metric>> availableMetricsDefs;
+    private final List<Class<? extends CompoundMetric>> availableCompoundMetricsDefs;
+
     private final SearchPlatform platform;
+    private final List<MetricEventListener> metricEventListeners = new ArrayList<>();
 
     /**
      * Builds a new {@link Engine} instance with the given data.
@@ -49,12 +53,26 @@ public Engine(
             final String configurationsFolderPath,
             final String corporaFolderPath,
             final String ratingsFolderPath,
-            final String templatesFolderPath) {
+            final String templatesFolderPath,
+            final List<String> metrics,
+            final List<String> compoundMetrics) {
         this.configurationsFolder = new File(configurationsFolderPath);
         this.corporaFolder = new File(corporaFolderPath);
         this.ratingsFolder = new File(ratingsFolderPath);
         this.templatesFolder = new File(templatesFolderPath);
         this.platform = platform;
+
+        this.availableMetricsDefs =
+                metrics.stream()
+                        .filter(Objects::nonNull)
+                        .map(this::newMetricDefinition)
+                        .collect(toList());
+
+        this.availableCompoundMetricsDefs =
+                compoundMetrics.stream()
+                        .filter(Objects::nonNull)
+                        .map(this::newCompoundMetricDefinition)
+                        .collect(toList());
     }
 
     /**
@@ -65,12 +83,6 @@ public Engine(
      */
     @SuppressWarnings("unchecked")
     public Evaluation evaluate(final Map<String, Object> configuration){
-        final List<Class<? extends Metric>> availableMetricsDefs =
-                ((List<String>)configuration.get("metrics")).stream()
-                    .filter(Objects::nonNull)
-                    .map(this::newMetricInstance)
-                    .collect(toList());
-
         final JsonNode ratingsNode = ratings();
 
         platform.beforeStart(configuration);
@@ -85,7 +97,10 @@ public Evaluation evaluate(final Map<String, Object> configuration){
         }
 
         final Evaluation evaluation = new Evaluation();
-        final Corpus corpus = evaluation.findOrCreate(data.getName(), Corpus::new);
+        final Corpus corpus =
+                withMetricEventListening(
+                    evaluation.findOrCreate(data.getName(), Corpus::new))
+                        .init(availableCompoundMetricsDefs);
 
         stream(safe(configurationsFolder.listFiles(file -> file.isDirectory() && !file.isHidden())))
                 .flatMap(versionFolder -> stream(safe(versionFolder.listFiles(file -> file.isDirectory() && !file.isHidden()))))
@@ -94,17 +109,21 @@ public Evaluation evaluate(final Map<String, Object> configuration){
                     final String version = folder.getParentFile().getName();
                     final String internalIndexName = indexName + "_" + version;
 
-                   // final ConfigurationVersion configurationVersion = corpus.add(new ConfigurationVersion(folder.getParentFile().getName()));
                     platform.load(data, folder, internalIndexName);
 
                     all(ratingsNode.get("topics"))
                             .forEach(topicNode -> {
                                 final Topic topic =
-                                        corpus.findOrCreate(topicNode.get("description").asText(), Topic::new);
+                                        withMetricEventListening(
+                                                corpus.findOrCreate(
+                                                        topicNode.get("description").asText(), Topic::new))
+                                                .init(availableCompoundMetricsDefs);
 
                                 all(topicNode.get("query_groups"))
                                         .forEach(queryGroup -> {
-                                            final QueryGroup group = topic.findOrCreate(queryGroup.get("name").asText(), QueryGroup::new);
+                                            final QueryGroup group =
+                                                    withMetricEventListening(
+                                                            topic.findOrCreate(queryGroup.get("name").asText(), QueryGroup::new)).init(availableCompoundMetricsDefs);
                                             all(queryGroup.get("queries"))
                                                     .forEach(queryNode -> {
                                                         String query = queryTemplate(queryNode.get("template").asText());
@@ -113,14 +132,16 @@ public Evaluation evaluate(final Map<String, Object> configuration){
                                                             query = query.replace(name, queryNode.get("placeholders").get(name).asText());
                                                         }
 
-                                                        final QueryEvaluation queryEvaluation = group.findOrCreate(query, QueryEvaluation::new);
+                                                        final QueryEvaluation queryEvaluation =
+                                                                withMetricEventListening(group.findOrCreate(query, QueryEvaluation::new)).init(availableCompoundMetricsDefs);
                                                         final ConfigurationVersion configurationVersion = queryEvaluation.findOrCreate(version, ConfigurationVersion::new);
 
                                                         final JsonNode relevantDocuments = queryGroup.get("relevant_documents");
                                                         final QueryOrSearchResponse response = platform.executeQuery(internalIndexName, query, Math.max(10, relevantDocuments.size()));
 
                                                         configurationVersion.prepare(availableMetrics(availableMetricsDefs, idFieldName, relevantDocuments, response.totalHits()));
                                                         response.hits().forEach(hit -> configurationVersion.stream().forEach(metric -> metric.collect(hit)));
+                                                        configurationVersion.stream().forEach(metric -> broadcast(metric, version));
                                                     });
                                         });
                             });
@@ -160,6 +181,16 @@ private List<Metric> availableMetrics(
                 .collect(toList());
     }
 
+    private <L extends MetricEventListener> L withMetricEventListening(final L listener) {
+        metricEventListeners.add(listener);
+        return listener;
+    }
+
+    private void broadcast(final Metric metric, final String version) {
+        final MetricEvent event = new MetricEvent(metric, version, this);
+        metricEventListeners.forEach(listener -> listener.newMetricHasBeenComputed(event));
+    }
+
     /**
      * Loads the query template associated with the given name.
      *
@@ -199,11 +230,20 @@ private Stream<JsonNode> all(final JsonNode source) {
     }
 
     @SuppressWarnings("unchecked")
-    private Class<? extends Metric> newMetricInstance(final String clazzName) {
+    private Class<? extends Metric> newMetricDefinition(final String clazzName) {
         try {
             return (Class<? extends Metric>) Class.forName(clazzName);
         } catch (final Exception exception) {
             throw new IllegalArgumentException(exception);
         }
     }
+
+    @SuppressWarnings("unchecked")
+    private Class<? extends CompoundMetric> newCompoundMetricDefinition(final String clazzName) {
+        try {
+            return (Class<? extends CompoundMetric>) Class.forName(clazzName);
+        } catch (final Exception exception) {
+            throw new IllegalArgumentException(exception);
+        }
+    }
 }
diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/ConfigurationVersion.java b/rre-core/src/main/java/io/sease/rre/core/domain/ConfigurationVersion.java
@@ -2,10 +2,12 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import io.sease.rre.core.domain.metrics.CompoundMetric;
 import io.sease.rre.core.domain.metrics.Metric;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Stream;
 
 /**
@@ -28,6 +30,12 @@ public List getChildren() {
         return super.getChildren();
     }
 
+    @Override
+    @JsonIgnore
+    public Map<String, List<CompoundMetric>> getCompoundMetrics() {
+        return super.getCompoundMetrics();
+    }
+
     /**
      * Adds to this {@link ConfigurationVersion} the given list of metrics.
      * Note that at this stage the metrics are empty, without data and value.

diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/DomainMember.java b/rre-core/src/main/java/io/sease/rre/core/domain/DomainMember.java
@@ -1,23 +1,31 @@
 package io.sease.rre.core.domain;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.sease.rre.core.domain.metrics.CompoundMetric;
+import io.sease.rre.core.event.MetricEvent;
+import io.sease.rre.core.event.MetricEventListener;
+
+import java.util.*;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 
+import static java.util.Optional.ofNullable;
+import static java.util.stream.Collectors.toList;
+
 /**
  * Supertype layer for all RRE domain entities.
  * It defines the common composite structure shared by all RRE entities.
  *
  * @author agazzarini
  * @since 1.0
  */
-public abstract class DomainMember<C extends DomainMember> {
+public abstract class DomainMember<C extends DomainMember> implements MetricEventListener {
     private String name;
 
     private Map<String, C> childrenLookupCache = new HashMap<>();
+    protected Map<String, List<CompoundMetric>> compoundMetrics = new LinkedHashMap<>();
+
+    protected List<Class<? extends CompoundMetric>> compoundMetricsDef;
 
     private List<C> children = new ArrayList<>();
 
@@ -31,6 +39,31 @@ public C add(final C child) {
         return child;
     }
 
+    public <T extends DomainMember> T init(final List<Class<? extends CompoundMetric>> metricsDef) {
+        this.compoundMetricsDef = metricsDef;
+        return (T) this;
+    }
+
+    @Override
+    public void newMetricHasBeenComputed(final MetricEvent event) {
+        compoundMetrics.computeIfAbsent(event.getVersion(), v -> newCompoundMetricSet())
+            .forEach(compoundMetric -> compoundMetric.collect(event.getMetric()));
+    }
+
+    private List<CompoundMetric> newCompoundMetricSet() {
+        return compoundMetricsDef
+                .stream()
+                .map(clazz -> {
+                    try {
+                        return clazz.newInstance();
+                    } catch (final Exception exception) {
+                        exception.printStackTrace();
+                        return null;
+                    }})
+                .filter(Objects::nonNull)
+                .collect(toList());
+    }
+
     public C findOrCreate(final String name, final Supplier<C> factory) {
         return childrenLookupCache.computeIfAbsent(name, key -> add((C) factory.get().setName(name)));
     }
@@ -40,6 +73,11 @@ public DomainMember setName(final String name) {
         return this;
     }
 
+    @JsonProperty("compound-metrics")
+    public Map<String, List<CompoundMetric>> getCompoundMetrics() {
+        return compoundMetrics;
+    }
+
     /**
      * Returns the children stream.
      *

diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/Evaluation.java b/rre-core/src/main/java/io/sease/rre/core/domain/Evaluation.java
@@ -1,12 +1,15 @@
 package io.sease.rre.core.domain;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.sease.rre.core.EventCollector;
+import io.sease.rre.core.domain.metrics.CompoundMetric;
 
 import java.text.DateFormat;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 
 /**
  * The evaluation result.
@@ -24,6 +27,12 @@ public Evaluation() {
         setName("Ranking Evaluation Report - created on " + DateFormat.getDateInstance(DateFormat.FULL, Locale.ENGLISH).format(new Date()));
     }
 
+    @Override
+    @JsonIgnore
+    public Map<String, List<CompoundMetric>> getCompoundMetrics() {
+        return super.getCompoundMetrics();
+    }
+
     @Override
     public void collect(final QueryEvaluation event) {
         // TODO

diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/QueryEvaluationCollector.java b/rre-core/src/main/java/io/sease/rre/core/domain/QueryEvaluationCollector.java
diff --git a/rre-core/src/main/java/io/sease/rre/core/domain/QueryGroup.java b/rre-core/src/main/java/io/sease/rre/core/domain/QueryGroup.java
@@ -1,16 +1,24 @@
 package io.sease.rre.core.domain;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import io.sease.rre.core.domain.metrics.CompoundMetric;
+import io.sease.rre.core.domain.metrics.Metric;
+import io.sease.rre.core.event.MetricEvent;
+import io.sease.rre.core.event.MetricEventListener;
 
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
+
+import static java.util.Optional.ofNullable;
 
 /**
  * A group of queries which are supposed to produce the same exact results.
  *
  * @author agazzarini
  * @since 1.0
  */
-public class QueryGroup extends DomainMember<QueryEvaluation> {
+public class QueryGroup extends DomainMember<QueryEvaluation> implements MetricEventListener {
     @Override
     @JsonProperty("query-evaluations")
     public List<QueryEvaluation> getChildren() {