Added template to EvaluationResult interface. Removed the printing of…

… debug messages of the confidence based evaluation.

src/main/java/org/aksw/gerbil/Experimenter.java

@@ -43,6 +43,8 @@ public class Experimenter implements Runnable {
     private EvaluatorFactory evFactory;
     private AnnotatorOutputWriter annotatorOutputWriter = null;
     private SameAsRetriever globalRetriever = null;
+    // just temporary
+    private boolean detailedResults = false;
 
     /**
      * Constructor
@@ -60,8 +62,8 @@ public Experimenter(Overseer overseer, ExperimentDAO experimentDAO, ExperimentTa
         this.evFactory = new EvaluatorFactory();
     }
 
-    public Experimenter(Overseer overseer, ExperimentDAO experimentDAO, SameAsRetriever globalRetriever, EvaluatorFactory evFactory,
-             ExperimentTaskConfiguration configs[], String experimentId) {
+    public Experimenter(Overseer overseer, ExperimentDAO experimentDAO, SameAsRetriever globalRetriever,
+            EvaluatorFactory evFactory, ExperimentTaskConfiguration configs[], String experimentId) {
         this.configs = configs;
         this.experimentId = experimentId;
         this.experimentDAO = experimentDAO;
@@ -71,12 +73,12 @@ public Experimenter(Overseer overseer, ExperimentDAO experimentDAO, SameAsRetrie
     }
 
     @Override
-    public void run() throws FaultyConfigurationException{
+    public void run() throws FaultyConfigurationException {
         Arrays.sort(configs, new ExpTaskConfigComparator());
         List<ExperimentTaskConfiguration> faultyConfigs = new ArrayList<ExperimentTaskConfiguration>();
         int taskId;
         for (int i = 0; i < configs.length; ++i) {
-            try{
+            try {
                 if (couldHaveCachedResult(configs[i])) {
                     taskId = experimentDAO.connectCachedResultOrCreateTask(configs[i].annotatorConfig.getName(),
                             configs[i].datasetConfig.getName(), configs[i].type.name(), configs[i].matching.name(),
@@ -92,15 +94,16 @@ public void run() throws FaultyConfigurationException{
                     ExperimentTask task = new ExperimentTask(taskId, experimentDAO, globalRetriever, evFactory,
                             configs[i]);
                     task.setAnnotatorOutputWriter(annotatorOutputWriter);
-                        overseer.startTask(task);
+                    task.setDetailedResults(detailedResults);
+                    overseer.startTask(task);
                 }
-            }catch (Exception e){
+            } catch (Exception e) {
                 faultyConfigs.add(configs[i]);
             }
         }
-        if(faultyConfigs.isEmpty()){
+        if (faultyConfigs.isEmpty()) {
             LOGGER.info("Experimenter finished the creation of tasks for experiment \"" + experimentId + "\"");
-        }else{
+        } else {
             LOGGER.error("Got few Exception while trying to start some needed tasks.");
             throw new FaultyConfigurationException(faultyConfigs);
         }
@@ -117,4 +120,8 @@ private boolean couldHaveCachedResult(ExperimentTaskConfiguration config) {
     public void setAnnotatorOutputWriter(AnnotatorOutputWriter annotatorOutputWriter) {
         this.annotatorOutputWriter = annotatorOutputWriter;
     }
+
+    public void setDetailedResults(boolean detailedResults) {
+        this.detailedResults = detailedResults;
+    }
 }

src/main/java/org/aksw/gerbil/database/ExperimentDAO.java

@@ -21,7 +21,6 @@
 
 import org.aksw.gerbil.datatypes.ErrorTypes;
 import org.aksw.gerbil.datatypes.ExperimentTaskStatus;
-import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 
 /**
  * This interface defines the methods a class has to implement for making the

src/main/java/org/aksw/gerbil/evaluate/AbstractEvaluationResult.java

@@ -0,0 +1,39 @@
+package org.aksw.gerbil.evaluate;
+
+import java.util.Objects;
+
+/**
+ * An abstract, parameterized implementation of the EvaluationResult interface
+ * that handles the name of the evaluation result. The value (i.e., mainly the
+ * {@link #getValue()} method) has to be implemented by the extending class.
+ * 
+ * @author Michael Röder (michael.roeder@uni-paderborn.de)
+ *
+ * @param <T>
+ */
+public abstract class AbstractEvaluationResult<T> implements EvaluationResult<T> {
+
+    /**
+     * The result's name.
+     */
+    protected String name;
+
+    /**
+     * Constructor.
+     * 
+     * @param name The result's name.
+     */
+    public AbstractEvaluationResult(String name) {
+        this.name = name;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public String toString() {
+        return name + "=" + Objects.toString(getValue());
+    }
+}

src/main/java/org/aksw/gerbil/evaluate/ClassSubTaskEvaluator.java

@@ -23,37 +23,40 @@
 import org.aksw.gerbil.transfer.nif.Marking;
 
 public class ClassSubTaskEvaluator<T extends Marking> extends SubTaskEvaluator<T> {
-
-	private Class<? extends Marking> cleanClass = Marking.class;
 
-    public ClassSubTaskEvaluator(ExperimentTaskConfiguration configuration, List<Evaluator<T>> evaluators, Class<? extends Marking> cleanClass) {
+    private Class<? extends Marking> cleanClass = Marking.class;
+
+    public ClassSubTaskEvaluator(ExperimentTaskConfiguration configuration, List<Evaluator<T>> evaluators,
+            Class<? extends Marking> cleanClass) {
         super(configuration, evaluators);
-        this.cleanClass=cleanClass;
+        this.cleanClass = cleanClass;
     }
 
-    public ClassSubTaskEvaluator(ExperimentTaskConfiguration configuration, Evaluator<T> evaluator,  Class<? extends Marking> cleanClass) {
+    public ClassSubTaskEvaluator(ExperimentTaskConfiguration configuration, Evaluator<T> evaluator,
+            Class<? extends Marking> cleanClass) {
         super(configuration, evaluator);
-        this.cleanClass=cleanClass;
+        this.cleanClass = cleanClass;
     }
 
     @Override
-    public void evaluate(List<List<T>> annotatorResults, List<List<T>> goldStandard, EvaluationResultContainer results) {
+    public void evaluate(List<List<T>> annotatorResults, List<List<T>> goldStandard,
+            EvaluationResultContainer results) {
         super.evaluate(cleanList(annotatorResults), cleanList(goldStandard), results);
     }
-    
-    private List<List<T>> cleanList(List<List<T>> results){
-    	List<List<T>> cleanedResults = new ArrayList<List<T>>();
-    	for(List<T> result : results) {
-    		List<T> cleanedResult = new ArrayList<T>();
-    		for(T marking : result) {
-    			
-    			if(cleanClass.isInstance(marking)) {
-    				cleanedResult.add(marking);
-    			}
-    		}
-    		cleanedResults.add(cleanedResult);
-    	}
-    	return cleanedResults;
+
+    private List<List<T>> cleanList(List<List<T>> results) {
+        List<List<T>> cleanedResults = new ArrayList<List<T>>();
+        for (List<T> result : results) {
+            List<T> cleanedResult = new ArrayList<T>();
+            for (T marking : result) {
+
+                if (cleanClass.isInstance(marking)) {
+                    cleanedResult.add(marking);
+                }
+            }
+            cleanedResults.add(cleanedResult);
+        }
+        return cleanedResults;
     }
 
 }

src/main/java/org/aksw/gerbil/evaluate/DoubleEvaluationResult.java

@@ -16,23 +16,17 @@
  */
 package org.aksw.gerbil.evaluate;
 
-public class DoubleEvaluationResult implements EvaluationResult {
+public class DoubleEvaluationResult extends AbstractEvaluationResult<Double> implements EvaluationResult<Double> {
 
-    private String name;
     private double value;
 
     public DoubleEvaluationResult(String name, double value) {
-        this.name = name;
+        super(name);
         this.value = value;
     }
-
-    @Override
-    public String getName() {
-        return name;
-    }
 
     @Override
-    public Object getValue() {
+    public Double getValue() {
         return value;
     }
 

src/main/java/org/aksw/gerbil/evaluate/EvaluationResult.java

@@ -16,9 +16,23 @@
  */
 package org.aksw.gerbil.evaluate;
 
-public interface EvaluationResult {
+/**
+ * An evaluation result is a simple name-value pair. The name is expected to be
+ * a String while the value is parameterized.
+ * 
+ * @author Michael Röder (michael.roeder@uni-paderborn.de)
+ *
+ * @param <T>
+ */
+public interface EvaluationResult<T> {
 
+    /**
+     * The result's name.
+     */
     public String getName();
-
-    public Object getValue();
+
+    /**
+     * The result's value.
+     */
+    public T getValue();
 }

src/main/java/org/aksw/gerbil/evaluate/EvaluationResultContainer.java

@@ -19,46 +19,46 @@
 import java.util.ArrayList;
 import java.util.List;
 
-public class EvaluationResultContainer implements EvaluationResult {
+public class EvaluationResultContainer implements EvaluationResult<List<EvaluationResult<?>>> {
 
-	private List<EvaluationResult> results;
+    private List<EvaluationResult<?>> results;
 
-	public EvaluationResultContainer() {
-		results = new ArrayList<EvaluationResult>();
-	}
+    public EvaluationResultContainer() {
+        results = new ArrayList<EvaluationResult<?>>();
+    }
 
-	/**
-	 * Copy constructor. Makes a shallow copy of the evaluation result list of
-	 * the given container.
-	 * 
-	 * @param container
-	 */
-	public EvaluationResultContainer(EvaluationResultContainer container) {
-		results = new ArrayList<EvaluationResult>(container.results);
-	}
+    /**
+     * Copy constructor. Makes a shallow copy of the evaluation result list of the
+     * given container.
+     * 
+     * @param container
+     */
+    public EvaluationResultContainer(EvaluationResultContainer container) {
+        results = new ArrayList<EvaluationResult<?>>(container.results);
+    }
 
-	@Override
-	public String getName() {
-		return null;
-	}
+    @Override
+    public String getName() {
+        return null;
+    }
 
-	@Override
-	public Object getValue() {
-		return results;
-	}
+    @Override
+    public List<EvaluationResult<?>> getValue() {
+        return results;
+    }
 
-	public void addResult(EvaluationResult result) {
-		results.add(result);
-	}
+    public void addResult(EvaluationResult<?> result) {
+        results.add(result);
+    }
 
-	public void addResults(EvaluationResult... results) {
-		for (int i = 0; i < results.length; i++) {
-			this.results.add(results[i]);
-		}
-	}
+    public void addResults(EvaluationResult<?>... results) {
+        for (int i = 0; i < results.length; i++) {
+            this.results.add(results[i]);
+        }
+    }
 
-	public List<EvaluationResult> getResults() {
-		return results;
-	}
+    public List<EvaluationResult<?>> getResults() {
+        return results;
+    }
 
 }

src/main/java/org/aksw/gerbil/evaluate/IntEvaluationResult.java

@@ -16,23 +16,17 @@
  */
 package org.aksw.gerbil.evaluate;
 
-public class IntEvaluationResult implements EvaluationResult {
+public class IntEvaluationResult extends AbstractEvaluationResult<Integer> implements EvaluationResult<Integer> {
 
-    private String name;
     private int value;
 
     public IntEvaluationResult(String name, int value) {
-        this.name = name;
+        super(name);
         this.value = value;
     }
 
     @Override
-    public String getName() {
-        return name;
-    }
-
-    @Override
-    public Object getValue() {
+    public Integer getValue() {
         return value;
     }
 

src/main/java/org/aksw/gerbil/evaluate/ObjectEvaluationResult.java

@@ -0,0 +1,17 @@
+package org.aksw.gerbil.evaluate;
+
+public class ObjectEvaluationResult<T> extends AbstractEvaluationResult<T> implements EvaluationResult<T> {
+
+    protected T value;
+
+    public ObjectEvaluationResult(String name, T value) {
+        super(name);
+        this.value = value;
+    }
+
+    @Override
+    public T getValue() {
+        return value;
+    }
+
+}

src/main/java/org/aksw/gerbil/evaluate/impl/ClassConsideringFMeasureCalculator.java

@@ -60,7 +60,7 @@ public ClassConsideringFMeasureCalculator(MatchingsCounter<T> matchingsCounter,
     public void evaluate(List<List<T>> annotatorResults, List<List<T>> goldStandard,
             EvaluationResultContainer results) {
         // the super class performs the matching counter calls
-        ScoredEvaluationCountsArray counts = generateMatchingCounts(annotatorResults, goldStandard);
+        ScoredEvaluationCountsArray counts = generateMatchingCounts(annotatorResults, goldStandard, results);
         if ((counts.truePositiveSums.length > 0) && (counts.falseNegativeSums.length > 0)
                 && (counts.falsePositiveSums.length > 0)) {
             double threshold = calculateMicroFMeasure(counts, results);
@@ -69,7 +69,7 @@ public void evaluate(List<List<T>> annotatorResults, List<List<T>> goldStandard,
             String classLabel;
             for (int i = 0; i < markingClasses.length; ++i) {
                 counts = generateMatchingCounts(markingFilters[i].filterListOfLists(annotatorResults),
-                        markingFilters[i].filterListOfLists(goldStandard));
+                        markingFilters[i].filterListOfLists(goldStandard), null);
                 if ((counts.truePositiveSums[0] + counts.falseNegativeSums[0] + counts.falsePositiveSums[0]) > 0) {
                     classLabel = markingClasses[i].getLabel();
                     calculateMicroFMeasure(counts, classLabel + MICRO_PRECISION_NAME_APPENDIX,