From 9728b4511e41fcf362fe483368f3ec172f6c6dfe Mon Sep 17 00:00:00 2001 From: Matt Pearce Date: Thu, 26 Mar 2020 14:07:13 +0000 Subject: [PATCH 1/3] Add generic evaluation plugin to allow non-Solr/ES search platforms to be used. --- rre-maven-plugin/pom.xml | 1 + .../rre-maven-evaluation-plugin/README.md | 70 +++++++++ .../rre-maven-evaluation-plugin/pom.xml | 41 ++++++ .../plugin/evaluation/RREvaluateMojo.java | 139 ++++++++++++++++++ .../ArgConstructorSearchPlatform.java | 73 +++++++++ .../plugin/evaluation/RREvaluateMojoTest.java | 55 +++++++ .../bad_searchplatform_pom.xml | 54 +++++++ .../searchplatform_args_constructor_pom.xml | 54 +++++++ 8 files changed, 487 insertions(+) create mode 100644 rre-maven-plugin/rre-maven-evaluation-plugin/README.md create mode 100644 rre-maven-plugin/rre-maven-evaluation-plugin/pom.xml create mode 100644 rre-maven-plugin/rre-maven-evaluation-plugin/src/main/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojo.java create mode 100644 rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/ArgConstructorSearchPlatform.java create mode 100644 rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojoTest.java create mode 100644 rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml create mode 100644 rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml diff --git a/rre-maven-plugin/pom.xml b/rre-maven-plugin/pom.xml index 181253d7..81f6fb89 100644 --- a/rre-maven-plugin/pom.xml +++ b/rre-maven-plugin/pom.xml @@ -34,6 +34,7 @@ rre-maven-elasticsearch-plugin rre-maven-external-elasticsearch-plugin rre-maven-external-solr-plugin + rre-maven-evaluation-plugin pom diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/README.md b/rre-maven-plugin/rre-maven-evaluation-plugin/README.md new file mode 100644 index 00000000..f374ac4a --- /dev/null +++ b/rre-maven-plugin/rre-maven-evaluation-plugin/README.md @@ -0,0 +1,70 @@ +Generic Maven Evaluation plugin +=============================== + +This evaluation plugin allows the use of alternative search APIs than the +supplied Solr and Elasticsearch choices. + + +## Usage + +An example pom.xml, excluding most RRE configuration, would look like: + +``` + + 4.0.0 + com.mycompany + my-evaluation-project + 1.0 + pom + + + 1.8 + 1.8 + + + + + + io.sease + rre-maven-evaluation-plugin + 1.0 + + + com.mycompany + my-rre-searchplatform + 1.1 + + + + com.mycompany.search.MySearchPlatform + + + ... config details ... + + + + + + search-quality-evaluation + package + + evaluate + + + + + + + +``` + + +## Implementing SearchPlatform + +To supply your own search platform, you need to implement the +[SearchPlatform](https://github.com/SeaseLtd/rated-ranking-evaluator/blob/master/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatform.java) +interface, which provides the connection between RRE and your search engine. +Your implementation should have a zero-argument constructor - the +configuration properties (set in the `searchPlatformConfiguration` block +in your pom.xml) will be passed in the `beforeStart()` phase. diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/pom.xml b/rre-maven-plugin/rre-maven-evaluation-plugin/pom.xml new file mode 100644 index 00000000..e5e030dc --- /dev/null +++ b/rre-maven-plugin/rre-maven-evaluation-plugin/pom.xml @@ -0,0 +1,41 @@ + + + + + rre-maven-plugin + io.sease + 1.0 + + + 4.0.0 + rre-maven-evaluation-plugin + maven-plugin + 1.0 + RRE - Maven Evaluation Plugin + + + + io.sease + rre-search-platform-api + ${project.version} + + + diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/main/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojo.java b/rre-maven-plugin/rre-maven-evaluation-plugin/src/main/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojo.java new file mode 100644 index 00000000..faced173 --- /dev/null +++ b/rre-maven-plugin/rre-maven-evaluation-plugin/src/main/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojo.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.sease.rre.maven.plugin.evaluation; + +import io.sease.rre.core.Engine; +import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; +import io.sease.rre.core.domain.metrics.MetricClassManager; +import io.sease.rre.core.evaluation.EvaluationConfiguration; +import io.sease.rre.persistence.PersistenceConfiguration; +import io.sease.rre.search.api.SearchPlatform; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * RREvaluation Mojo (Generic search platform binding). + * + * @author mattflax (mpearce@opensourceconnections.com) + * @since 1.0 + */ +@Mojo(name = "evaluate", inheritByDefault = false, defaultPhase = LifecyclePhase.PACKAGE) +public class RREvaluateMojo extends AbstractMojo { + + @Parameter(name = "configurations-folder", defaultValue = "${basedir}/src/etc/configuration_sets") + private String configurationsFolder; + + @Parameter(name = "corpora-folder", defaultValue = "${basedir}/src/etc/corpora") + private String corporaFolder; + + @Parameter(name = "ratings-folder", defaultValue = "${basedir}/src/etc/ratings)") + private String ratingsFolder; + + @Parameter(name = "templates-folder", defaultValue = "${basedir}/src/etc/templates") + private String templatesFolder; + + @Parameter(name = "force-refresh", defaultValue = "true") + private boolean forceRefresh; + + @Parameter(name = "checksum-file") + private String checksumFile; + + @Parameter(name = "metrics", defaultValue = "io.sease.rre.core.domain.metrics.impl.PrecisionAtOne,io.sease.rre.core.domain.metrics.impl.PrecisionAtTwo,io.sease.rre.core.domain.metrics.impl.PrecisionAtThree,io.sease.rre.core.domain.metrics.impl.PrecisionAtTen") + private List metrics; + + @Parameter(name = "parameterizedMetrics") @SuppressWarnings("rawtypes") + private Map parameterizedMetrics; + + @Parameter(name = "fields", defaultValue = "*,score") + private String fields; + + @Parameter(name = "include") + private List include; + + @Parameter(name = "exclude") + private List exclude; + + @Parameter(name = "maximumGrade", defaultValue = "3") + private float maximumGrade; + + @Parameter(name = "missingGrade", defaultValue = "2") + private float missingGrade; + + @Parameter(name = "persistence") + private PersistenceConfiguration persistence = PersistenceConfiguration.DEFAULT_CONFIG; + + @Parameter(name = "evaluation") + private EvaluationConfiguration evaluation = EvaluationConfiguration.DEFAULT_CONFIG; + + @Parameter(name = "searchPlatform", required = true) + private String searchPlatform; + + @Parameter(name = "searchPlatformConfiguration") + private Map searchPlatformConfiguration = new HashMap<>(); + + @Override + public void execute() throws MojoExecutionException { + try (final SearchPlatform platform = buildSearchPlatform()) { + final MetricClassManager metricClassManager = MetricClassConfigurationManager.getInstance() + .setDefaultMaximumGrade(maximumGrade) + .setDefaultMissingGrade(missingGrade) + .buildMetricClassManager(metrics, parameterizedMetrics); + final Engine engine = new Engine( + platform, + configurationsFolder, + corporaFolder, + ratingsFolder, + templatesFolder, + metricClassManager, + fields.split(","), + exclude, + include, + checksumFile, + persistence, + evaluation); + engine.evaluate(searchPlatformConfiguration); + } catch (final IOException exception) { + throw new MojoExecutionException(exception.getMessage(), exception); + } + } + + @SuppressWarnings("unchecked") + protected SearchPlatform buildSearchPlatform() throws MojoExecutionException { + final SearchPlatform platform; + + try { + Class klazz = (Class) Class.forName(searchPlatform); + platform = klazz.newInstance(); + } catch (ClassNotFoundException e) { + throw new MojoExecutionException("Caught ClassNotFoundException trying to build SearchPlatform instance: " + e.getMessage(), e); + } catch (InstantiationException e) { + throw new MojoExecutionException("Caught InstantiationException trying to construct SearchPlatform instance: " + e.getMessage(), e); + } catch (IllegalAccessException e) { + throw new MojoExecutionException("Caught IllegalAccessException for SearchPlatform instance: " + e.getMessage(), e); + } + + return platform; + } +} \ No newline at end of file diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/ArgConstructorSearchPlatform.java b/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/ArgConstructorSearchPlatform.java new file mode 100644 index 00000000..cfad45b4 --- /dev/null +++ b/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/ArgConstructorSearchPlatform.java @@ -0,0 +1,73 @@ +package io.sease.rre.maven.plugin.evaluation; + +import io.sease.rre.search.api.QueryOrSearchResponse; +import io.sease.rre.search.api.SearchPlatform; + +import java.io.File; +import java.util.Map; + +/** + * A dummy search platform implementation for testing the Evaluation Mojo. + * + * @author Matt Pearce (mpearce@opensourceconnections.com) + */ +public class ArgConstructorSearchPlatform implements SearchPlatform { + + public ArgConstructorSearchPlatform(String dummyText) { + + } + + @Override + public void beforeStart(Map configuration) { + + } + + @Override + public void load(File dataToBeIndexed, File configFolder, String collection, String version) { + + } + + @Override + public void start() { + + } + + @Override + public void afterStart() { + + } + + @Override + public void beforeStop() { + + } + + @Override + public QueryOrSearchResponse executeQuery(String collection, String version, String query, String[] fields, int maxRows) { + return null; + } + + @Override + public String getName() { + return null; + } + + @Override + public boolean isRefreshRequired() { + return false; + } + + @Override + public boolean isSearchPlatformConfiguration(String indexName, File file) { + return false; + } + + @Override + public boolean isCorporaRequired() { + return false; + } + + @Override + public void close() { + } +} diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojoTest.java b/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojoTest.java new file mode 100644 index 00000000..1a8b3270 --- /dev/null +++ b/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojoTest.java @@ -0,0 +1,55 @@ +package io.sease.rre.maven.plugin.evaluation; + +import org.apache.maven.plugin.Mojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.testing.MojoRule; +import org.junit.Rule; +import org.junit.Test; + +import static org.junit.Assert.*; + + +/** + * Unit tests for the generic evaluation Maven plugin. + * + * @author Matt Pearce (mpearce@opensourceconnections.com) + */ +public class RREvaluateMojoTest { + + @Rule + public MojoRule rule = new MojoRule() { + @Override + protected void before() { + } + + @Override + protected void after() { + } + }; + + @Test + public void throwsMojoException_whenSearchPlatformNotFound() throws Exception { + Mojo mojo = rule.lookupMojo("evaluate", "src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml"); + assertNotNull(mojo); + + try { + mojo.execute(); + fail("Expected MojoExecutionException"); + } catch (MojoExecutionException e) { + assertTrue(e.getCause() instanceof ClassNotFoundException); + } + } + + @Test + public void throwsMojoException_whenSearchPlatformCannotBeConstructed() throws Exception { + Mojo mojo = rule.lookupMojo("evaluate", "src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml"); + assertNotNull(mojo); + + try { + mojo.execute(); + fail("Expected MojoExecutionException"); + } catch (MojoExecutionException e) { + assertTrue(e.getCause() instanceof InstantiationException); + } + } +} diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml b/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml new file mode 100644 index 00000000..fab1bd9d --- /dev/null +++ b/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + io.sease + rre-maven-evaluation-plugin + 1.0 + pom + + + 1.8 + 1.8 + + + + + + io.sease + rre-maven-evaluation-plugin + 1.0 + + no.such.SearchPlatform + + + + search-quality-evaluation + package + + evaluate + + + + + + + diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml b/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml new file mode 100644 index 00000000..f8867e96 --- /dev/null +++ b/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + io.sease + rre-maven-evaluation-plugin + 1.0 + pom + + + 1.8 + 1.8 + + + + + + io.sease + rre-maven-evaluation-plugin + 1.0 + + io.sease.rre.maven.plugin.evaluation.ArgConstructorSearchPlatform + + + + search-quality-evaluation + package + + evaluate + + + + + + + From b4cc2c55b74e688594a3dd787815e66816a4f75f Mon Sep 17 00:00:00 2001 From: Matt Pearce Date: Thu, 26 Mar 2020 15:22:09 +0000 Subject: [PATCH 2/3] Rename plugin to rre-maven-generic-search-plugin. --- rre-maven-plugin/pom.xml | 2 +- .../README.md | 6 +++--- .../pom.xml | 2 +- .../io/sease/rre/maven/plugin/search}/RREvaluateMojo.java | 2 +- .../maven/plugin/search}/ArgConstructorSearchPlatform.java | 2 +- .../sease/rre/maven/plugin/search}/RREvaluateMojoTest.java | 4 ++-- .../resources/evaluateMojoTests/bad_searchplatform_pom.xml | 2 +- .../searchplatform_args_constructor_pom.xml | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) rename rre-maven-plugin/{rre-maven-evaluation-plugin => rre-maven-generic-search-plugin}/README.md (93%) rename rre-maven-plugin/{rre-maven-evaluation-plugin => rre-maven-generic-search-plugin}/pom.xml (96%) rename rre-maven-plugin/{rre-maven-evaluation-plugin/src/main/java/io/sease/rre/maven/plugin/evaluation => rre-maven-generic-search-plugin/src/main/java/io/sease/rre/maven/plugin/search}/RREvaluateMojo.java (99%) rename rre-maven-plugin/{rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation => rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search}/ArgConstructorSearchPlatform.java (96%) rename rre-maven-plugin/{rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation => rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search}/RREvaluateMojoTest.java (93%) rename rre-maven-plugin/{rre-maven-evaluation-plugin => rre-maven-generic-search-plugin}/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml (96%) rename rre-maven-plugin/{rre-maven-evaluation-plugin => rre-maven-generic-search-plugin}/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml (91%) diff --git a/rre-maven-plugin/pom.xml b/rre-maven-plugin/pom.xml index 81f6fb89..f380cfae 100644 --- a/rre-maven-plugin/pom.xml +++ b/rre-maven-plugin/pom.xml @@ -34,7 +34,7 @@ rre-maven-elasticsearch-plugin rre-maven-external-elasticsearch-plugin rre-maven-external-solr-plugin - rre-maven-evaluation-plugin + rre-maven-generic-search-plugin pom diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/README.md b/rre-maven-plugin/rre-maven-generic-search-plugin/README.md similarity index 93% rename from rre-maven-plugin/rre-maven-evaluation-plugin/README.md rename to rre-maven-plugin/rre-maven-generic-search-plugin/README.md index f374ac4a..c0dde2cb 100644 --- a/rre-maven-plugin/rre-maven-evaluation-plugin/README.md +++ b/rre-maven-plugin/rre-maven-generic-search-plugin/README.md @@ -1,7 +1,7 @@ -Generic Maven Evaluation plugin +Generic Maven Search plugin =============================== -This evaluation plugin allows the use of alternative search APIs than the +This search plugin allows the use of alternative search APIs than the supplied Solr and Elasticsearch choices. @@ -27,7 +27,7 @@ An example pom.xml, excluding most RRE configuration, would look like: io.sease - rre-maven-evaluation-plugin + rre-maven-generic-search-plugin 1.0 diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/pom.xml b/rre-maven-plugin/rre-maven-generic-search-plugin/pom.xml similarity index 96% rename from rre-maven-plugin/rre-maven-evaluation-plugin/pom.xml rename to rre-maven-plugin/rre-maven-generic-search-plugin/pom.xml index e5e030dc..8897f434 100644 --- a/rre-maven-plugin/rre-maven-evaluation-plugin/pom.xml +++ b/rre-maven-plugin/rre-maven-generic-search-plugin/pom.xml @@ -26,7 +26,7 @@ 4.0.0 - rre-maven-evaluation-plugin + rre-maven-generic-search-plugin maven-plugin 1.0 RRE - Maven Evaluation Plugin diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/main/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojo.java b/rre-maven-plugin/rre-maven-generic-search-plugin/src/main/java/io/sease/rre/maven/plugin/search/RREvaluateMojo.java similarity index 99% rename from rre-maven-plugin/rre-maven-evaluation-plugin/src/main/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojo.java rename to rre-maven-plugin/rre-maven-generic-search-plugin/src/main/java/io/sease/rre/maven/plugin/search/RREvaluateMojo.java index faced173..e0b0e065 100644 --- a/rre-maven-plugin/rre-maven-evaluation-plugin/src/main/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojo.java +++ b/rre-maven-plugin/rre-maven-generic-search-plugin/src/main/java/io/sease/rre/maven/plugin/search/RREvaluateMojo.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.sease.rre.maven.plugin.evaluation; +package io.sease.rre.maven.plugin.search; import io.sease.rre.core.Engine; import io.sease.rre.core.domain.metrics.MetricClassConfigurationManager; diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/ArgConstructorSearchPlatform.java b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/ArgConstructorSearchPlatform.java similarity index 96% rename from rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/ArgConstructorSearchPlatform.java rename to rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/ArgConstructorSearchPlatform.java index cfad45b4..801419b2 100644 --- a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/ArgConstructorSearchPlatform.java +++ b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/ArgConstructorSearchPlatform.java @@ -1,4 +1,4 @@ -package io.sease.rre.maven.plugin.evaluation; +package io.sease.rre.maven.plugin.search; import io.sease.rre.search.api.QueryOrSearchResponse; import io.sease.rre.search.api.SearchPlatform; diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojoTest.java b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/RREvaluateMojoTest.java similarity index 93% rename from rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojoTest.java rename to rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/RREvaluateMojoTest.java index 1a8b3270..b1463526 100644 --- a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/java/io/sease/rre/maven/plugin/evaluation/RREvaluateMojoTest.java +++ b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/java/io/sease/rre/maven/plugin/search/RREvaluateMojoTest.java @@ -1,4 +1,4 @@ -package io.sease.rre.maven.plugin.evaluation; +package io.sease.rre.maven.plugin.search; import org.apache.maven.plugin.Mojo; import org.apache.maven.plugin.MojoExecutionException; @@ -10,7 +10,7 @@ /** - * Unit tests for the generic evaluation Maven plugin. + * Unit tests for the generic search Maven plugin. * * @author Matt Pearce (mpearce@opensourceconnections.com) */ diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml similarity index 96% rename from rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml rename to rre-maven-plugin/rre-maven-generic-search-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml index fab1bd9d..67b4c8c8 100644 --- a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml +++ b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/resources/evaluateMojoTests/bad_searchplatform_pom.xml @@ -34,7 +34,7 @@ io.sease - rre-maven-evaluation-plugin + rre-maven-generic-search-plugin 1.0 no.such.SearchPlatform diff --git a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml similarity index 91% rename from rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml rename to rre-maven-plugin/rre-maven-generic-search-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml index f8867e96..caf7447d 100644 --- a/rre-maven-plugin/rre-maven-evaluation-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml +++ b/rre-maven-plugin/rre-maven-generic-search-plugin/src/test/resources/evaluateMojoTests/searchplatform_args_constructor_pom.xml @@ -34,10 +34,10 @@ io.sease - rre-maven-evaluation-plugin + rre-maven-generic-search-plugin 1.0 - io.sease.rre.maven.plugin.evaluation.ArgConstructorSearchPlatform + io.sease.rre.maven.plugin.search.ArgConstructorSearchPlatform From 1e67a2aa2ae0895f86a0b821bc3ec5a55715f4d6 Mon Sep 17 00:00:00 2001 From: Matt Pearce Date: Thu, 26 Mar 2020 15:49:57 +0000 Subject: [PATCH 3/3] Add archetype for the generic search plugin. --- rre-maven-archetype/pom.xml | 1 + .../pom.xml | 29 +++++ .../META-INF/maven/archetype-metadata.xml | 47 +++++++ .../resources/archetype-resources/README.md | 31 +++++ .../resources/archetype-resources/pom.xml | 117 ++++++++++++++++++ .../src/etc/configuration_sets/README.md | 12 ++ .../etc/configuration_sets/v1.0/settings.json | 3 + .../etc/configuration_sets/v1.1/settings.json | 3 + .../src/etc/ratings/README.md | 2 + .../src/etc/ratings/ratings_example.json | 67 ++++++++++ .../src/etc/templates/README.md | 11 ++ .../filter_by_number_of_strings.json | 4 + .../src/etc/templates/only_q.json | 3 + 13 files changed, 330 insertions(+) create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/pom.xml create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/README.md create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/pom.xml create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/README.md create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.0/settings.json create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.1/settings.json create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/README.md create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/ratings_example.json create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/README.md create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/filter_by_number_of_strings.json create mode 100644 rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/only_q.json diff --git a/rre-maven-archetype/pom.xml b/rre-maven-archetype/pom.xml index c7e7094a..4402f5e0 100644 --- a/rre-maven-archetype/pom.xml +++ b/rre-maven-archetype/pom.xml @@ -34,5 +34,6 @@ rre-maven-elasticsearch-archetype rre-maven-external-elasticsearch-archetype rre-maven-external-solr-archetype + rre-maven-generic-search-archetype \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/pom.xml b/rre-maven-archetype/rre-maven-generic-search-archetype/pom.xml new file mode 100644 index 00000000..e1f12251 --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/pom.xml @@ -0,0 +1,29 @@ + + + + rre-maven-archetype + io.sease + 1.0 + + 4.0.0 + + rre-maven-generic-search-archetype + RRE - Maven Generic Search Archetype + + + + + org.apache.rat + apache-rat-plugin + + + **/*.md + **/*.json + + + + + + \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml new file mode 100644 index 00000000..fb9a794c --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml @@ -0,0 +1,47 @@ + + + + + + src/etc/ratings + + ratings_example.json + README.md + + + + src/etc/templates + + only_q.json + filter_by_number_of_strings.json + README.md + + + + src/etc/configuration_sets + + */** + + + + \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/README.md b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/README.md new file mode 100644 index 00000000..b87ac1fb --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/README.md @@ -0,0 +1,31 @@ +Generic Maven RRE Search plugin +=============================== + +This archetype provides the basic setup necessary to connect RRE to a generic +search API. Unlike the other archetypes, this requires more configuration in +its pom.xml. In particular, you **must** set the following configuration +properties: + +- a dependency in the rre-maven-generic-search-plugin that refers to the +implementation of SearchPlatform you intend to use. +- `searchPlatform` in the configuration options must contain the name of +the search platform implementation class to be used. +- `searchPlatformConfiguration` may be used to pass an optional set of +configuration dependencies into the SearchPlatform. + +The supplied pom.xml file contains placeholders for this configuration. + +In addition, you are likely to need to change the configuration settings +files to contain the relevant information to communicate with your +search API. + + +## Implementing SearchPlatform + +To supply your own search platform, you need to implement the +[SearchPlatform](https://github.com/SeaseLtd/rated-ranking-evaluator/blob/master/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatform.java) +interface, which provides the connection between RRE and your search engine. +Your implementation should have a zero-argument constructor - the +configuration properties (set in the `searchPlatformConfiguration` block +in your pom.xml) will be passed in the `beforeStart()` phase. + \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/pom.xml b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/pom.xml new file mode 100644 index 00000000..bf45fc5c --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/pom.xml @@ -0,0 +1,117 @@ + + + + 4.0.0 + ${groupId} + ${artifactId} + ${version} + pom + + 1.8 + 1.8 + + + + + sease + https://raw.github.com/SeaseLtd/rated-ranking-evaluator/mvn-repo + + + + + + io.sease + rre-maven-generic-search-plugin + 1.0 + + + + com.mysearch + my-searchplatform-implementation + 1.1 + + + + + src/etc/configuration_sets + src/etc/ratings + src/etc/templates + *,score + + io.sease.rre.core.domain.metrics.impl.Precision + io.sease.rre.core.domain.metrics.impl.Recall + io.sease.rre.core.domain.metrics.impl.ReciprocalRank + io.sease.rre.core.domain.metrics.impl.AveragePrecision + io.sease.rre.core.domain.metrics.impl.NDCGAtTen + io.sease.rre.core.domain.metrics.impl.PrecisionAtOne + io.sease.rre.core.domain.metrics.impl.PrecisionAtTwo + io.sease.rre.core.domain.metrics.impl.PrecisionAtThree + io.sease.rre.core.domain.metrics.impl.PrecisionAtTen + + + true + false + 4 + + + + com.mysearch.MySearchPlatform + + + someConfigValue + + + + + search-quality-evaluation + package + + evaluate + + + + + + io.sease + rre-maven-report-plugin + 1.0 + + + spreadsheet + + + + + + + search-quality-evaluation-reporting + package + + report + + + + + + + diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/README.md b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/README.md new file mode 100644 index 00000000..abefa226 --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/README.md @@ -0,0 +1,12 @@ +This folder contains one subfolder for each configuration version. +Each version folder should contain a settings.json file with details of +how to connect to the appropriate search API. + +This is an example: + +* configuration_sets + * v1.0 + * settings.json + * v1.1 + * settings.json + \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.0/settings.json b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.0/settings.json new file mode 100644 index 00000000..b6c615b6 --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.0/settings.json @@ -0,0 +1,3 @@ +{ + "baseUrl": "http://generic.search.com/search" +} \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.1/settings.json b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.1/settings.json new file mode 100644 index 00000000..b6c615b6 --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.1/settings.json @@ -0,0 +1,3 @@ +{ + "baseUrl": "http://generic.search.com/search" +} \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/README.md b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/README.md new file mode 100644 index 00000000..6eb8be13 --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/README.md @@ -0,0 +1,2 @@ +Under the ratings folder you should have at least 1 ratings file. +A ratings file is connected with a dataset and contains a set of queries that compose the evaluation execution. \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/ratings_example.json b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/ratings_example.json new file mode 100644 index 00000000..7383709e --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/ratings_example.json @@ -0,0 +1,67 @@ +{ + "index": "core1", + "id_field": "uniqueId", + "topics": [ + { + "description": "Fender basses", + "query_groups": [ + { + "name": "Brand search", + "description": "The group tests several searches on the Fender brand", + "queries": [ + { + "template": "only_q.json", + "placeholders": { + "$query": "fender" + } + }, + { + "template": "only_q.json", + "placeholders": { + "$query": "fender Bass" + } + }, + { + "template": "filter_by_number_of_strings.json", + "placeholders": { + "$query": "Fender", + "$strings": 4 + } + } + ], + "relevant_documents": { + "1": { + "gain": 3 + }, + "2": { + "gain": 3 + } + } + }, + { + "name": "Jazz bass search", + "description": "Several searches on a given model (Jazz bass)", + "queries": [ + { + "template": "only_q.json", + "placeholders": { + "$query": "jazz" + } + }, + { + "template": "only_q.json", + "placeholders": { + "$query": "Jazz bass" + } + } + ], + "relevant_documents": { + "1": { + "gain": 3 + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/README.md b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/README.md new file mode 100644 index 00000000..b900f825 --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/README.md @@ -0,0 +1,11 @@ +This folder will contain the query templates associated with the evaluation suite. +A template is a JSON file containing a JSON object with name->value(s) pairs corresponding to query parameters. +Although it is completely ok to have statically-defined values here, usually you will be using placeholders. + +```javascript + { + "q": "$query", + "fq": "language:$lang" + } +``` +The placeholders values will be defined within the ratings file, specifically in the queries definitions. \ No newline at end of file diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/filter_by_number_of_strings.json b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/filter_by_number_of_strings.json new file mode 100644 index 00000000..50b8cd98 --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/filter_by_number_of_strings.json @@ -0,0 +1,4 @@ +{ + "q": "$query", + "fq": "number_of_strings:$strings" +} diff --git a/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/only_q.json b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/only_q.json new file mode 100644 index 00000000..b15f4d4e --- /dev/null +++ b/rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/only_q.json @@ -0,0 +1,3 @@ +{ + "q": "$query" +}