Merge pull request #112 from mattflax/generic_evaluation_plugin

Add Maven plugin to connect to any search API

rre-maven-archetype/pom.xml

@@ -34,5 +34,6 @@
         <module>rre-maven-elasticsearch-archetype</module>
         <module>rre-maven-external-elasticsearch-archetype</module>
         <module>rre-maven-external-solr-archetype</module>
+        <module>rre-maven-generic-search-archetype</module>
     </modules>
 </project>

rre-maven-archetype/rre-maven-generic-search-archetype/pom.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>rre-maven-archetype</artifactId>
+        <groupId>io.sease</groupId>
+        <version>1.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>rre-maven-generic-search-archetype</artifactId>
+    <name>RRE - Maven Generic Search Archetype</name>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.rat</groupId>
+                <artifactId>apache-rat-plugin</artifactId>
+                <configuration>
+                    <excludes>
+                        <exclude>**/*.md</exclude>
+                        <exclude>**/*.json</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<archetype-descriptor xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"
+                      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                      xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0 http://maven.apache.org/xsd/archetype-descriptor-1.0.0.xsd"
+                      name="RRE Solr Project Layout" partial="true">
+    <fileSets>
+        <fileSet encoding="UTF-8">
+            <directory>src/etc/ratings</directory>
+            <includes>
+                <include>ratings_example.json</include>
+                <include>README.md</include>
+            </includes>
+        </fileSet>
+        <fileSet encoding="UTF-8">
+            <directory>src/etc/templates</directory>
+            <includes>
+                <include>only_q.json</include>
+                <include>filter_by_number_of_strings.json</include>
+                <include>README.md</include>
+            </includes>
+        </fileSet>
+        <fileSet encoding="UTF-8">
+            <directory>src/etc/configuration_sets</directory>
+            <includes>
+                <include>*/**</include>
+            </includes>
+        </fileSet>
+    </fileSets>
+</archetype-descriptor>

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/README.md

@@ -0,0 +1,31 @@
+Generic Maven RRE Search plugin
+===============================
+
+This archetype provides the basic setup necessary to connect RRE to a generic
+search API. Unlike the other archetypes, this requires more configuration in
+its pom.xml. In particular, you **must** set the following configuration
+properties:
+
+- a dependency in the rre-maven-generic-search-plugin that refers to the
+implementation of SearchPlatform you intend to use.
+- `searchPlatform` in the configuration options must contain the name of
+the search platform implementation class to be used.
+- `searchPlatformConfiguration` may be used to pass an optional set of
+configuration dependencies into the SearchPlatform.
+
+The supplied pom.xml file contains placeholders for this configuration.
+
+In addition, you are likely to need to change the configuration settings
+files to contain the relevant information to communicate with your
+search API.
+
+
+## Implementing SearchPlatform
+
+To supply your own search platform, you need to implement the 
+[SearchPlatform](https://github.com/SeaseLtd/rated-ranking-evaluator/blob/master/rre-search-platform/rre-search-platform-api/src/main/java/io/sease/rre/search/api/SearchPlatform.java)
+interface, which provides the connection between RRE and your search engine.
+Your implementation should have a zero-argument constructor - the 
+configuration properties (set in the `searchPlatformConfiguration` block
+in your pom.xml) will be passed in the `beforeStart()` phase.
+

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/pom.xml

@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>${groupId}</groupId>
+    <artifactId>${artifactId}</artifactId>
+    <version>${version}</version>
+    <packaging>pom</packaging>
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+    </properties>
+
+    <pluginRepositories>
+       <pluginRepository>
+          <id>sease</id>
+          <url>https://raw.github.com/SeaseLtd/rated-ranking-evaluator/mvn-repo</url>
+       </pluginRepository>
+    </pluginRepositories>         
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>io.sease</groupId>
+                <artifactId>rre-maven-generic-search-plugin</artifactId>
+                <version>1.0</version>
+                <dependencies>
+                    <!-- Add a dependency here on your implementation of SearchPlatform -->
+                    <dependency>
+                        <groupId>com.mysearch</groupId>
+                        <artifactId>my-searchplatform-implementation</artifactId>
+                        <version>1.1</version>
+                    </dependency>
+                </dependencies>
+                <!-- the configuration below is provided just for example, as it perfectly matches default values -->
+                <configuration>
+                    <configurations-folder>src/etc/configuration_sets</configurations-folder>
+                    <ratings-folder>src/etc/ratings</ratings-folder>
+                    <templates-folder>src/etc/templates</templates-folder>
+                    <fields>*,score</fields>
+                    <metrics>
+                        <param>io.sease.rre.core.domain.metrics.impl.Precision</param>
+                        <param>io.sease.rre.core.domain.metrics.impl.Recall</param>
+                        <param>io.sease.rre.core.domain.metrics.impl.ReciprocalRank</param>
+                        <param>io.sease.rre.core.domain.metrics.impl.AveragePrecision</param>
+                        <param>io.sease.rre.core.domain.metrics.impl.NDCGAtTen</param>
+                        <param>io.sease.rre.core.domain.metrics.impl.PrecisionAtOne</param>
+                        <param>io.sease.rre.core.domain.metrics.impl.PrecisionAtTwo</param>
+                        <param>io.sease.rre.core.domain.metrics.impl.PrecisionAtThree</param>
+                        <param>io.sease.rre.core.domain.metrics.impl.PrecisionAtTen</param>
+                    </metrics>
+                    <evaluation>
+                        <runAsync>true</runAsync>
+                        <runQueriesAsync>false</runQueriesAsync>
+                        <threadpoolSize>4</threadpoolSize>
+                    </evaluation>
+                    <!-- Configuration for generic search platform -->
+                    <!-- SearchPlatform implementation class - REQUIRED -->
+                    <searchPlatform>com.mysearch.MySearchPlatform</searchPlatform>
+                    <!-- Optional SearchPlatform configuration details -->
+                    <searchPlatformConfiguration>
+                        <configProperty>someConfigValue</configProperty>
+                    </searchPlatformConfiguration>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>search-quality-evaluation</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>evaluate</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>io.sease</groupId>
+                <artifactId>rre-maven-report-plugin</artifactId>
+                <version>1.0</version>
+                <configuration>
+                    <formats>
+                        <param>spreadsheet</param>
+                        <!-- IMPORTANT: uncomment the following line if you're running the RRE server -->
+                        <!--
+                            <param>rre-server</param>
+                        -->
+                    </formats>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>search-quality-evaluation-reporting</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/README.md

@@ -0,0 +1,12 @@
+This folder contains one subfolder for each configuration version. 
+Each version folder should contain a settings.json file with details of
+how to connect to the appropriate search API.
+
+This is an example:
+
+* configuration_sets  
+  * v1.0
+    * settings.json
+  * v1.1
+    * settings.json
+

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.0/settings.json

@@ -0,0 +1,3 @@
+{
+  "baseUrl": "http://generic.search.com/search"
+}

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/configuration_sets/v1.1/settings.json

@@ -0,0 +1,3 @@
+{
+  "baseUrl": "http://generic.search.com/search"
+}

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/README.md

@@ -0,0 +1,2 @@
+Under the ratings folder you should have at least 1 ratings file. 
+A ratings file is connected with a dataset and contains a set of queries that compose the evaluation execution. 

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/ratings/ratings_example.json

@@ -0,0 +1,67 @@
+{
+  "index": "core1",
+  "id_field": "uniqueId",
+  "topics": [
+    {
+      "description": "Fender basses",
+      "query_groups": [
+        {
+          "name": "Brand search",
+          "description": "The group tests several searches on the Fender brand",
+          "queries": [
+            {
+              "template": "only_q.json",
+              "placeholders": {
+                "$query": "fender"
+              }
+            },
+            {
+              "template": "only_q.json",
+              "placeholders": {
+                "$query": "fender Bass"
+              }
+            },
+            {
+              "template": "filter_by_number_of_strings.json",
+              "placeholders": {
+                "$query": "Fender",
+                "$strings": 4
+              }
+            }
+          ],
+          "relevant_documents": {
+            "1": {
+              "gain": 3
+            },
+            "2": {
+              "gain": 3
+            }
+          }
+        },
+        {
+          "name": "Jazz bass search",
+          "description": "Several searches on a given model (Jazz bass)",
+          "queries": [
+            {
+              "template": "only_q.json",
+              "placeholders": {
+                "$query": "jazz"
+              }
+            },
+            {
+              "template": "only_q.json",
+              "placeholders": {
+                "$query": "Jazz bass"
+              }
+            }
+          ],
+          "relevant_documents": {
+            "1": {
+              "gain": 3
+            }
+          }
+        }
+      ]
+    }
+  ]
+}

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/README.md

@@ -0,0 +1,11 @@
+This folder will contain the query templates associated with the evaluation suite. 
+A template is a JSON file containing a JSON object with name->value(s) pairs corresponding to query parameters. 
+Although it is completely ok to have statically-defined values here, usually you will be using placeholders.
+
+```javascript
+  {
+    "q": "$query",
+    "fq": "language:$lang"
+  }
+```
+The placeholders values will be defined within the ratings file, specifically in the queries definitions. 

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/filter_by_number_of_strings.json

@@ -0,0 +1,4 @@
+{
+  "q": "$query",
+  "fq": "number_of_strings:$strings"
+}

rre-maven-archetype/rre-maven-generic-search-archetype/src/main/resources/archetype-resources/src/etc/templates/only_q.json

@@ -0,0 +1,3 @@
+{
+  "q": "$query"
+}

rre-maven-plugin/pom.xml

@@ -34,6 +34,7 @@
         <module>rre-maven-elasticsearch-plugin</module>
         <module>rre-maven-external-elasticsearch-plugin</module>
         <module>rre-maven-external-solr-plugin</module>
+        <module>rre-maven-generic-search-plugin</module>
     </modules>
     <packaging>pom</packaging>
     <properties>