[WIP] Update ES v7.8.1 (#323)

* Update dependencies for ES 7.8.1

* Refactor Rest endpoint classes

* Fix the 7.8.1 update

Co-authored-by: = <danworley@gmail.com>

build.gradle

@@ -16,6 +16,7 @@ version = "${ltrVersion}-es${elasticsearchVersion}"
 apply plugin: 'java'
 apply plugin: 'idea'
 apply plugin: 'elasticsearch.esplugin'
+apply plugin: 'elasticsearch.rest-resources'
 
 // license of this project
 licenseFile = rootProject.file('LICENSE.txt')
@@ -50,10 +51,10 @@ repositories {
 
 dependencies {
   compile "org.apache.lucene:lucene-expressions:${luceneVersion}"
-  compile 'org.antlr:antlr4-runtime:4.5.1-1'
-  compile 'org.ow2.asm:asm:5.0.4'
-  compile 'org.ow2.asm:asm-commons:5.0.4'
-  compile 'org.ow2.asm:asm-tree:5.0.4'
+  compile "org.antlr:antlr4-runtime:${antlrVersion}"
+  compile "org.ow2.asm:asm:${ow2Version}"
+  compile "org.ow2.asm:asm-commons:${ow2Version}"
+  compile "org.ow2.asm:asm-tree:${ow2Version}"
   compile 'com.o19s:RankyMcRankFace:0.1.1'
   compile "com.github.spullara.mustache.java:compiler:0.9.3"
 }

gradle.properties

@@ -1,3 +1,7 @@
-ltrVersion = 1.3.0
-elasticsearchVersion = 7.8.0
+ltrVersion = 1.3.1
+
+# Dependencies versions
+elasticsearchVersion = 7.8.1
 luceneVersion = 8.5.1
+antlrVersion=4.5.1-1
+ow2Version=7.2

src/main/java/com/o19s/es/ltr/LtrQueryParserPlugin.java

@@ -47,10 +47,12 @@
 import com.o19s.es.ltr.ranker.parser.XGBoostJsonParser;
 import com.o19s.es.ltr.ranker.ranklib.RankLibScriptEngine;
 import com.o19s.es.ltr.ranker.ranklib.RanklibModelParser;
-import com.o19s.es.ltr.rest.RestAddFeatureToSet;
 import com.o19s.es.ltr.rest.RestCreateModelFromSet;
+import com.o19s.es.ltr.rest.RestFeatureManager;
+import com.o19s.es.ltr.rest.RestSearchStoreElements;
+import com.o19s.es.ltr.rest.RestStoreManager;
+import com.o19s.es.ltr.rest.RestAddFeatureToSet;
 import com.o19s.es.ltr.rest.RestFeatureStoreCaches;
-import com.o19s.es.ltr.rest.RestSimpleFeatureStore;
 import com.o19s.es.ltr.utils.FeatureStoreLoader;
 import com.o19s.es.ltr.utils.Suppliers;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -156,7 +158,13 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
                                              SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
                                              Supplier<DiscoveryNodes> nodesInCluster) {
         List<RestHandler> list = new ArrayList<>();
-        RestSimpleFeatureStore.register(list, restController);
+
+        for (String type : ValidatingLtrQueryBuilder.SUPPORTED_TYPES) {
+            list.add(new RestFeatureManager(type));
+            list.add(new RestSearchStoreElements(type));
+        }
+        list.add(new RestStoreManager());
+
         list.add(new RestFeatureStoreCaches());
         list.add(new RestCreateModelFromSet());
         list.add(new RestAddFeatureToSet());

src/main/java/com/o19s/es/ltr/rest/AutoDetectParser.java

@@ -0,0 +1,71 @@
+package com.o19s.es.ltr.rest;
+
+import com.o19s.es.ltr.feature.FeatureValidation;
+import com.o19s.es.ltr.feature.store.StorableElement;
+import com.o19s.es.ltr.feature.store.StoredFeature;
+import com.o19s.es.ltr.feature.store.StoredFeatureSet;
+import com.o19s.es.ltr.feature.store.StoredLtrModel;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+import static com.o19s.es.ltr.query.ValidatingLtrQueryBuilder.SUPPORTED_TYPES;
+import static java.util.stream.Collectors.joining;
+
+class AutoDetectParser {
+    private String expectedName;
+    private StorableElement element;
+    private FeatureValidation validation;
+
+    private static final ObjectParser<AutoDetectParser, String> PARSER = new ObjectParser<>("storable_elements");
+
+    static {
+        PARSER.declareObject(AutoDetectParser::setElement,
+                StoredFeature::parse,
+                new ParseField(StoredFeature.TYPE));
+        PARSER.declareObject(AutoDetectParser::setElement,
+                StoredFeatureSet::parse,
+                new ParseField(StoredFeatureSet.TYPE));
+        PARSER.declareObject(AutoDetectParser::setElement,
+                StoredLtrModel::parse,
+                new ParseField(StoredLtrModel.TYPE));
+        PARSER.declareObject((b, v) -> b.validation = v,
+                (p, c) -> FeatureValidation.PARSER.apply(p, null),
+                new ParseField("validation"));
+    }
+
+    AutoDetectParser(String name) {
+        this.expectedName = name;
+    }
+
+    public void parse(XContentParser parser) throws IOException {
+        PARSER.parse(parser, this, expectedName);
+        if (element == null) {
+            throw new ParsingException(parser.getTokenLocation(), "Element of type [" + SUPPORTED_TYPES.stream().collect(joining(",")) +
+                    "] is mandatory.");
+        }
+    }
+
+    public StorableElement getElement() {
+        return element;
+    }
+
+    public void setElement(StorableElement element) {
+        if (this.element != null) {
+            throw new IllegalArgumentException("[" + element.type() + "] already set, only one element can be set at a time (" +
+                    SUPPORTED_TYPES.stream().collect(joining(",")) + ").");
+        }
+        this.element = element;
+    }
+
+    public void setValidation(FeatureValidation validation) {
+        this.validation = validation;
+    }
+
+    public FeatureValidation getValidation() {
+        return validation;
+    }
+}

src/main/java/com/o19s/es/ltr/rest/FeatureStoreBaseRestHandler.java

@@ -21,8 +21,6 @@
 import org.elasticsearch.rest.RestRequest;
 
 public abstract class FeatureStoreBaseRestHandler extends BaseRestHandler {
-    protected FeatureStoreBaseRestHandler() {
-    }
 
     protected String indexName(RestRequest request) {
         if (request.hasParam("store")) {

src/main/java/com/o19s/es/ltr/rest/RestAddFeatureToSet.java

@@ -52,6 +52,7 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         String store = indexName(request);
+
         String setName = request.param("name");
         String routing = request.param("routing");
         String featureQuery = null;

src/main/java/com/o19s/es/ltr/rest/RestFeatureManager.java

@@ -0,0 +1,160 @@
+package com.o19s.es.ltr.rest;
+
+import com.o19s.es.ltr.action.ClearCachesAction;
+import com.o19s.es.ltr.action.FeatureStoreAction;
+import com.o19s.es.ltr.feature.store.StorableElement;
+import com.o19s.es.ltr.feature.store.StoredFeature;
+import com.o19s.es.ltr.feature.store.StoredFeatureSet;
+import com.o19s.es.ltr.feature.store.StoredLtrModel;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.RestStatusToXContentListener;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.List;
+
+import static com.o19s.es.ltr.feature.store.StorableElement.generateId;
+import static com.o19s.es.ltr.feature.store.index.IndexFeatureStore.ES_TYPE;
+import static com.o19s.es.ltr.query.ValidatingLtrQueryBuilder.SUPPORTED_TYPES;
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
+import static org.elasticsearch.rest.RestStatus.OK;
+
+public class RestFeatureManager extends FeatureStoreBaseRestHandler {
+    private final String type;
+
+    public RestFeatureManager(String type) {
+        this.type = type;
+    }
+
+    @Override
+    public String getName() {
+        return "Add, update or delete a " + type;
+    }
+
+    @Override
+    public List<Route> routes() {
+        return unmodifiableList(asList(
+                new Route(RestRequest.Method.PUT, "/_ltr/{store}/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.PUT, "/_ltr/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.POST, "/_ltr/{store}/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.POST, "/_ltr/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.DELETE, "/_ltr/{store}/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.DELETE, "/_ltr/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.GET, "/_ltr/{store}/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.GET, "/_ltr/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.HEAD, "/_ltr/{store}/_" + this.type + "/{name}"),
+                new Route(RestRequest.Method.HEAD, "/_ltr/_" + this.type + "/{name}")
+        ));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        String indexName = indexName(request);
+        if (request.method() == RestRequest.Method.DELETE) {
+            return delete(client, type, indexName, request);
+        } else if (request.method() == RestRequest.Method.HEAD || request.method() == RestRequest.Method.GET) {
+            return get(client, type, indexName, request);
+        } else {
+            return addOrUpdate(client, type, indexName, request);
+        }
+    }
+
+    RestChannelConsumer delete(NodeClient client, String type, String indexName, RestRequest request) {
+        assert SUPPORTED_TYPES.contains(type);
+        String name = request.param("name");
+        String id = generateId(type, name);
+        String routing = request.param("routing");
+        return (channel) ->  {
+            RestStatusToXContentListener<DeleteResponse> restR = new RestStatusToXContentListener<>(channel, (r) -> r.getLocation(routing));
+            client.prepareDelete(indexName, ES_TYPE, id)
+                    .setRouting(routing)
+                    .execute(ActionListener.wrap((deleteResponse) -> {
+                                // wrap the response so we can send another request to clear the cache
+                                // usually we send only one transport request from the rest layer
+                                // it's still unclear which direction we should take (thick or thin REST layer?)
+                                ClearCachesAction.ClearCachesNodesRequest clearCache = new ClearCachesAction.ClearCachesNodesRequest();
+                                switch (type) {
+                                    case StoredFeature.TYPE:
+                                        clearCache.clearFeature(indexName, name);
+                                        break;
+                                    case StoredFeatureSet.TYPE:
+                                        clearCache.clearFeatureSet(indexName, name);
+                                        break;
+                                    case StoredLtrModel.TYPE:
+                                        clearCache.clearModel(indexName, name);
+                                        break;
+                                }
+                                client.execute(ClearCachesAction.INSTANCE, clearCache, ActionListener.wrap(
+                                        (r) -> restR.onResponse(deleteResponse),
+                                        // Is it good to fail the whole request if cache invalidation failed?
+                                        restR::onFailure
+                                ));
+                            },
+                            restR::onFailure
+                    ));
+        };
+    }
+
+    RestChannelConsumer get(NodeClient client, String type, String indexName, RestRequest request) {
+        assert SUPPORTED_TYPES.contains(type);
+        String name = request.param("name");
+        String routing = request.param("routing");
+        String id = generateId(type, name);
+        return (channel) -> client.prepareGet(indexName, ES_TYPE, id)
+                .setRouting(routing)
+                .execute(new RestToXContentListener<GetResponse>(channel) {
+                    @Override
+                    protected RestStatus getStatus(final GetResponse response) {
+                        return response.isExists() ? OK : NOT_FOUND;
+                    }
+                });
+    }
+
+    RestChannelConsumer addOrUpdate(NodeClient client, String type, String indexName, RestRequest request) throws IOException {
+        assert SUPPORTED_TYPES.contains(type);
+        String routing = request.param("routing");
+        if (!request.hasContentOrSourceParam()) {
+            throw new IllegalArgumentException("Missing content or source param.");
+        }
+        String name = request.param("name");
+        AutoDetectParser parserState = new AutoDetectParser(name);
+        request.applyContentParser(parserState::parse);
+        StorableElement elt = parserState.getElement();
+        if (!type.equals(elt.type())) {
+            throw new IllegalArgumentException("Excepted a [" + type + "] but encountered [" + elt.type() + "]");
+        }
+
+        // Validation happens here when parsing the stored element.
+        if (!elt.name().equals(name)) {
+            throw new IllegalArgumentException("Name mismatch, send request with [" + elt.name() + "] but [" + name + "] used in the URL");
+        }
+        if (request.method() == RestRequest.Method.POST && !elt.updatable()) {
+            try {
+                throw new IllegalArgumentException("Element of type [" + elt.type() + "] are not updatable, " +
+                        "please create a new one instead.");
+            } catch (IllegalArgumentException iae) {
+                return (channel) -> channel.sendResponse(new BytesRestResponse(channel, RestStatus.METHOD_NOT_ALLOWED, iae));
+            }
+        }
+        FeatureStoreAction.FeatureStoreRequestBuilder builder = new FeatureStoreAction.FeatureStoreRequestBuilder(
+                client, FeatureStoreAction.INSTANCE);
+        if (request.method() == RestRequest.Method.PUT) {
+            builder.request().setAction(FeatureStoreAction.FeatureStoreRequest.Action.CREATE);
+        } else {
+            builder.request().setAction(FeatureStoreAction.FeatureStoreRequest.Action.UPDATE);
+        }
+        builder.request().setStorableElement(elt);
+        builder.request().setRouting(routing);
+        builder.request().setStore(indexName);
+        builder.request().setValidation(parserState.getValidation());
+        return (channel) -> builder.execute(new RestStatusToXContentListener<>(channel, (r) -> r.getResponse().getLocation(routing)));
+    }
+}

src/main/java/com/o19s/es/ltr/rest/RestFeatureStoreCaches.java

@@ -53,9 +53,9 @@ public String getName() {
     @Override
     public List<Route> routes() {
         return unmodifiableList(asList(
-                new Route(RestRequest.Method.POST, "/_ltr/_clearcache"),
-                new Route(RestRequest.Method.POST, "/_ltr/{store}/_clearcache"),
-                new Route(RestRequest.Method.GET, "/_ltr/_cachestats")
+            new Route(RestRequest.Method.POST, "/_ltr/_clearcache"),
+            new Route(RestRequest.Method.POST, "/_ltr/{store}/_clearcache"),
+            new Route(RestRequest.Method.GET, "/_ltr/_cachestats")
         ));
     }
 

src/main/java/com/o19s/es/ltr/rest/RestSearchStoreElements.java

@@ -0,0 +1,58 @@
+package com.o19s.es.ltr.rest;
+
+import com.o19s.es.ltr.feature.store.index.IndexFeatureStore;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestStatusToXContentListener;
+
+import java.util.List;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+
+public class RestSearchStoreElements extends FeatureStoreBaseRestHandler {
+    private final String type;
+
+    public RestSearchStoreElements(String type) {
+        this.type = type;
+    }
+
+    @Override
+    public String getName() {
+        return "Search for " + type + " elements in the LTR feature store";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return unmodifiableList(asList(
+                new Route(RestRequest.Method.GET, "/_ltr/{store}/_" + type),
+                new Route(RestRequest.Method.GET, "/_ltr/_" + type)
+        ));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) {
+        return search(client, type, indexName(request), request);
+    }
+
+    RestChannelConsumer search(NodeClient client, String type, String indexName, RestRequest request) {
+        String prefix = request.param("prefix");
+        int from = request.paramAsInt("from", 0);
+        int size = request.paramAsInt("size", 20);
+        BoolQueryBuilder qb = boolQuery().filter(termQuery("type", type));
+        if (prefix != null && !prefix.isEmpty()) {
+            qb.must(matchQuery("name.prefix", prefix));
+        }
+        return (channel) -> client.prepareSearch(indexName)
+                .setTypes(IndexFeatureStore.ES_TYPE)
+                .setQuery(qb)
+                .setSize(size)
+                .setFrom(from)
+                .execute(new RestStatusToXContentListener<>(channel));
+    }
+
+}