[ML][Data Frame] Add optional defer_validation param to PUT (#44455)

* [ML][Data Frame] Add optional defer_validation param to PUT * addressing PR comments * reverting bad replace * addressing pr comments * Update put-transform.asciidoc * Update put-transform.asciidoc * Update put-transform.asciidoc
elastic · Jul 22, 2019 · a9cc0e1 · a9cc0e1
1 parent 73f8f1f
commit a9cc0e1
Show file tree

Hide file tree

Showing 20 changed files with 316 additions and 117 deletions.
diff --git a/...nt/rest-high-level/src/main/java/org/elasticsearch/client/DataFrameRequestConverters.java b/...nt/rest-high-level/src/main/java/org/elasticsearch/client/DataFrameRequestConverters.java
@@ -39,6 +39,7 @@
 import static org.elasticsearch.client.RequestConverters.createEntity;
 import static org.elasticsearch.client.dataframe.DeleteDataFrameTransformRequest.FORCE;
 import static org.elasticsearch.client.dataframe.GetDataFrameTransformRequest.ALLOW_NO_MATCH;
+import static org.elasticsearch.client.dataframe.PutDataFrameTransformRequest.DEFER_VALIDATION;
 
 final class DataFrameRequestConverters {
 
@@ -51,6 +52,9 @@ static Request putDataFrameTransform(PutDataFrameTransformRequest putRequest) th
                 .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         request.setEntity(createEntity(putRequest, REQUEST_BODY_CONTENT_TYPE));
+        if (putRequest.getDeferValidation() != null) {
+            request.addParameter(DEFER_VALIDATION, Boolean.toString(putRequest.getDeferValidation()));
+        }
         return request;
     }
 

diff --git a/...-level/src/main/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequest.java b/...-level/src/main/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequest.java
@@ -31,7 +31,9 @@
 
 public class PutDataFrameTransformRequest implements ToXContentObject, Validatable {
 
+    public static final String DEFER_VALIDATION = "defer_validation";
     private final DataFrameTransformConfig config;
+    private Boolean deferValidation;
 
     public PutDataFrameTransformRequest(DataFrameTransformConfig config) {
         this.config = config;
@@ -41,6 +43,19 @@ public DataFrameTransformConfig getConfig() {
         return config;
     }
 
+    public Boolean getDeferValidation() {
+        return deferValidation;
+    }
+
+    /**
+     * Indicates if deferrable validations should be skipped until the transform starts
+     *
+     * @param deferValidation {@code true} will cause validations to be deferred
+     */
+    public void setDeferValidation(boolean deferValidation) {
+        this.deferValidation = deferValidation;
+    }
+
     @Override
     public Optional<ValidationException> validate() {
         ValidationException validationException = new ValidationException();

diff --git a/...st-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java b/...st-high-level/src/test/java/org/elasticsearch/client/DataFrameRequestConvertersTests.java
@@ -68,14 +68,17 @@ public void testPutDataFrameTransform() throws IOException {
         PutDataFrameTransformRequest putRequest = new PutDataFrameTransformRequest(
                 DataFrameTransformConfigTests.randomDataFrameTransformConfig());
         Request request = DataFrameRequestConverters.putDataFrameTransform(putRequest);
-
+        assertThat(request.getParameters(), not(hasKey("defer_validation")));
         assertEquals(HttpPut.METHOD_NAME, request.getMethod());
         assertThat(request.getEndpoint(), equalTo("/_data_frame/transforms/" + putRequest.getConfig().getId()));
 
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) {
             DataFrameTransformConfig parsedConfig = DataFrameTransformConfig.PARSER.apply(parser, null);
             assertThat(parsedConfig, equalTo(putRequest.getConfig()));
         }
+        putRequest.setDeferValidation(true);
+        request = DataFrameRequestConverters.putDataFrameTransform(putRequest);
+        assertThat(request.getParameters(), hasEntry("defer_validation", Boolean.toString(putRequest.getDeferValidation())));
     }
 
     public void testDeleteDataFrameTransform() {

diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java
@@ -180,6 +180,22 @@ public void testCreateDelete() throws IOException {
         assertThat(deleteError.getMessage(), containsString("Transform with id [test-crud] could not be found"));
     }
 
+    public void testCreateDeleteWithDefer() throws IOException {
+        String sourceIndex = "missing-source-index";
+
+        String id = "test-with-defer";
+        DataFrameTransformConfig transform = validDataFrameTransformConfig(id, sourceIndex, "pivot-dest");
+        DataFrameClient client = highLevelClient().dataFrame();
+        PutDataFrameTransformRequest request = new PutDataFrameTransformRequest(transform);
+        request.setDeferValidation(true);
+        AcknowledgedResponse ack = execute(request, client::putDataFrameTransform, client::putDataFrameTransformAsync);
+        assertTrue(ack.isAcknowledged());
+
+        ack = execute(new DeleteDataFrameTransformRequest(transform.getId()), client::deleteDataFrameTransform,
+            client::deleteDataFrameTransformAsync);
+        assertTrue(ack.isAcknowledged());
+    }
+
     public void testGetTransform() throws IOException {
         String sourceIndex = "transform-source";
         createIndex(sourceIndex);

diff --git a/...c/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java b/...c/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java
@@ -166,6 +166,7 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
             // tag::put-data-frame-transform-request
             PutDataFrameTransformRequest request =
                     new PutDataFrameTransformRequest(transformConfig); // <1>
+            request.setDeferValidation(false); // <2>
             // end::put-data-frame-transform-request
 
             // tag::put-data-frame-transform-execute

diff --git a/docs/java-rest/high-level/dataframe/put_data_frame.asciidoc b/docs/java-rest/high-level/dataframe/put_data_frame.asciidoc
@@ -20,6 +20,10 @@ A +{request}+ requires the following argument:
 include-tagged::{doc-tests-file}[{api}-request]
 --------------------------------------------------
 <1> The configuration of the {dataframe-transform} to create
+<2> Whether or not to wait to run deferrable validations until `_start` is called.
+This option should be used with care as the created {dataframe-transform} will run
+with the privileges of the user creating it. Meaning, if they do not have privileges,
+such an error will not be visible until `_start` is called.
 
 [id="{upid}-{api}-config"]
 ==== Data Frame Transform Configuration

diff --git a/docs/reference/data-frames/apis/put-transform.asciidoc b/docs/reference/data-frames/apis/put-transform.asciidoc
@@ -45,6 +45,20 @@ IMPORTANT:  You must use {kib} or this API to create a {dataframe-transform}.
   can contain lowercase alphanumeric characters (a-z and 0-9), hyphens, and
   underscores. It must start and end with alphanumeric characters.
 
+[[put-data-frame-transform-query-parms]]
+==== {api-query-parms-title}
+
+`defer_validation`::
+ (Optional, boolean) When `true`, this will cause deferrable validations to not run.
+ This behavior may be desired if the source index does not exist until
+ after the the {dataframe-transform} is created.
+ Deferred validations are always ran when the {dataframe-transform} is started, 
+ with the exception of privilege checks. If the user who created the transform does 
+ not have the required privileges on the source and destination indices then the 
+ transform will start but then fail when it attempts the unauthorized operation.
+
+ The default value is `false`.
+
 [[put-data-frame-transform-request-body]]
 ==== {api-request-body-title}
 

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java
@@ -33,6 +33,7 @@ public final class DataFrameField {
     public static final ParseField SYNC = new ParseField("sync");
     public static final ParseField TIME_BASED_SYNC = new ParseField("time");
     public static final ParseField DELAY = new ParseField("delay");
+    public static final ParseField DEFER_VALIDATION = new ParseField("defer_validation");
 
     public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match");
     /**

diff --git a/.../main/java/org/elasticsearch/xpack/core/dataframe/action/PutDataFrameTransformAction.java b/.../main/java/org/elasticsearch/xpack/core/dataframe/action/PutDataFrameTransformAction.java
@@ -6,15 +6,14 @@
 
 package org.elasticsearch.xpack.core.dataframe.action;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContentObject;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
@@ -41,21 +40,28 @@ private PutDataFrameTransformAction() {
         super(NAME, AcknowledgedResponse::new);
     }
 
-    public static class Request extends AcknowledgedRequest<Request> implements ToXContentObject {
+    public static class Request extends AcknowledgedRequest<Request> {
 
         private final DataFrameTransformConfig config;
+        private final boolean deferValidation;
 
-        public Request(DataFrameTransformConfig config) {
+        public Request(DataFrameTransformConfig config, boolean deferValidation) {
             this.config = config;
+            this.deferValidation = deferValidation;
         }
 
         public Request(StreamInput in) throws IOException {
             super(in);
             this.config = new DataFrameTransformConfig(in);
+            if (in.getVersion().onOrAfter(Version.CURRENT)) {
+                this.deferValidation = in.readBoolean();
+            } else {
+                this.deferValidation = false;
+            }
         }
 
-        public static Request fromXContent(final XContentParser parser, final String id) throws IOException {
-            return new Request(DataFrameTransformConfig.fromXContent(parser, id, false));
+        public static Request fromXContent(final XContentParser parser, final String id, final boolean deferValidation) {
+            return new Request(DataFrameTransformConfig.fromXContent(parser, id, false), deferValidation);
         }
 
         /**
@@ -111,24 +117,26 @@ public ActionRequestValidationException validate() {
             return validationException;
         }
 
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            return this.config.toXContent(builder, params);
-        }
-
         public DataFrameTransformConfig getConfig() {
             return config;
         }
 
+        public boolean isDeferValidation() {
+            return deferValidation;
+        }
+
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             this.config.writeTo(out);
+            if (out.getVersion().onOrAfter(Version.CURRENT)) {
+                out.writeBoolean(this.deferValidation);
+            }
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(config);
+            return Objects.hash(config, deferValidation);
         }
 
         @Override
@@ -140,7 +148,7 @@ public boolean equals(Object obj) {
                 return false;
             }
             Request other = (Request) obj;
-            return Objects.equals(config, other.config);
+            return Objects.equals(config, other.config) && this.deferValidation == other.deferValidation;
         }
     }
 

diff --git a/...rg/elasticsearch/xpack/core/dataframe/action/PutDataFrameTransformActionRequestTests.java b/...rg/elasticsearch/xpack/core/dataframe/action/PutDataFrameTransformActionRequestTests.java
@@ -6,41 +6,49 @@
 
 package org.elasticsearch.xpack.core.dataframe.action;
 
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractWireSerializingTestCase;
+import org.elasticsearch.xpack.core.dataframe.DataFrameField;
 import org.elasticsearch.xpack.core.dataframe.action.PutDataFrameTransformAction.Request;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfig;
 import org.elasticsearch.xpack.core.dataframe.transforms.DataFrameTransformConfigTests;
+import org.elasticsearch.xpack.core.dataframe.transforms.SyncConfig;
+import org.elasticsearch.xpack.core.dataframe.transforms.TimeSyncConfig;
 import org.junit.Before;
 
-import java.io.IOException;
+import java.util.List;
 
-public class PutDataFrameTransformActionRequestTests extends AbstractSerializingDataFrameTestCase<Request> {
+import static java.util.Collections.emptyList;
+
+public class PutDataFrameTransformActionRequestTests extends AbstractWireSerializingTestCase<Request> {
     private String transformId;
 
     @Before
     public void setupTransformId() {
         transformId = randomAlphaOfLengthBetween(1, 10);
     }
 
-    @Override
-    protected Request doParseInstance(XContentParser parser) throws IOException {
-        return Request.fromXContent(parser, transformId);
-    }
-
     @Override
     protected Writeable.Reader<Request> instanceReader() {
         return Request::new;
     }
 
     @Override
-    protected boolean supportsUnknownFields() {
-        return false;
+    protected Request createTestInstance() {
+        DataFrameTransformConfig config = DataFrameTransformConfigTests.randomDataFrameTransformConfigWithoutHeaders(transformId);
+        return new Request(config, randomBoolean());
     }
 
     @Override
-    protected Request createTestInstance() {
-        DataFrameTransformConfig config = DataFrameTransformConfigTests.randomDataFrameTransformConfigWithoutHeaders(transformId);
-        return new Request(config);
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
+
+        List<NamedWriteableRegistry.Entry> namedWriteables = searchModule.getNamedWriteables();
+        namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, DataFrameField.TIME_BASED_SYNC.getPreferredName(),
+            TimeSyncConfig::new));
+        return new NamedWriteableRegistry(namedWriteables);
     }
 }