Merge remote-tracking branch 'elastic/master' into remove-warn-date

* elastic/master: Remove Watcher Account "unsecure" settings (elastic#36736) Add cache cleaning task for ML snapshot (elastic#37505) Update jdk used by the docker builds (elastic#37621) Remove an unused constant in PutMappingRequest. Update get users to allow unknown fields (elastic#37593) Do not add index event listener if CCR disabled (elastic#37432) Add local session timeouts to leader node (elastic#37438)
jasontedor · Jan 20, 2019 · 696da03 · 696da03
2 parents 2472b0a + 5308746
commit 696da03
Show file tree

Hide file tree

Showing 38 changed files with 542 additions and 344 deletions.
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutMappingRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/PutMappingRequest.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.client.indices;
 
-import com.carrotsearch.hppc.ObjectHashSet;
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -43,11 +42,6 @@
  */
 public class PutMappingRequest extends TimedRequest implements IndicesRequest, ToXContentObject {
 
-    private static ObjectHashSet<String> RESERVED_FIELDS = ObjectHashSet.from(
-            "_uid", "_id", "_type", "_source",  "_all", "_analyzer", "_parent", "_routing", "_index",
-            "_size", "_timestamp", "_ttl", "_field_names"
-    );
-
     private final String[] indices;
     private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, true);
 

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetUsersResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/security/GetUsersResponse.java
@@ -95,7 +95,7 @@ public int hashCode() {
     public static final ParseField ENABLED = new ParseField("enabled");
 
     @SuppressWarnings("unchecked")
-    public static final ConstructingObjectParser<ParsedUser, String> USER_PARSER = new ConstructingObjectParser<>("user_info",
+    public static final ConstructingObjectParser<ParsedUser, String> USER_PARSER = new ConstructingObjectParser<>("user_info", true,
         (constructorObjects) -> {
             int i = 0;
             final String username = (String) constructorObjects[i++];

diff --git a/...est-high-level/src/test/java/org/elasticsearch/client/security/GetUsersResponseTests.java b/...est-high-level/src/test/java/org/elasticsearch/client/security/GetUsersResponseTests.java
@@ -19,58 +19,97 @@
 package org.elasticsearch.client.security;
 
 import org.elasticsearch.client.security.user.User;
-import org.elasticsearch.common.xcontent.DeprecationHandler;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.test.XContentTestUtils;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
-import static org.hamcrest.Matchers.equalTo;
+import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
 
 /** tests the Response for getting users from the security HLRC */
 public class GetUsersResponseTests extends ESTestCase {
+
     public void testFromXContent() throws IOException {
-        String json =
-            "{\n" +
-                "  \"jacknich\": {\n" +
-                "    \"username\": \"jacknich\",\n" +
-                "    \"roles\": [\n" +
-                "      \"admin\", \"other_role1\"\n" +
-                "    ],\n" +
-                "    \"full_name\": \"Jack Nicholson\",\n" +
-                "    \"email\": \"jacknich@example.com\",\n" +
-                "    \"metadata\": { \"intelligence\" : 7 },\n" +
-                "    \"enabled\": true\n" +
-                "  }\n" +
-                "}";
-        final GetUsersResponse response = GetUsersResponse.fromXContent((XContentType.JSON.xContent().createParser(
-            new NamedXContentRegistry(Collections.emptyList()), new DeprecationHandler() {
-                @Override
-                public void usedDeprecatedName(String usedName, String modernName) {
-                }
+        xContentTester(this::createParser,
+            GetUsersResponseTests::createTestInstance,
+            this::toXContent,
+            GetUsersResponse::fromXContent)
+            .supportsUnknownFields(false)
+            .assertToXContentEquivalence(false)
+            .test();
+    }
+
+    private XContentBuilder toXContentUser(User user, boolean enabled, XContentBuilder builder) throws IOException {
+        XContentBuilder tempBuilder = JsonXContent.contentBuilder();
+        tempBuilder.startObject();
+        tempBuilder.field("username", user.getUsername());
+        tempBuilder.array("roles", user.getRoles().toArray());
+        tempBuilder.field("full_name", user.getFullName());
+        tempBuilder.field("email", user.getEmail());
+        tempBuilder.field("metadata", user.getMetadata());
+        tempBuilder.field("enabled", enabled);
+        tempBuilder.endObject();
+
+        // This sub object should support unknown fields, but metadata cannot contain complex extra objects or it will fail
+        Predicate<String> excludeFilter = path -> path.equals("metadata");
+        BytesReference newBytes = XContentTestUtils.insertRandomFields(XContentType.JSON, BytesReference.bytes(tempBuilder),
+            excludeFilter, random());
+        builder.rawValue(newBytes.streamInput(), XContentType.JSON);
+        return builder;
+    }
+
+    private XContentBuilder toXContent(GetUsersResponse response, XContentBuilder builder) throws IOException {
+        builder.startObject();
+
+        List<User> disabledUsers = new ArrayList<>(response.getUsers());
+        disabledUsers.removeAll(response.getEnabledUsers());
+
+        for (User user : disabledUsers) {
+            builder.field(user.getUsername());
+            toXContentUser(user, false, builder);
+        }
+        for (User user : response.getEnabledUsers()) {
+            builder.field(user.getUsername());
+            toXContentUser(user, true, builder);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    private static GetUsersResponse createTestInstance() {
+        final Set<User> users = new HashSet<>();
+        final Set<User> enabledUsers = new HashSet<>();
+        Map<String, Object> metadata = new HashMap<>();
+        metadata.put(randomAlphaOfLengthBetween(1, 5), randomInt());
 
-                @Override
-                public void usedDeprecatedField(String usedName, String replacedWith) {
-                }
-            }, json)));
-        assertThat(response.getUsers().size(), equalTo(1));
-        final User user = response.getUsers().iterator().next();
-        assertThat(user.getUsername(), equalTo("jacknich"));
-        assertThat(user.getRoles().size(), equalTo(2));
-        assertThat(user.getFullName(), equalTo("Jack Nicholson"));
-        assertThat(user.getEmail(), equalTo("jacknich@example.com"));
-        final Map<String, Object> metadata = new HashMap<>();
-        metadata.put("intelligence", 7);
-        assertThat(metadata, equalTo(user.getMetadata()));
+        final User user1 = new User(randomAlphaOfLength(8),
+            Arrays.asList(new String[] {randomAlphaOfLength(5), randomAlphaOfLength(5)}),
+            metadata, randomAlphaOfLength(10), null);
+        users.add(user1);
+        enabledUsers.add(user1);
+        Map<String, Object> metadata2 = new HashMap<>();
+        metadata2.put(randomAlphaOfLengthBetween(1, 5), randomInt());
+        metadata2.put(randomAlphaOfLengthBetween(1, 5), randomBoolean());
+
+        final User user2 = new User(randomAlphaOfLength(8),
+            Arrays.asList(new String[] {randomAlphaOfLength(5), randomAlphaOfLength(5)}),
+            metadata2, randomAlphaOfLength(10), null);
+        users.add(user2);
+        return new GetUsersResponse(users, enabledUsers);
     }
 
     public void testEqualsHashCode() {

diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle
@@ -19,8 +19,8 @@ dependencies {
 ext.expansions = { oss ->
   return [
     'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz",
-    'jdkUrl' : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
-    'jdkVersion' : '11.0.1',
+    'jdkUrl' : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz',
+    'jdkVersion' : '11.0.2',
     'license': oss ? 'Apache-2.0' : 'Elastic License',
     'version' : VersionProperties.elasticsearch
   ]
@@ -58,6 +58,7 @@ void addCopyDockerContextTask(final boolean oss) {
 
 void addCopyDockerfileTask(final boolean oss) {
   task(taskName("copy", oss, "Dockerfile"), type: Copy) {
+    inputs.properties(expansions(oss)) // ensure task is run when ext.expansions is changed
     mustRunAfter(taskName("copy", oss, "DockerContext"))
     into files(oss)
 
@@ -82,7 +83,7 @@ void addBuildDockerImage(final boolean oss) {
       ]
     }
     executable 'docker'
-    final List<String> dockerArgs = ['build', files(oss), '--pull']
+    final List<String> dockerArgs = ['build', files(oss), '--pull', '--no-cache']
     for (final String tag : tags) {
       dockerArgs.add('--tag')
       dockerArgs.add(tag)

diff --git a/docs/reference/migration/migrate_7_0/settings.asciidoc b/docs/reference/migration/migrate_7_0/settings.asciidoc
@@ -131,3 +131,26 @@ The removal of these default settings also removes the ability for a component t
 fallback to a default configuration when using TLS. Each component (realm, transport, http,
 http client, etc) must now be configured with their own settings for TLS if it is being
 used.
+
+[float]
+[[watcher-notifications-account-settings]]
+==== Watcher notifications account settings
+
+The following settings have been removed in favor of the secure variants.
+The <<secure-settings, secure settings>> have to be defined inside each cluster
+node's keystore, i.e., they are not to be specified via the cluster settings API.
+
+- `xpack.notification.email.account.<id>.smtp.password`, instead use
+`xpack.notification.email.account.<id>.smtp.secure_password`
+- `xpack.notification.hipchat.account.<id>.auth_token`, instead use
+`xpack.notification.hipchat.account.<id>.secure_auth_token`
+- `xpack.notification.jira.account.<id>.url`, instead use
+`xpack.notification.jira.account.<id>.secure_url`
+- `xpack.notification.jira.account.<id>.user`, instead use
+`xpack.notification.jira.account.<id>.secure_user`
+- `xpack.notification.jira.account.<id>.password`, instead use
+`xpack.notification.jira.account.<id>.secure_password`
+- `xpack.notification.pagerduty.account.<id>.service_api_key`, instead use
+`xpack.notification.pagerduty.account.<id>.secure_service_api_key`
+- `xpack.notification.slack.account.<id>.url`, instead use
+`xpack.notification.slack.account.<id>.secure_url`
diff --git a/docs/reference/settings/notification-settings.asciidoc b/docs/reference/settings/notification-settings.asciidoc
@@ -115,7 +115,7 @@ can specify the following email account attributes:
   `smtp.user` (<<cluster-update-settings,Dynamic>>);;
   The user name for SMTP. Required.
 
-  `smtp.password` (<<cluster-update-settings,Dynamic>>);;
+  `smtp.secure_password` (<<secure-settings,Secure>>);;
   The password for the specified SMTP user.
 
   `smtp.starttls.enable` (<<cluster-update-settings,Dynamic>>);;
@@ -222,9 +222,8 @@ via HipChat. You can specify the following HipChat account attributes:
   The HipChat account profile to use: `integration`,
                       `user`, or `v1`. Required.
 
-  `auth_token`;;
-  The authentication token to use to access
-                      the HipChat API. Required.
+  `secure_auth_token` (<<secure-settings,Secure>>);;
+  The authentication token to use to access the HipChat API. Required.
 
   `host`;;
   The HipChat server hostname. Defaults to `api.hipchat.com`.
@@ -268,9 +267,8 @@ via Slack. You can specify the following Slack account attributes:
 
 [[slack-account-attributes]]
 
-  `url`;;
-  The Incoming Webhook URL to use to post
-                                    messages to Slack. Required.
+  `secure_url` (<<secure-settings,Secure>>);;
+  The Incoming Webhook URL to use to post messages to Slack. Required.
 
   `message_defaults.from`;;
   The sender name to display in the
@@ -309,13 +307,13 @@ issues in Jira. You can specify the following Jira account attributes:
 
 [[jira-account-attributes]]
 
-  `url`;;
+  `secure_url` (<<secure-settings,Secure>>);;
   The URL of the Jira Software server. Required.
 
-  `user`;;
+  `secure_user` (<<secure-settings,Secure>>);;
   The name of the user to connect to the Jira Software server. Required.
 
-  `password`;;
+  `secure_password` (<<secure-settings,Secure>>);;
   The password of the user to connect to the Jira Software server. Required.
 
   `issue_defaults`;;
@@ -341,7 +339,7 @@ via PagerDuty. You can specify the following PagerDuty account attributes:
   A name for the PagerDuty account associated with the API key you
   are using to access PagerDuty. Required.
 
-  `service_api_key`;;
+  `secure_service_api_key` (<<secure-settings,Secure>>);;
    The https://developer.pagerduty.com/documentation/rest/authentication[
    PagerDuty API key] to use to access PagerDuty. Required.
 

diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
@@ -161,10 +161,10 @@ public Collection<Object> createComponents(
             return emptyList();
         }
 
-        CcrRestoreSourceService restoreSourceService = new CcrRestoreSourceService();
-        this.restoreSourceService.set(restoreSourceService);
         CcrSettings ccrSettings = new CcrSettings(settings, clusterService.getClusterSettings());
         this.ccrSettings.set(ccrSettings);
+        CcrRestoreSourceService restoreSourceService = new CcrRestoreSourceService(threadPool, ccrSettings);
+        this.restoreSourceService.set(restoreSourceService);
         return Arrays.asList(
             ccrLicenseChecker,
             restoreSourceService,
@@ -306,7 +306,9 @@ public Map<String, Repository.Factory> getInternalRepositories(Environment env,
 
     @Override
     public void onIndexModule(IndexModule indexModule) {
-        indexModule.addIndexEventListener(this.restoreSourceService.get());
+        if (enabled) {
+            indexModule.addIndexEventListener(this.restoreSourceService.get());
+        }
     }
 
     protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }

diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java
@@ -43,6 +43,14 @@ public final class CcrSettings {
         Setting.byteSizeSetting("ccr.indices.recovery.max_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB),
             Setting.Property.Dynamic, Setting.Property.NodeScope);
 
+    /**
+     * The leader must open resources for a ccr recovery. If there is no activity for this interval of time,
+     * the leader will close the restore session.
+     */
+    public static final Setting<TimeValue> INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING =
+        Setting.timeSetting("ccr.indices.recovery.recovery_activity_timeout", TimeValue.timeValueSeconds(60),
+            Setting.Property.Dynamic, Setting.Property.NodeScope);
+
     /**
      * The settings defined by CCR.
      *
@@ -53,22 +61,33 @@ static List<Setting<?>> getSettings() {
                 XPackSettings.CCR_ENABLED_SETTING,
                 CCR_FOLLOWING_INDEX_SETTING,
                 RECOVERY_MAX_BYTES_PER_SECOND,
+                INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING,
                 CCR_AUTO_FOLLOW_WAIT_FOR_METADATA_TIMEOUT);
     }
 
     private final CombinedRateLimiter ccrRateLimiter;
+    private volatile TimeValue recoveryActivityTimeout;
 
     public CcrSettings(Settings settings, ClusterSettings clusterSettings) {
+        this.recoveryActivityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings);
         this.ccrRateLimiter = new CombinedRateLimiter(RECOVERY_MAX_BYTES_PER_SECOND.get(settings));
         clusterSettings.addSettingsUpdateConsumer(RECOVERY_MAX_BYTES_PER_SECOND, this::setMaxBytesPerSec);
+        clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setRecoveryActivityTimeout);
     }
 
     private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) {
         ccrRateLimiter.setMBPerSec(maxBytesPerSec);
     }
 
+    private void setRecoveryActivityTimeout(TimeValue recoveryActivityTimeout) {
+        this.recoveryActivityTimeout = recoveryActivityTimeout;
+    }
+
     public CombinedRateLimiter getRateLimiter() {
         return ccrRateLimiter;
     }
 
+    public TimeValue getRecoveryActivityTimeout() {
+        return recoveryActivityTimeout;
+    }
 }