From d56d2cf08c97944c163dc36be3608ff800353246 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 23 Mar 2020 13:50:02 +0100 Subject: [PATCH] Upgrade to AWS SDK 1.11.749 (#53962) Upgrading AWS SDK to v1.11.749. Required building clients inside privileged contexts because some class loading that requires privileges now happens there and working around a new SDK bug in the S3 client builder. Closes #53191 --- plugins/discovery-ec2/build.gradle | 2 +- .../aws-java-sdk-core-1.11.636.jar.sha1 | 1 - .../aws-java-sdk-core-1.11.749.jar.sha1 | 1 + .../aws-java-sdk-ec2-1.11.636.jar.sha1 | 1 - .../aws-java-sdk-ec2-1.11.749.jar.sha1 | 1 + .../discovery/ec2/AmazonEC2Fixture.java | 8 ++++++++ .../discovery/ec2/AwsEc2ServiceImpl.java | 20 ++++++++++--------- .../ec2/Ec2DiscoveryPluginTests.java | 14 +++++-------- plugins/repository-s3/build.gradle | 2 +- .../aws-java-sdk-core-1.11.636.jar.sha1 | 1 - .../aws-java-sdk-core-1.11.749.jar.sha1 | 1 + .../aws-java-sdk-s3-1.11.636.jar.sha1 | 1 - .../aws-java-sdk-s3-1.11.749.jar.sha1 | 1 + .../licenses/jmespath-java-1.11.636.jar.sha1 | 1 - .../licenses/jmespath-java-1.11.749.jar.sha1 | 1 + .../repositories/s3/S3Service.java | 9 +++++++-- 16 files changed, 38 insertions(+), 27 deletions(-) delete mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1 diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle index 72d3c5b294bd3..c45d0f9f457c0 100644 --- a/plugins/discovery-ec2/build.gradle +++ b/plugins/discovery-ec2/build.gradle @@ -25,7 +25,7 @@ esplugin { } versions << [ - 'aws': '1.11.636' + 'aws': '1.11.749' ] dependencies { diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1 deleted file mode 100644 index b9ee9c102dbcb..0000000000000 --- a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.636.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84c9f180f8f60f6f1433c9c5253fcb704593b121 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1 new file mode 100644 index 0000000000000..7bc18d6d4f681 --- /dev/null +++ b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.11.749.jar.sha1 @@ -0,0 +1 @@ +1da5c1549295cfeebc67fc1c7539785a9441755b \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1 deleted file mode 100644 index ed737c808c1de..0000000000000 --- a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.636.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d32fc4ae314dbee9717302a3119cba0f735c04b1 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1 new file mode 100644 index 0000000000000..c7c7220005fc3 --- /dev/null +++ b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.11.749.jar.sha1 @@ -0,0 +1 @@ +0865e0937c6500acf62ce9c8964eac76a8718f5f \ No newline at end of file diff --git a/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java b/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java index 32abcdc43e645..ecaaceea5cc26 100644 --- a/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java +++ b/plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java @@ -22,6 +22,7 @@ import org.apache.http.NameValuePair; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; import org.apache.http.client.utils.URLEncodedUtils; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.SuppressForbidden; @@ -106,6 +107,13 @@ protected Response handle(final Request request) throws IOException { return new Response(RestStatus.OK.getStatus(), headers, "my_iam_profile".getBytes(UTF_8)); } + if (instanceProfile && "/latest/api/token".equals(request.getPath()) + && HttpPut.METHOD_NAME.equals(request.getMethod())) { + // TODO: Implement IMDSv2 behavior here. For now this just returns a 403 which makes the SDK fall back to IMDSv1 + // which is implemented in this fixture + return new Response(RestStatus.FORBIDDEN.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE); + } + if ((containerCredentials && "/ecs_credentials_endpoint".equals(request.getPath()) && HttpGet.METHOD_NAME.equals(request.getMethod())) || diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java index ac18775fd1592..1d6f69927ab6d 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java @@ -24,9 +24,10 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; @@ -45,17 +46,18 @@ class AwsEc2ServiceImpl implements AwsEc2Service { private AmazonEC2 buildClient(Ec2ClientSettings clientSettings) { final AWSCredentialsProvider credentials = buildCredentials(logger, clientSettings); final ClientConfiguration configuration = buildConfiguration(clientSettings); - final AmazonEC2 client = buildClient(credentials, configuration); - if (Strings.hasText(clientSettings.endpoint)) { - logger.debug("using explicit ec2 endpoint [{}]", clientSettings.endpoint); - client.setEndpoint(clientSettings.endpoint); - } - return client; + return buildClient(credentials, configuration, clientSettings.endpoint); } // proxy for testing - AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) { - return new AmazonEC2Client(credentials, configuration); + AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration, String endpoint) { + final AmazonEC2ClientBuilder builder = AmazonEC2ClientBuilder.standard().withCredentials(credentials) + .withClientConfiguration(configuration); + if (Strings.hasText(endpoint)) { + logger.debug("using explicit ec2 endpoint [{}]", endpoint); + builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); + } + return SocketAccess.doPrivileged(builder::build); } // pkg private for tests diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java index 661b5815b4029..2e53a5f614101 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java @@ -39,7 +39,6 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; public class Ec2DiscoveryPluginTests extends ESTestCase { @@ -96,7 +95,7 @@ public void testNodeAttributesErrorLenient() throws Exception { public void testDefaultEndpoint() throws IOException { try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(Settings.EMPTY)) { final String endpoint = ((AmazonEC2Mock) plugin.ec2Service.client().client()).endpoint; - assertThat(endpoint, nullValue()); + assertThat(endpoint, is("")); } } @@ -199,8 +198,9 @@ private static class Ec2DiscoveryPluginMock extends Ec2DiscoveryPlugin { Ec2DiscoveryPluginMock(Settings settings) { super(settings, new AwsEc2ServiceImpl() { @Override - AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration) { - return new AmazonEC2Mock(credentials, configuration); + AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration, + String endpoint) { + return new AmazonEC2Mock(credentials, configuration, endpoint); } }); } @@ -212,13 +212,9 @@ private static class AmazonEC2Mock extends AbstractAmazonEC2 { final AWSCredentialsProvider credentials; final ClientConfiguration configuration; - AmazonEC2Mock(AWSCredentialsProvider credentials, ClientConfiguration configuration) { + AmazonEC2Mock(AWSCredentialsProvider credentials, ClientConfiguration configuration, String endpoint) { this.credentials = credentials; this.configuration = configuration; - } - - @Override - public void setEndpoint(String endpoint) throws IllegalArgumentException { this.endpoint = endpoint; } diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 04624f8512976..a7c8d49a50728 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -30,7 +30,7 @@ esplugin { } versions << [ - 'aws': '1.11.636' + 'aws': '1.11.749' ] dependencies { diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1 deleted file mode 100644 index b9ee9c102dbcb..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.636.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84c9f180f8f60f6f1433c9c5253fcb704593b121 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1 new file mode 100644 index 0000000000000..7bc18d6d4f681 --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.11.749.jar.sha1 @@ -0,0 +1 @@ +1da5c1549295cfeebc67fc1c7539785a9441755b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1 deleted file mode 100644 index 1e05e98d240d2..0000000000000 --- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.636.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f86fc1993ac8122f6f02a8eb9b467b5f945cd76b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1 new file mode 100644 index 0000000000000..af794dc59dd7f --- /dev/null +++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.11.749.jar.sha1 @@ -0,0 +1 @@ +7d069f82723907ccdbd0c91ef0ac76046f5c9652 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1 deleted file mode 100644 index 70c0d3633af07..0000000000000 --- a/plugins/repository-s3/licenses/jmespath-java-1.11.636.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e468c349ce410171a1d5df7fa0fa377d52c5d651 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1 b/plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1 new file mode 100644 index 0000000000000..3467802d074c7 --- /dev/null +++ b/plugins/repository-s3/licenses/jmespath-java-1.11.749.jar.sha1 @@ -0,0 +1 @@ +778866bc557dba508ee0eab2a0c5bfde468e49e6 \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 77117550dc0d9..73149bf92073a 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -141,7 +141,12 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { builder.withCredentials(buildCredentials(logger, clientSettings)); builder.withClientConfiguration(buildConfiguration(clientSettings)); - final String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME; + String endpoint = Strings.hasLength(clientSettings.endpoint) ? clientSettings.endpoint : Constants.S3_HOSTNAME; + if ((endpoint.startsWith("http://") || endpoint.startsWith("https://")) == false) { + // Manually add the schema to the endpoint to work around https://github.com/aws/aws-sdk-java/issues/2274 + // TODO: Remove this once fixed in the AWS SDK + endpoint = clientSettings.protocol.toString() + "://" + endpoint; + } final String region = Strings.hasLength(clientSettings.region) ? clientSettings.region : null; logger.debug("using endpoint [{}] and region [{}]", endpoint, region); @@ -160,7 +165,7 @@ AmazonS3 buildClient(final S3ClientSettings clientSettings) { if (clientSettings.disableChunkedEncoding) { builder.disableChunkedEncoding(); } - return builder.build(); + return SocketAccess.doPrivileged(builder::build); } // pkg private for tests