diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index d4061fc5d74fb..fac44056fb4a4 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -40,11 +40,15 @@ Those that must be stored in the keystore are marked as `Secure`. `access_key`:: - An s3 access key. The `secret_key` setting must also be specified. (Secure) + An ec2 access key. The `secret_key` setting must also be specified. (Secure) `secret_key`:: - An s3 secret key. The `access_key` setting must also be specified. (Secure) + An ec2 secret key. The `access_key` setting must also be specified. (Secure) + +`session_token`:: + An ec2 session token. The `access_key` and `secret_key` settings must also + be specified. (Secure) `endpoint`:: diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index 6701d53c24047..0d73e35f18ec3 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -73,6 +73,10 @@ are marked as `Secure`. An s3 secret key. The `access_key` setting must also be specified. (Secure) +`session_token`:: + An s3 session token. The `access_key` and `secret_key` settings must also + be specified. (Secure) + `endpoint`:: The s3 service endpoint to connect to. This will be automatically diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java index 67902174630ea..a65500d9e2289 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImpl.java @@ -19,12 +19,9 @@ package org.elasticsearch.discovery.ec2; -import java.util.Random; -import java.util.concurrent.atomic.AtomicReference; - import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; @@ -39,6 +36,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.LazyInitializable; +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; + class AwsEc2ServiceImpl extends AbstractComponent implements AwsEc2Service { public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/"; @@ -99,7 +99,7 @@ static ClientConfiguration buildConfiguration(Logger logger, Ec2ClientSettings c // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, Ec2ClientSettings clientSettings) { - final BasicAWSCredentials credentials = clientSettings.credentials; + final AWSCredentials credentials = clientSettings.credentials; if (credentials == null) { logger.debug("Using either environment variables, system properties or instance profile credentials"); return new DefaultAWSCredentialsProviderChain(); diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java index b42b0d546001a..d76c9e820b8b1 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2ClientSettings.java @@ -21,14 +21,20 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; - +import com.amazonaws.auth.BasicSessionCredentials; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.unit.TimeValue; + import java.util.Locale; /** @@ -42,6 +48,9 @@ final class Ec2ClientSettings { /** The secret key (ie password) for connecting to ec2. */ static final Setting SECRET_KEY_SETTING = SecureSetting.secureString("discovery.ec2.secret_key", null); + /** The session token for connecting to ec2. */ + static final Setting SESSION_TOKEN_SETTING = SecureSetting.secureString("discovery.ec2.session_token", null); + /** The host name of a proxy to connect to ec2 through. */ static final Setting PROXY_HOST_SETTING = Setting.simpleString("discovery.ec2.proxy.host", Property.NodeScope); @@ -66,8 +75,12 @@ final class Ec2ClientSettings { static final Setting READ_TIMEOUT_SETTING = Setting.timeSetting("discovery.ec2.read_timeout", TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope); + private static final Logger logger = Loggers.getLogger(Ec2ClientSettings.class); + + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(logger); + /** Credentials to authenticate with ec2. */ - final BasicAWSCredentials credentials; + final AWSCredentials credentials; /** * The ec2 endpoint the client should talk to, or empty string to use the @@ -96,7 +109,7 @@ final class Ec2ClientSettings { /** The read timeout for the ec2 client. */ final int readTimeoutMillis; - protected Ec2ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, + protected Ec2ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis) { this.credentials = credentials; this.endpoint = endpoint; @@ -108,26 +121,45 @@ protected Ec2ClientSettings(BasicAWSCredentials credentials, String endpoint, Pr this.readTimeoutMillis = readTimeoutMillis; } - static BasicAWSCredentials loadCredentials(Settings settings) { - try (SecureString accessKey = ACCESS_KEY_SETTING.get(settings); - SecureString secretKey = SECRET_KEY_SETTING.get(settings);) { - if (accessKey.length() != 0) { - if (secretKey.length() != 0) { - return new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + static AWSCredentials loadCredentials(Settings settings) { + try (SecureString key = ACCESS_KEY_SETTING.get(settings); + SecureString secret = SECRET_KEY_SETTING.get(settings); + SecureString sessionToken = SESSION_TOKEN_SETTING.get(settings)) { + if (key.length() == 0 && secret.length() == 0) { + if (sessionToken.length() > 0) { + throw new SettingsException("Setting [{}] is set but [{}] and [{}] are not", + SESSION_TOKEN_SETTING.getKey(), ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey()); + } + + logger.debug("Using either environment variables, system properties or instance profile credentials"); + return null; + } else { + if (key.length() == 0) { + DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future", + SECRET_KEY_SETTING.getKey(), ACCESS_KEY_SETTING.getKey()); + } + if (secret.length() == 0) { + DEPRECATION_LOGGER.deprecated("Setting [{}] is set but [{}] is not, which will be unsupported in future", + ACCESS_KEY_SETTING.getKey(), SECRET_KEY_SETTING.getKey()); + } + + final AWSCredentials credentials; + if (sessionToken.length() == 0) { + logger.debug("Using basic key/secret credentials"); + credentials = new BasicAWSCredentials(key.toString(), secret.toString()); } else { - throw new IllegalArgumentException("Missing secret key for ec2 client."); + logger.debug("Using basic session credentials"); + credentials = new BasicSessionCredentials(key.toString(), secret.toString(), sessionToken.toString()); } - } else if (secretKey.length() != 0) { - throw new IllegalArgumentException("Missing access key for ec2 client."); + return credentials; } - return null; } } // pkg private for tests /** Parse settings for a single client. */ static Ec2ClientSettings getClientSettings(Settings settings) { - final BasicAWSCredentials credentials = loadCredentials(settings); + final AWSCredentials credentials = loadCredentials(settings); try (SecureString proxyUsername = PROXY_USERNAME_SETTING.get(settings); SecureString proxyPassword = PROXY_PASSWORD_SETTING.get(settings)) { return new Ec2ClientSettings( diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java index 79f653d5bde55..d476d00eef860 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPlugin.java @@ -107,6 +107,7 @@ public List> getSettings() { // Register EC2 discovery settings: discovery.ec2 Ec2ClientSettings.ACCESS_KEY_SETTING, Ec2ClientSettings.SECRET_KEY_SETTING, + Ec2ClientSettings.SESSION_TOKEN_SETTING, Ec2ClientSettings.ENDPOINT_SETTING, Ec2ClientSettings.PROTOCOL_SETTING, Ec2ClientSettings.PROXY_HOST_SETTING, diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java index a13fe47a632ae..148e58d7b3c06 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java @@ -23,10 +23,11 @@ import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.BasicSessionCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.discovery.ec2.AwsEc2ServiceImpl; +import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.instanceOf; @@ -44,15 +45,53 @@ public void testAWSCredentialsWithElasticsearchAwsSettings() { final MockSecureSettings secureSettings = new MockSecureSettings(); secureSettings.setString("discovery.ec2.access_key", "aws_key"); secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); - final Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - launchAWSCredentialsWithElasticsearchSettingsTest(settings, "aws_key", "aws_secret"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); } - protected void launchAWSCredentialsWithElasticsearchSettingsTest(Settings settings, String expectedKey, String expectedSecret) { - final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, Ec2ClientSettings.getClientSettings(settings)) - .getCredentials(); - assertThat(credentials.getAWSAccessKeyId(), is(expectedKey)); - assertThat(credentials.getAWSSecretKey(), is(expectedSecret)); + public void testAWSSessionCredentialsWithElasticsearchAwsSettings() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.access_key", "aws_key"); + secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); + secureSettings.setString("discovery.ec2.session_token", "aws_session_token"); + final BasicSessionCredentials credentials = (BasicSessionCredentials) AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); + assertThat(credentials.getSessionToken(), is("aws_session_token")); + } + + public void testDeprecationOfLoneAccessKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.access_key", "aws_key"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("aws_key")); + assertThat(credentials.getAWSSecretKey(), is("")); + assertSettingDeprecationsAndWarnings(new String[]{}, + "Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not, which will be unsupported in future"); + } + + public void testDeprecationOfLoneSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.secret_key", "aws_secret"); + final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())).getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("")); + assertThat(credentials.getAWSSecretKey(), is("aws_secret")); + assertSettingDeprecationsAndWarnings(new String[]{}, + "Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not, which will be unsupported in future"); + } + + public void testRejectionOfLoneSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("discovery.ec2.session_token", "aws_session_token"); + SettingsException e = expectThrows(SettingsException.class, () -> AwsEc2ServiceImpl.buildCredentials(logger, + Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build()))); + assertThat(e.getMessage(), is( + "Setting [discovery.ec2.session_token] is set but [discovery.ec2.access_key] and [discovery.ec2.secret_key] are not")); } public void testAWSDefaultConfiguration() { diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java index 6001ab56d5042..720ffaddd74a5 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java @@ -19,22 +19,24 @@ package org.elasticsearch.discovery.ec2; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; +import org.elasticsearch.test.ESTestCase; + import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -import org.elasticsearch.discovery.ec2.AwsEc2Service; -import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.discovery.ec2.Ec2DiscoveryPlugin; -import org.elasticsearch.node.Node; -import org.elasticsearch.test.ESTestCase; - public class Ec2DiscoveryPluginTests extends ESTestCase { private Settings getNodeAttributes(Settings settings, String url) { @@ -106,6 +108,10 @@ public void testClientSettingsReInit() throws IOException { final MockSecureSettings mockSecure1 = new MockSecureSettings(); mockSecure1.setString(Ec2ClientSettings.ACCESS_KEY_SETTING.getKey(), "ec2_access_1"); mockSecure1.setString(Ec2ClientSettings.SECRET_KEY_SETTING.getKey(), "ec2_secret_1"); + final boolean mockSecure1HasSessionToken = randomBoolean(); + if (mockSecure1HasSessionToken) { + mockSecure1.setString(Ec2ClientSettings.SESSION_TOKEN_SETTING.getKey(), "ec2_session_token_1"); + } mockSecure1.setString(Ec2ClientSettings.PROXY_USERNAME_SETTING.getKey(), "proxy_username_1"); mockSecure1.setString(Ec2ClientSettings.PROXY_PASSWORD_SETTING.getKey(), "proxy_password_1"); final Settings settings1 = Settings.builder() @@ -117,6 +123,10 @@ public void testClientSettingsReInit() throws IOException { final MockSecureSettings mockSecure2 = new MockSecureSettings(); mockSecure2.setString(Ec2ClientSettings.ACCESS_KEY_SETTING.getKey(), "ec2_access_2"); mockSecure2.setString(Ec2ClientSettings.SECRET_KEY_SETTING.getKey(), "ec2_secret_2"); + final boolean mockSecure2HasSessionToken = randomBoolean(); + if (mockSecure2HasSessionToken) { + mockSecure2.setString(Ec2ClientSettings.SESSION_TOKEN_SETTING.getKey(), "ec2_session_token_2"); + } mockSecure2.setString(Ec2ClientSettings.PROXY_USERNAME_SETTING.getKey(), "proxy_username_2"); mockSecure2.setString(Ec2ClientSettings.PROXY_PASSWORD_SETTING.getKey(), "proxy_password_2"); final Settings settings2 = Settings.builder() @@ -127,27 +137,50 @@ public void testClientSettingsReInit() throws IOException { .build(); try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(settings1)) { try (AmazonEc2Reference clientReference = plugin.ec2Service.client()) { - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); - assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); + { + final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("ec2_access_1")); + assertThat(credentials.getAWSSecretKey(), is("ec2_secret_1")); + if (mockSecure1HasSessionToken) { + assertThat(credentials, instanceOf(BasicSessionCredentials.class)); + assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_1")); + } else { + assertThat(credentials, instanceOf(BasicAWSCredentials.class)); + } + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); + assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); + } // reload secure settings2 plugin.reload(settings2); // client is not released, it is still using the old settings - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); - assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); - assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); + { + final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials(); + if (mockSecure1HasSessionToken) { + assertThat(credentials, instanceOf(BasicSessionCredentials.class)); + assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_1")); + } else { + assertThat(credentials, instanceOf(BasicAWSCredentials.class)); + } + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1")); + assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881)); + assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1")); + } } try (AmazonEc2Reference clientReference = plugin.ec2Service.client()) { - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSAccessKeyId(), is("ec2_access_2")); - assertThat(((AmazonEC2Mock) clientReference.client()).credentials.getCredentials().getAWSSecretKey(), is("ec2_secret_2")); + final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials(); + assertThat(credentials.getAWSAccessKeyId(), is("ec2_access_2")); + assertThat(credentials.getAWSSecretKey(), is("ec2_secret_2")); + if (mockSecure2HasSessionToken) { + assertThat(credentials, instanceOf(BasicSessionCredentials.class)); + assertThat(((BasicSessionCredentials)credentials).getSessionToken(), is("ec2_session_token_2")); + } else { + assertThat(credentials, instanceOf(BasicAWSCredentials.class)); + } assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_2")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_2")); assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_2")); diff --git a/plugins/repository-s3/qa/amazon-s3/build.gradle b/plugins/repository-s3/qa/amazon-s3/build.gradle index dbbffdebded47..b6cc4a6de310d 100644 --- a/plugins/repository-s3/qa/amazon-s3/build.gradle +++ b/plugins/repository-s3/qa/amazon-s3/build.gradle @@ -31,47 +31,81 @@ integTestCluster { plugin ':plugins:repository-s3' } +forbiddenApisTest { + // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage + bundledSignatures -= 'jdk-non-portable' + bundledSignatures += 'jdk-internal' +} + boolean useFixture = false -String s3AccessKey = System.getenv("amazon_s3_access_key") -String s3SecretKey = System.getenv("amazon_s3_secret_key") -String s3Bucket = System.getenv("amazon_s3_bucket") -String s3BasePath = System.getenv("amazon_s3_base_path") +// We test against two repositories, one which uses the usual two-part "permanent" credentials and +// the other which uses three-part "temporary" or "session" credentials. + +String s3PermanentAccessKey = System.getenv("amazon_s3_access_key") +String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key") +String s3PermanentBucket = System.getenv("amazon_s3_bucket") +String s3PermanentBasePath = System.getenv("amazon_s3_base_path") + +String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary") +String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary") +String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary") +String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary") +String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") + +// If all these variables are missing then we are testing against the internal fixture instead, which has the following +// credentials hard-coded in. + +if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath + && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) { + + s3PermanentAccessKey = 's3_integration_test_permanent_access_key' + s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' + s3PermanentBucket = 'permanent_bucket_test' + s3PermanentBasePath = 'integration_test' + + s3TemporaryAccessKey = 's3_integration_test_temporary_access_key' + s3TemporarySecretKey = 's3_integration_test_temporary_secret_key' + s3TemporaryBucket = 'temporary_bucket_test' + s3TemporaryBasePath = 'integration_test' + s3TemporarySessionToken = 's3_integration_test_temporary_session_token' -if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { - s3AccessKey = 's3_integration_test_access_key' - s3SecretKey = 's3_integration_test_secret_key' - s3Bucket = 'bucket_test' - s3BasePath = 'integration_test' useFixture = true } /** A task to start the AmazonS3Fixture which emulates a S3 service **/ task s3Fixture(type: AntFixture) { - dependsOn testClasses + dependsOn compileTestJava env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" executable = new File(project.runtimeJavaHome, 'bin/java') - args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3Bucket + args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3PermanentBucket, s3TemporaryBucket } Map expansions = [ - 'bucket': s3Bucket, - 'base_path': s3BasePath + 'permanent_bucket': s3PermanentBucket, + 'permanent_base_path': s3PermanentBasePath, + 'temporary_bucket': s3TemporaryBucket, + 'temporary_base_path': s3TemporaryBasePath ] - processTestResources { inputs.properties(expansions) MavenFilteringHack.filter(it, expansions) } integTestCluster { - keystoreSetting 's3.client.integration_test.access_key', s3AccessKey - keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey + keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey + keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey + + keystoreSetting 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey + keystoreSetting 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey + keystoreSetting 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken if (useFixture) { + println "Using internal test service to test the repository-s3 plugin" dependsOn s3Fixture /* Use a closure on the string to delay evaluation until tests are executed */ - setting 's3.client.integration_test.endpoint', "http://${-> s3Fixture.addressAndPort}" + setting 's3.client.integration_test_permanent.endpoint', "http://${-> s3Fixture.addressAndPort}" + setting 's3.client.integration_test_temporary.endpoint', "http://${-> s3Fixture.addressAndPort}" } else { println "Using an external service to test the repository-s3 plugin" } diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index d1034aff48248..fcb208258aa03 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -52,13 +52,19 @@ public class AmazonS3Fixture extends AbstractHttpFixture { /** Request handlers for the requests made by the S3 client **/ private final PathTrie handlers; + private final String permanentBucketName; + private final String temporaryBucketName; /** * Creates a {@link AmazonS3Fixture} */ - private AmazonS3Fixture(final String workingDir, final String bucket) { + private AmazonS3Fixture(final String workingDir, final String permanentBucketName, final String temporaryBucketName) { super(workingDir); - this.buckets.put(bucket, new Bucket(bucket)); + this.permanentBucketName = permanentBucketName; + this.temporaryBucketName = temporaryBucketName; + + this.buckets.put(permanentBucketName, new Bucket(permanentBucketName)); + this.buckets.put(temporaryBucketName, new Bucket(temporaryBucketName)); this.handlers = defaultHandlers(buckets); } @@ -67,21 +73,47 @@ protected Response handle(final Request request) throws IOException { final RequestHandler handler = handlers.retrieve(request.getMethod() + " " + request.getPath(), request.getParameters()); if (handler != null) { final String authorization = request.getHeader("Authorization"); - if (authorization == null - || (authorization.length() > 0 && authorization.contains("s3_integration_test_access_key") == false)) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Access Denied", ""); + final String permittedBucket; + if (authorization.contains("s3_integration_test_permanent_access_key")) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (sessionToken != null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Unexpected session token", ""); + } + permittedBucket = permanentBucketName; + } else if (authorization.contains("s3_integration_test_temporary_access_key")) { + final String sessionToken = request.getHeader("x-amz-security-token"); + if (sessionToken == null) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "No session token", ""); + } + if (sessionToken.equals("s3_integration_test_temporary_session_token") == false) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad session token", ""); + } + permittedBucket = temporaryBucketName; + } else { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); + } + + if (handler != null) { + final String bucket = request.getParam("bucket"); + if (bucket != null && permittedBucket.equals(bucket) == false) { + // allow a null bucket to support bucket-free APIs + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", ""); + } + return handler.handle(request); + } else { + return newInternalError(request.getId(), "No handler defined for request [" + request + "]"); } - return handler.handle(request); } return null; } public static void main(final String[] args) throws Exception { - if (args == null || args.length != 2) { - throw new IllegalArgumentException("AmazonS3Fixture "); + if (args == null || args.length != 3) { + throw new IllegalArgumentException( + "AmazonS3Fixture "); } - final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1]); + final AmazonS3Fixture fixture = new AmazonS3Fixture(args[0], args[1], args[2]); fixture.listen(); } diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml similarity index 73% rename from plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml rename to plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml index 56e2b2cb8fa16..bb934d0931ca9 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository_permanent_credentials.yml @@ -3,34 +3,35 @@ --- setup: - # Register repository + # Register repository with permanent credentials - do: snapshot.create_repository: - repository: repository + repository: repository_permanent body: type: s3 settings: - bucket: ${bucket} - client: integration_test - base_path: ${base_path} + bucket: ${permanent_bucket} + client: integration_test_permanent + base_path: ${permanent_base_path} canned_acl: private storage_class: standard --- -"Snapshot/Restore with repository-s3": +"Snapshot/Restore with repository-s3 using permanent credentials": # Get repository - do: snapshot.get_repository: - repository: repository + repository: repository_permanent - - match: { repository.settings.bucket : ${bucket} } - - match: { repository.settings.client : "integration_test" } - - match: { repository.settings.base_path : ${base_path} } - - match: { repository.settings.canned_acl : "private" } - - match: { repository.settings.storage_class : "standard" } - - is_false: repository.settings.access_key - - is_false: repository.settings.secret_key + - match: { repository_permanent.settings.bucket : ${permanent_bucket} } + - match: { repository_permanent.settings.client : "integration_test_permanent" } + - match: { repository_permanent.settings.base_path : ${permanent_base_path} } + - match: { repository_permanent.settings.canned_acl : "private" } + - match: { repository_permanent.settings.storage_class : "standard" } + - is_false: repository_permanent.settings.access_key + - is_false: repository_permanent.settings.secret_key + - is_false: repository_permanent.settings.session_token # Index documents - do: @@ -62,7 +63,7 @@ setup: # Create a first snapshot - do: snapshot.create: - repository: repository + repository: repository_permanent snapshot: snapshot-one wait_for_completion: true @@ -73,7 +74,7 @@ setup: - do: snapshot.status: - repository: repository + repository: repository_permanent snapshot: snapshot-one - is_true: snapshots @@ -115,7 +116,7 @@ setup: # Create a second snapshot - do: snapshot.create: - repository: repository + repository: repository_permanent snapshot: snapshot-two wait_for_completion: true @@ -125,7 +126,7 @@ setup: - do: snapshot.get: - repository: repository + repository: repository_permanent snapshot: snapshot-one,snapshot-two - is_true: snapshots @@ -140,7 +141,7 @@ setup: # Restore the second snapshot - do: snapshot.restore: - repository: repository + repository: repository_permanent snapshot: snapshot-two wait_for_completion: true @@ -158,7 +159,7 @@ setup: # Restore the first snapshot - do: snapshot.restore: - repository: repository + repository: repository_permanent snapshot: snapshot-one wait_for_completion: true @@ -171,12 +172,12 @@ setup: # Remove the snapshots - do: snapshot.delete: - repository: repository + repository: repository_permanent snapshot: snapshot-two - do: snapshot.delete: - repository: repository + repository: repository_permanent snapshot: snapshot-one --- @@ -185,7 +186,7 @@ setup: - do: catch: /repository_exception/ snapshot.create_repository: - repository: repository + repository: repository_permanent body: type: s3 settings: @@ -198,11 +199,11 @@ setup: - do: catch: /repository_exception/ snapshot.create_repository: - repository: repository + repository: repository_permanent body: type: s3 settings: - bucket: repository + bucket: repository_permanent client: unknown --- @@ -211,7 +212,7 @@ setup: - do: catch: /snapshot_missing_exception/ snapshot.get: - repository: repository + repository: repository_permanent snapshot: missing --- @@ -220,7 +221,7 @@ setup: - do: catch: /snapshot_missing_exception/ snapshot.delete: - repository: repository + repository: repository_permanent snapshot: missing --- @@ -229,7 +230,7 @@ setup: - do: catch: /snapshot_restore_exception/ snapshot.restore: - repository: repository + repository: repository_permanent snapshot: missing wait_for_completion: true @@ -239,4 +240,4 @@ teardown: # Remove our repository - do: snapshot.delete_repository: - repository: repository + repository: repository_permanent diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml new file mode 100644 index 0000000000000..5da4f739cd522 --- /dev/null +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository_temporary_credentials.yml @@ -0,0 +1,243 @@ +# Integration tests for repository-s3 + +--- +setup: + + # Register repository with temporary credentials + - do: + snapshot.create_repository: + repository: repository_temporary + body: + type: s3 + settings: + bucket: ${temporary_bucket} + client: integration_test_temporary + base_path: ${temporary_base_path} + canned_acl: private + storage_class: standard + +--- +"Snapshot/Restore with repository-s3 using temporary credentials": + + # Get repository + - do: + snapshot.get_repository: + repository: repository_temporary + + - match: { repository_temporary.settings.bucket : ${temporary_bucket} } + - match: { repository_temporary.settings.client : "integration_test_temporary" } + - match: { repository_temporary.settings.base_path : ${temporary_base_path} } + - match: { repository_temporary.settings.canned_acl : "private" } + - match: { repository_temporary.settings.storage_class : "standard" } + - is_false: repository_temporary.settings.access_key + - is_false: repository_temporary.settings.secret_key + - is_false: repository_temporary.settings.session_token + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Create a first snapshot + - do: + snapshot.create: + repository: repository_temporary + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.status: + repository: repository_temporary + snapshot: snapshot-one + + - is_true: snapshots + - match: { snapshots.0.snapshot: snapshot-one } + - match: { snapshots.0.state : SUCCESS } + + # Index more documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 4 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 5 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 6 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 7 + - snapshot: two + + - do: + count: + index: docs + + - match: {count: 7} + + # Create a second snapshot + - do: + snapshot.create: + repository: repository_temporary + snapshot: snapshot-two + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-two } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.get: + repository: repository_temporary + snapshot: snapshot-one,snapshot-two + + - is_true: snapshots + - match: { snapshots.0.state : SUCCESS } + - match: { snapshots.1.state : SUCCESS } + + # Delete the index + - do: + indices.delete: + index: docs + + # Restore the second snapshot + - do: + snapshot.restore: + repository: repository_temporary + snapshot: snapshot-two + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 7} + + # Delete the index again + - do: + indices.delete: + index: docs + + # Restore the first snapshot + - do: + snapshot.restore: + repository: repository_temporary + snapshot: snapshot-one + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 3} + + # Remove the snapshots + - do: + snapshot.delete: + repository: repository_temporary + snapshot: snapshot-two + + - do: + snapshot.delete: + repository: repository_temporary + snapshot: snapshot-one + +--- +"Register a repository with a non existing bucket": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_temporary + body: + type: s3 + settings: + bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE + client: integration_test + +--- +"Register a repository with a non existing client": + + - do: + catch: /repository_exception/ + snapshot.create_repository: + repository: repository_temporary + body: + type: s3 + settings: + bucket: repository_temporary + client: unknown + +--- +"Get a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.get: + repository: repository_temporary + snapshot: missing + +--- +"Delete a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.delete: + repository: repository_temporary + snapshot: missing + +--- +"Restore a non existing snapshot": + + - do: + catch: /snapshot_restore_exception/ + snapshot.restore: + repository: repository_temporary + snapshot: missing + wait_for_completion: true + +--- +teardown: + + # Remove our repository + - do: + snapshot.delete_repository: + repository: repository_temporary diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index ef6088fe154bf..795304541be35 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -26,8 +26,10 @@ import java.util.Set; import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.settings.SecureSetting; import org.elasticsearch.common.settings.SecureString; @@ -52,6 +54,10 @@ final class S3ClientSettings { static final Setting.AffixSetting SECRET_KEY_SETTING = Setting.affixKeySetting(PREFIX, "secret_key", key -> SecureSetting.secureString(key, null)); + /** The secret key (ie password) for connecting to s3. */ + static final Setting.AffixSetting SESSION_TOKEN_SETTING = Setting.affixKeySetting(PREFIX, "session_token", + key -> SecureSetting.secureString(key, null)); + /** An override for the s3 endpoint to connect to. */ static final Setting.AffixSetting ENDPOINT_SETTING = Setting.affixKeySetting(PREFIX, "endpoint", key -> new Setting<>(key, "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope)); @@ -89,7 +95,7 @@ final class S3ClientSettings { key -> Setting.boolSetting(key, ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope)); /** Credentials to authenticate with s3. */ - final BasicAWSCredentials credentials; + final AWSCredentials credentials; /** The s3 endpoint the client should talk to, or empty string to use the default. */ final String endpoint; @@ -120,7 +126,7 @@ final class S3ClientSettings { /** Whether the s3 client should use an exponential backoff retry policy. */ final boolean throttleRetries; - protected S3ClientSettings(BasicAWSCredentials credentials, String endpoint, Protocol protocol, + protected S3ClientSettings(AWSCredentials credentials, String endpoint, Protocol protocol, String proxyHost, int proxyPort, String proxyUsername, String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries) { this.credentials = credentials; @@ -190,26 +196,36 @@ static BasicAWSCredentials loadDeprecatedCredentials(Settings repositorySettings } } - static BasicAWSCredentials loadCredentials(Settings settings, String clientName) { + static AWSCredentials loadCredentials(Settings settings, String clientName) { try (SecureString accessKey = getConfigValue(settings, clientName, ACCESS_KEY_SETTING); - SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING);) { + SecureString secretKey = getConfigValue(settings, clientName, SECRET_KEY_SETTING); + SecureString sessionToken = getConfigValue(settings, clientName, SESSION_TOKEN_SETTING)) { if (accessKey.length() != 0) { if (secretKey.length() != 0) { - return new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + if (sessionToken.length() != 0) { + return new BasicSessionCredentials(accessKey.toString(), secretKey.toString(), sessionToken.toString()); + } else { + return new BasicAWSCredentials(accessKey.toString(), secretKey.toString()); + } } else { throw new IllegalArgumentException("Missing secret key for s3 client [" + clientName + "]"); } - } else if (secretKey.length() != 0) { - throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]"); + } else { + if (secretKey.length() != 0) { + throw new IllegalArgumentException("Missing access key for s3 client [" + clientName + "]"); + } + if (sessionToken.length() != 0) { + throw new IllegalArgumentException("Missing access key and secret key for s3 client [" + clientName + "]"); + } + return null; } - return null; } } // pkg private for tests /** Parse settings for a single client. */ static S3ClientSettings getClientSettings(Settings settings, String clientName) { - final BasicAWSCredentials credentials = S3ClientSettings.loadCredentials(settings, clientName); + final AWSCredentials credentials = S3ClientSettings.loadCredentials(settings, clientName); try (SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING); SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING)) { return new S3ClientSettings( diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index 6a605319114fe..79a5187059f38 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -92,6 +92,7 @@ public List> getSettings() { // named s3 client configuration settings S3ClientSettings.ACCESS_KEY_SETTING, S3ClientSettings.SECRET_KEY_SETTING, + S3ClientSettings.SESSION_TOKEN_SETTING, S3ClientSettings.ENDPOINT_SETTING, S3ClientSettings.PROTOCOL_SETTING, S3ClientSettings.PROXY_HOST_SETTING, diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 58c83bd84da49..cb12555d9eaa2 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -22,7 +22,6 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.InstanceProfileCredentialsProvider; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.internal.StaticCredentialsProvider; @@ -133,7 +132,7 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) { - final BasicAWSCredentials credentials = clientSettings.credentials; + final AWSCredentials credentials = clientSettings.credentials; if (credentials == null) { logger.debug("Using instance profile credentials"); return new PrivilegedInstanceProfileCredentialsProvider(); diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java new file mode 100644 index 0000000000000..e629f43f8a3d3 --- /dev/null +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -0,0 +1,123 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.s3; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; + +import java.util.Map; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.isEmptyString; +import static org.hamcrest.Matchers.nullValue; + +public class S3ClientSettingsTests extends ESTestCase { + public void testThereIsADefaultClientByDefault() { + final Map settings = S3ClientSettings.load(Settings.EMPTY); + assertThat(settings.keySet(), contains("default")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.credentials, nullValue()); + assertThat(defaultSettings.endpoint, isEmptyString()); + assertThat(defaultSettings.protocol, is(Protocol.HTTPS)); + assertThat(defaultSettings.proxyHost, isEmptyString()); + assertThat(defaultSettings.proxyPort, is(80)); + assertThat(defaultSettings.proxyUsername, isEmptyString()); + assertThat(defaultSettings.proxyPassword, isEmptyString()); + assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT)); + assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); + assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES)); + } + + public void testDefaultClientSettingsCanBeSet() { + final Map settings = S3ClientSettings.load(Settings.builder() + .put("s3.client.default.max_retries", 10).build()); + assertThat(settings.keySet(), contains("default")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.maxRetries, is(10)); + } + + public void testNondefaultClientCreatedBySettingItsSettings() { + final Map settings = S3ClientSettings.load(Settings.builder() + .put("s3.client.another_client.max_retries", 10).build()); + assertThat(settings.keySet(), contains("default", "another_client")); + + final S3ClientSettings defaultSettings = settings.get("default"); + assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); + + final S3ClientSettings anotherClientSettings = settings.get("another_client"); + assertThat(anotherClientSettings.maxRetries, is(10)); + } + + public void testRejectionOfLoneAccessKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "aws_key"); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); + assertThat(e.getMessage(), is("Missing secret key for s3 client [default]")); + } + + public void testRejectionOfLoneSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.secret_key", "aws_key"); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); + assertThat(e.getMessage(), is("Missing access key for s3 client [default]")); + } + + public void testRejectionOfLoneSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.session_token", "aws_key"); + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build())); + assertThat(e.getMessage(), is("Missing access key and secret key for s3 client [default]")); + } + + public void testCredentialsTypeWithAccessKeyAndSecretKey() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "access_key"); + secureSettings.setString("s3.client.default.secret_key", "secret_key"); + final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final S3ClientSettings defaultSettings = settings.get("default"); + BasicAWSCredentials credentials = (BasicAWSCredentials) defaultSettings.credentials; + assertThat(credentials.getAWSAccessKeyId(), is("access_key")); + assertThat(credentials.getAWSSecretKey(), is("secret_key")); + } + + public void testCredentialsTypeWithAccessKeyAndSecretKeyAndSessionToken() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.access_key", "access_key"); + secureSettings.setString("s3.client.default.secret_key", "secret_key"); + secureSettings.setString("s3.client.default.session_token", "session_token"); + final Map settings = S3ClientSettings.load(Settings.builder().setSecureSettings(secureSettings).build()); + final S3ClientSettings defaultSettings = settings.get("default"); + BasicSessionCredentials credentials = (BasicSessionCredentials) defaultSettings.credentials; + assertThat(credentials.getAWSAccessKeyId(), is("access_key")); + assertThat(credentials.getAWSSecretKey(), is("secret_key")); + assertThat(credentials.getSessionToken(), is("session_token")); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java b/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java index f7d4843c1c03e..ad5f56d7fc0f3 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SettingsException.java @@ -42,4 +42,8 @@ public SettingsException(String message, Throwable cause) { public SettingsException(StreamInput in) throws IOException { super(in); } + + public SettingsException(String msg, Object... args) { + super(msg, args); + } }