From f5ea951e7f917484a8d44ddb9dcc666220053a15 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Thu, 17 Feb 2022 23:27:13 +0100 Subject: [PATCH] Add support of SOCKS proxies for S3 repository Signed-off-by: Andrey Pleskach --- .../repositories/s3/ProxySettings.java | 123 ++++++++++++++ .../repositories/s3/S3ClientSettings.java | 154 +++++++++++------- .../opensearch/repositories/s3/S3Service.java | 60 ++++++- .../plugin-metadata/plugin-security.policy | 3 + .../s3/AwsS3ServiceImplTests.java | 62 ++++++- .../s3/S3ClientSettingsTests.java | 82 +++++++++- 6 files changed, 411 insertions(+), 73 deletions(-) create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/ProxySettings.java diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/ProxySettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/ProxySettings.java new file mode 100644 index 0000000000000..430af0096d8b5 --- /dev/null +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/ProxySettings.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.repositories.s3; + +import com.amazonaws.Protocol; +import org.opensearch.common.Strings; +import org.opensearch.common.settings.SettingsException; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.Objects; + +public class ProxySettings { + public static final ProxySettings NO_PROXY_SETTINGS = new ProxySettings(ProxyType.DIRECT, null, -1, null, null); + + public static enum ProxyType { + HTTP(Protocol.HTTP.name()), + HTTPS(Protocol.HTTPS.name()), + SOCKS("SOCKS"), + DIRECT("DIRECT"); + + private final String name; + + private ProxyType(String name) { + this.name = name; + } + + public Protocol toProtocol() { + if (this == DIRECT) { + // We check it in settings, + // the probability that it could be thrown is small, but how knows + throw new SettingsException("Couldn't convert to S3 protocol"); + } else if (this == SOCKS) { + throw new SettingsException("Couldn't convert to S3 protocol. SOCKS is not supported"); + } + return Protocol.valueOf(name()); + } + + } + + private final ProxyType type; + + private final String host; + + private final String username; + + private final String password; + + private final int port; + + public String getHost() { + return host; + } + + public ProxySettings(final ProxyType type, final String host, final int port, final String username, final String password) { + this.type = type; + this.host = host; + this.port = port; + this.username = username; + this.password = password; + } + + public ProxyType getType() { + return this.type; + } + + public String getHostName() { + return host; + } + + public int getPort() { + return port; + } + + public InetSocketAddress getAddress() { + try { + return new InetSocketAddress(InetAddress.getByName(host), port); + } catch (UnknownHostException e) { + // this error won't be thrown since validation of the host name is in the S3ClientSettings + throw new RuntimeException(e); + } + } + + public String getUsername() { + return this.username; + } + + public String getPassword() { + return this.password; + } + + public boolean isAuthenticated() { + return Strings.isNullOrEmpty(username) == false && Strings.isNullOrEmpty(password) == false; + } + + public ProxySettings recreateWithNewHostAndPort(final String host, final int port) { + return new ProxySettings(type, host, port, username, password); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final ProxySettings that = (ProxySettings) o; + return port == that.port + && type == that.type + && Objects.equals(host, that.host) + && Objects.equals(username, that.username) + && Objects.equals(password, that.password); + } + + @Override + public int hashCode() { + return Objects.hash(type, host, username, password, port); + } +} diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index 805f48aae9b2d..e02c7cae89378 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -34,13 +34,18 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; +import org.opensearch.common.Strings; +import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.common.settings.SecureSetting; import org.opensearch.common.settings.SecureString; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsException; import org.opensearch.common.unit.TimeValue; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Collections; import java.util.HashMap; import java.util.Locale; @@ -54,6 +59,8 @@ */ final class S3ClientSettings { + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(S3ClientSettings.class); + // prefix for s3 client settings private static final String PREFIX = "s3.client."; @@ -95,6 +102,13 @@ final class S3ClientSettings { key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope) ); + /** The protocol to use to connect to s3. */ + static final Setting.AffixSetting PROXY_TYPE_SETTING = Setting.affixKeySetting( + PREFIX, + "proxy.type", + key -> new Setting<>(key, "direct", s -> ProxySettings.ProxyType.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope) + ); + /** The host name of a proxy to connect to s3 through. */ static final Setting.AffixSetting PROXY_HOST_SETTING = Setting.affixKeySetting( PREFIX, @@ -106,7 +120,7 @@ final class S3ClientSettings { static final Setting.AffixSetting PROXY_PORT_SETTING = Setting.affixKeySetting( PREFIX, "proxy.port", - key -> Setting.intSetting(key, 80, 0, 1 << 16, Property.NodeScope) + key -> Setting.intSetting(key, 80, 0, (1 << 16) - 1, Property.NodeScope) ); /** The username of a proxy to connect to s3 through. */ @@ -181,19 +195,8 @@ final class S3ClientSettings { /** The protocol to use to talk to s3. Defaults to https. */ final Protocol protocol; - /** An optional proxy host that requests to s3 should be made through. */ - final String proxyHost; - - /** The port number the proxy host should be connected on. */ - final int proxyPort; - - // these should be "secure" yet the api for the s3 client only takes String, so storing them - // as SecureString here won't really help with anything - /** An optional username for the proxy host, for basic authentication. */ - final String proxyUsername; - - /** An optional password for the proxy host, for basic authentication. */ - final String proxyPassword; + /** An optional proxy settings that requests to s3 should be made through. */ + final ProxySettings proxySettings; /** The read timeout for the s3 client. */ final int readTimeoutMillis; @@ -220,25 +223,18 @@ private S3ClientSettings( S3BasicCredentials credentials, String endpoint, Protocol protocol, - String proxyHost, - int proxyPort, - String proxyUsername, - String proxyPassword, int readTimeoutMillis, int maxRetries, boolean throttleRetries, boolean pathStyleAccess, boolean disableChunkedEncoding, String region, - String signerOverride + String signerOverride, + ProxySettings proxySettings ) { this.credentials = credentials; this.endpoint = endpoint; this.protocol = protocol; - this.proxyHost = proxyHost; - this.proxyPort = proxyPort; - this.proxyUsername = proxyUsername; - this.proxyPassword = proxyPassword; this.readTimeoutMillis = readTimeoutMillis; this.maxRetries = maxRetries; this.throttleRetries = throttleRetries; @@ -246,6 +242,7 @@ private S3ClientSettings( this.disableChunkedEncoding = disableChunkedEncoding; this.region = region; this.signerOverride = signerOverride; + this.proxySettings = proxySettings; } /** @@ -263,8 +260,10 @@ S3ClientSettings refine(Settings repositorySettings) { final String newEndpoint = getRepoSettingOrDefault(ENDPOINT_SETTING, normalizedSettings, endpoint); final Protocol newProtocol = getRepoSettingOrDefault(PROTOCOL_SETTING, normalizedSettings, protocol); - final String newProxyHost = getRepoSettingOrDefault(PROXY_HOST_SETTING, normalizedSettings, proxyHost); - final int newProxyPort = getRepoSettingOrDefault(PROXY_PORT_SETTING, normalizedSettings, proxyPort); + + final String newProxyHost = getRepoSettingOrDefault(PROXY_HOST_SETTING, normalizedSettings, proxySettings.getHostName()); + final int newProxyPort = getRepoSettingOrDefault(PROXY_PORT_SETTING, normalizedSettings, proxySettings.getPort()); + final int newReadTimeoutMillis = Math.toIntExact( getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis() ); @@ -286,8 +285,8 @@ S3ClientSettings refine(Settings repositorySettings) { final String newSignerOverride = getRepoSettingOrDefault(SIGNER_OVERRIDE, normalizedSettings, signerOverride); if (Objects.equals(endpoint, newEndpoint) && protocol == newProtocol - && Objects.equals(proxyHost, newProxyHost) - && proxyPort == newProxyPort + && Objects.equals(proxySettings.getHostName(), newProxyHost) + && proxySettings.getPort() == newProxyPort && newReadTimeoutMillis == readTimeoutMillis && maxRetries == newMaxRetries && newThrottleRetries == throttleRetries @@ -298,21 +297,20 @@ S3ClientSettings refine(Settings repositorySettings) { && Objects.equals(signerOverride, newSignerOverride)) { return this; } + + validateInetAddressFor(newProxyHost); return new S3ClientSettings( newCredentials, newEndpoint, newProtocol, - newProxyHost, - newProxyPort, - proxyUsername, - proxyPassword, newReadTimeoutMillis, newMaxRetries, newThrottleRetries, newPathStyleAccess, newDisableChunkedEncoding, newRegion, - newSignerOverride + newSignerOverride, + proxySettings.recreateWithNewHostAndPort(newProxyHost, newProxyPort) ); } @@ -401,27 +399,69 @@ private static S3BasicCredentials loadCredentials(Settings settings, String clie // pkg private for tests /** Parse settings for a single client. */ static S3ClientSettings getClientSettings(final Settings settings, final String clientName) { - try ( - SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING); - SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING) - ) { - return new S3ClientSettings( - S3ClientSettings.loadCredentials(settings, clientName), - getConfigValue(settings, clientName, ENDPOINT_SETTING), - getConfigValue(settings, clientName, PROTOCOL_SETTING), - getConfigValue(settings, clientName, PROXY_HOST_SETTING), - getConfigValue(settings, clientName, PROXY_PORT_SETTING), - proxyUsername.toString(), - proxyPassword.toString(), - Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), - getConfigValue(settings, clientName, MAX_RETRIES_SETTING), - getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), - getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS), - getConfigValue(settings, clientName, DISABLE_CHUNKED_ENCODING), - getConfigValue(settings, clientName, REGION), - getConfigValue(settings, clientName, SIGNER_OVERRIDE) + final Protocol awsProtocol = getConfigValue(settings, clientName, PROTOCOL_SETTING); + return new S3ClientSettings( + S3ClientSettings.loadCredentials(settings, clientName), + getConfigValue(settings, clientName, ENDPOINT_SETTING), + awsProtocol, + Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), + getConfigValue(settings, clientName, MAX_RETRIES_SETTING), + getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), + getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS), + getConfigValue(settings, clientName, DISABLE_CHUNKED_ENCODING), + getConfigValue(settings, clientName, REGION), + getConfigValue(settings, clientName, SIGNER_OVERRIDE), + validateAndCreateProxySettings(settings, clientName, awsProtocol) + ); + } + + static ProxySettings validateAndCreateProxySettings(final Settings settings, final String clientName, final Protocol awsProtocol) { + ProxySettings.ProxyType proxyType = getConfigValue(settings, clientName, PROXY_TYPE_SETTING); + final String proxyHost = getConfigValue(settings, clientName, PROXY_HOST_SETTING); + final int proxyPort = getConfigValue(settings, clientName, PROXY_PORT_SETTING); + final SecureString proxyUserName = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING); + final SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING); + if (awsProtocol != Protocol.HTTPS && proxyType == ProxySettings.ProxyType.DIRECT && Strings.hasText(proxyHost)) { + // This is backward compatibility for the current behaviour. + // The default value for Protocol settings is HTTPS, + // The expectation of ex-developers that protocol is the same as the proxy protocol + // which is a separate setting for AWS SDK. + // In this case, proxy type should be the same as a protocol, + // when proxy host and port have been set + proxyType = ProxySettings.ProxyType.valueOf(awsProtocol.name()); + deprecationLogger.deprecate( + PROTOCOL_SETTING.getConcreteSettingForNamespace(clientName).getKey(), + "Using of " + + PROTOCOL_SETTING.getConcreteSettingForNamespace(clientName).getKey() + + " as proxy type is deprecated and will be removed in future releases. Please use " + + PROXY_TYPE_SETTING.getConcreteSettingForNamespace(clientName).getKey() + + " instead to specify proxy type." ); } + // Validate proxy settings + if (proxyType == ProxySettings.ProxyType.DIRECT + && (proxyPort != 80 || Strings.hasText(proxyHost) || Strings.hasText(proxyUserName) || Strings.hasText(proxyPassword))) { + throw new SettingsException("S3 proxy port or host or username or password have been set but proxy type is not defined."); + } + if (proxyType != ProxySettings.ProxyType.DIRECT && Strings.isEmpty(proxyHost)) { + throw new SettingsException("S3 proxy type has been set but proxy host or port is not defined."); + } + if (proxyType == ProxySettings.ProxyType.DIRECT) { + return ProxySettings.NO_PROXY_SETTINGS; + } + if (awsProtocol == Protocol.HTTP && proxyType == ProxySettings.ProxyType.SOCKS) { + throw new SettingsException("SOCKS proxy is not supported for HTTP protocol"); + } + validateInetAddressFor(proxyHost); + return new ProxySettings(proxyType, proxyHost, proxyPort, proxyUserName.toString(), proxyPassword.toString()); + } + + static void validateInetAddressFor(final String proxyHost) { + try { + InetAddress.getByName(proxyHost); + } catch (final UnknownHostException e) { + throw new SettingsException("S3 proxy host is unknown.", e); + } } @Override @@ -433,16 +473,13 @@ public boolean equals(final Object o) { return false; } final S3ClientSettings that = (S3ClientSettings) o; - return proxyPort == that.proxyPort - && readTimeoutMillis == that.readTimeoutMillis + return readTimeoutMillis == that.readTimeoutMillis && maxRetries == that.maxRetries && throttleRetries == that.throttleRetries && Objects.equals(credentials, that.credentials) && Objects.equals(endpoint, that.endpoint) && protocol == that.protocol - && Objects.equals(proxyHost, that.proxyHost) - && Objects.equals(proxyUsername, that.proxyUsername) - && Objects.equals(proxyPassword, that.proxyPassword) + && proxySettings.equals(that.proxySettings) && Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding) && Objects.equals(region, that.region) && Objects.equals(signerOverride, that.signerOverride); @@ -454,10 +491,7 @@ public int hashCode() { credentials, endpoint, protocol, - proxyHost, - proxyPort, - proxyUsername, - proxyPassword, + proxySettings, readTimeoutMillis, maxRetries, throttleRetries, diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index 1f5cb2a752eef..3ce19378ac05c 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -39,10 +39,16 @@ import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.http.IdleConnectionReaper; +import com.amazonaws.http.SystemPropertyTlsKeyManagersProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.internal.SdkSSLContext; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.internal.Constants; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.protocol.HttpContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.cluster.metadata.RepositoryMetadata; @@ -50,7 +56,15 @@ import org.opensearch.common.collect.MapBuilder; import org.opensearch.common.settings.Settings; +import javax.net.ssl.SSLContext; import java.io.Closeable; +import java.io.IOException; +import java.net.Authenticator; +import java.net.InetSocketAddress; +import java.net.PasswordAuthentication; +import java.net.Proxy; +import java.net.Socket; +import java.security.SecureRandom; import java.util.Map; import static java.util.Collections.emptyMap; @@ -189,12 +203,32 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { clientConfiguration.setResponseMetadataCacheSize(0); clientConfiguration.setProtocol(clientSettings.protocol); - if (Strings.hasText(clientSettings.proxyHost)) { - // TODO: remove this leniency, these settings should exist together and be validated - clientConfiguration.setProxyHost(clientSettings.proxyHost); - clientConfiguration.setProxyPort(clientSettings.proxyPort); - clientConfiguration.setProxyUsername(clientSettings.proxyUsername); - clientConfiguration.setProxyPassword(clientSettings.proxyPassword); + if (clientSettings.proxySettings != ProxySettings.NO_PROXY_SETTINGS) { + if (clientSettings.proxySettings.getType() == ProxySettings.ProxyType.SOCKS) { + SocketAccess.doPrivilegedVoid(() -> { + if (clientSettings.proxySettings.isAuthenticated()) { + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication( + clientSettings.proxySettings.getUsername(), + clientSettings.proxySettings.getPassword().toCharArray() + ); + } + }); + } + clientConfiguration.getApacheHttpClientConfig() + .setSslSocketFactory(createSocksSslConnectionSocketFactory(clientSettings.proxySettings.getAddress())); + }); + } else { + if (clientSettings.proxySettings.getType() != ProxySettings.ProxyType.DIRECT) { + clientConfiguration.setProxyProtocol(clientSettings.proxySettings.getType().toProtocol()); + } + clientConfiguration.setProxyHost(clientSettings.proxySettings.getHostName()); + clientConfiguration.setProxyPort(clientSettings.proxySettings.getPort()); + clientConfiguration.setProxyUsername(clientSettings.proxySettings.getUsername()); + clientConfiguration.setProxyPassword(clientSettings.proxySettings.getPassword()); + } } if (Strings.hasLength(clientSettings.signerOverride)) { @@ -208,6 +242,20 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { return clientConfiguration; } + private static SSLConnectionSocketFactory createSocksSslConnectionSocketFactory(final InetSocketAddress address) { + // This part was taken from AWS settings + final SSLContext sslCtx = SdkSSLContext.getPreferredSSLContext( + new SystemPropertyTlsKeyManagersProvider().getKeyManagers(), + new SecureRandom() + ); + return new SdkTLSSocketFactory(sslCtx, new DefaultHostnameVerifier()) { + @Override + public Socket createSocket(final HttpContext ctx) throws IOException { + return new Socket(new Proxy(Proxy.Type.SOCKS, address)); + } + }; + } + // pkg private for tests static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) { final S3BasicCredentials credentials = clientSettings.credentials; diff --git a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy index 8c9b91418ed53..f6c154bb3b14d 100644 --- a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy @@ -51,6 +51,9 @@ grant { // s3 client opens socket connections for to access repository permission java.net.SocketPermission "*", "connect"; + // s3 client set Authenticator for proxy username/password + permission java.net.NetPermission "setDefaultAuthenticator"; + // only for tests : org.opensearch.repositories.s3.S3RepositoryPlugin permission java.util.PropertyPermission "opensearch.allow_insecure_settings", "read,write"; }; diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java index 0f1bfdf7b7d6b..38d9ebf337731 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java @@ -36,17 +36,19 @@ import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; - import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; +import java.io.IOException; import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.opensearch.repositories.s3.S3ClientSettings.PROTOCOL_SETTING; +import static org.opensearch.repositories.s3.S3ClientSettings.PROXY_TYPE_SETTING; public class AwsS3ServiceImplTests extends OpenSearchTestCase { @@ -140,14 +142,14 @@ public void testAWSConfigurationWithAwsSettings() { final Settings settings = Settings.builder() .setSecureSettings(secureSettings) .put("s3.client.default.protocol", "http") - .put("s3.client.default.proxy.host", "aws_proxy_host") + .put("s3.client.default.proxy.host", "127.0.0.10") .put("s3.client.default.proxy.port", 8080) .put("s3.client.default.read_timeout", "10s") .build(); launchAWSConfigurationTest( settings, Protocol.HTTP, - "aws_proxy_host", + "127.0.0.10", 8080, "aws_proxy_username", "aws_proxy_password", @@ -155,6 +157,60 @@ public void testAWSConfigurationWithAwsSettings() { ClientConfiguration.DEFAULT_THROTTLE_RETRIES, 10000 ); + assertWarnings( + "Using of " + + PROTOCOL_SETTING.getConcreteSettingForNamespace("default").getKey() + + " as proxy type is deprecated and will be removed in future releases. Please use " + + PROXY_TYPE_SETTING.getConcreteSettingForNamespace("default").getKey() + + " instead to specify proxy type." + ); + } + + public void testProxyTypeOverrideProtocolSettings() { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username"); + secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password"); + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("s3.client.default.protocol", "http") + .put("s3.client.default.proxy.type", "https") + .put("s3.client.default.proxy.host", "127.0.0.10") + .put("s3.client.default.proxy.port", 8080) + .put("s3.client.default.read_timeout", "10s") + .build(); + launchAWSConfigurationTest( + settings, + Protocol.HTTP, + "127.0.0.10", + 8080, + "aws_proxy_username", + "aws_proxy_password", + 3, + ClientConfiguration.DEFAULT_THROTTLE_RETRIES, + 10000 + ); + } + + public void testSocksProxyConfiguration() throws IOException { + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username"); + secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password"); + final Settings settings = Settings.builder() + .setSecureSettings(secureSettings) + .put("s3.client.default.proxy.type", "socks") + .put("s3.client.default.proxy.host", "127.0.0.10") + .put("s3.client.default.proxy.port", 8080) + .put("s3.client.default.read_timeout", "10s") + .build(); + + final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default"); + final ClientConfiguration configuration = S3Service.buildConfiguration(clientSettings); + + assertEquals(Protocol.HTTPS, configuration.getProtocol()); + assertEquals(Protocol.HTTP, configuration.getProxyProtocol()); // default value in SDK + assertEquals(-1, configuration.getProxyPort()); + assertNull(configuration.getProxyUsername()); + assertNull(configuration.getProxyPassword()); } public void testRepositoryMaxRetries() { diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java index ea0b554df880e..462ed5377ff9a 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java @@ -37,8 +37,12 @@ import com.amazonaws.services.s3.AmazonS3Client; import org.opensearch.common.settings.MockSecureSettings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsException; import org.opensearch.test.OpenSearchTestCase; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.contains; @@ -55,10 +59,7 @@ public void testThereIsADefaultClientByDefault() { assertThat(defaultSettings.credentials, nullValue()); assertThat(defaultSettings.endpoint, is(emptyString())); assertThat(defaultSettings.protocol, is(Protocol.HTTPS)); - assertThat(defaultSettings.proxyHost, is(emptyString())); - assertThat(defaultSettings.proxyPort, is(80)); - assertThat(defaultSettings.proxyUsername, is(emptyString())); - assertThat(defaultSettings.proxyPassword, is(emptyString())); + assertThat(defaultSettings.proxySettings, is(ProxySettings.NO_PROXY_SETTINGS)); assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT)); assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES)); @@ -215,4 +216,77 @@ public void testSignerOverrideCanBeSet() { ClientConfiguration configuration = S3Service.buildConfiguration(settings.get("other")); assertThat(configuration.getSignerOverride(), is(signerOverride)); } + + public void testSetProxySettings() throws Exception { + final int port = randomIntBetween(10, 1080); + final String userName = randomAlphaOfLength(10); + final String password = randomAlphaOfLength(10); + final String proxyType = randomFrom("http", "https", "socks"); + + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.proxy.username", userName); + secureSettings.setString("s3.client.default.proxy.password", password); + + final Settings settings = Settings.builder() + .put("s3.client.default.proxy.type", proxyType) + .put("s3.client.default.proxy.host", randomFrom("127.0.0.10")) + .put("s3.client.default.proxy.port", randomFrom(port)) + .setSecureSettings(secureSettings) + .build(); + + final S3ClientSettings s3ClientSettings = S3ClientSettings.load(settings).get("default"); + + assertEquals(ProxySettings.ProxyType.valueOf(proxyType.toUpperCase(Locale.ROOT)), s3ClientSettings.proxySettings.getType()); + assertEquals(new InetSocketAddress(InetAddress.getByName("127.0.0.10"), port), s3ClientSettings.proxySettings.getAddress()); + assertEquals(userName, s3ClientSettings.proxySettings.getUsername()); + assertEquals(password, s3ClientSettings.proxySettings.getPassword()); + } + + public void testProxyWrongHost() { + final Settings settings = Settings.builder() + .put("s3.client.default.proxy.type", randomFrom("socks", "http")) + .put("s3.client.default.proxy.host", "thisisnotavalidhostorwehavebeensuperunlucky") + .put("s3.client.default.proxy.port", 8080) + .build(); + final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + assertEquals("S3 proxy host is unknown.", e.getMessage()); + } + + public void testProxyTypeNotSet() { + final Settings hostPortSettings = Settings.builder() + .put("s3.client.default.proxy.host", "127.0.0.1") + .put("s3.client.default.proxy.port", 8080) + .build(); + + SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(hostPortSettings)); + assertEquals("S3 proxy port or host or username or password have been set but proxy type is not defined.", e.getMessage()); + + final MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("s3.client.default.proxy.username", "aaaa"); + secureSettings.setString("s3.client.default.proxy.password", "bbbb"); + final Settings usernamePasswordSettings = Settings.builder().setSecureSettings(secureSettings).build(); + + e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(usernamePasswordSettings)); + assertEquals("S3 proxy port or host or username or password have been set but proxy type is not defined.", e.getMessage()); + } + + public void testProxyHostNotSet() { + final Settings settings = Settings.builder() + .put("s3.client.default.proxy.port", 8080) + .put("s3.client.default.proxy.type", randomFrom("socks", "http", "https")) + .build(); + final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + assertEquals("S3 proxy type has been set but proxy host or port is not defined.", e.getMessage()); + } + + public void testSocksDoesNotSupportForHttpProtocol() { + final Settings settings = Settings.builder() + .put("s3.client.default.proxy.host", "127.0.0.1") + .put("s3.client.default.proxy.port", 8080) + .put("s3.client.default.protocol", "http") + .put("s3.client.default.proxy.type", "socks") + .build(); + expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings)); + } + }