Skip to content

Commit

Permalink
Remove dependency on Wiremock
Browse files Browse the repository at this point in the history 
Signed-off-by: Thomas Farr <tsfarr@amazon.com>
  • Loading branch information
@Xtansia
Xtansia committed Nov 5, 2024
1 parent 8da02d4 commit d837e5c
Show file tree
Hide file tree
Showing 5 changed files with 472 additions and 102 deletions.
27 changes: 20 additions & 7 deletions java-client/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,12 @@ sourceSets {
}

tasks.withType<ProcessResources> {
expand(
"version" to version,
"git_revision" to (if (rootProject.extra.has("gitHashFull")) rootProject.extra["gitHashFull"] else "unknown")
)
filesMatching("**/*.properties") {
expand(
"version" to version,
"git_revision" to (if (rootProject.extra.has("gitHashFull")) rootProject.extra["gitHashFull"] else "unknown")
)
}
}

tasks.withType<Javadoc>().configureEach{
Expand Down Expand Up @@ -137,6 +139,20 @@ tasks.build {
dependsOn("spotlessJavaCheck")
}

tasks.compileTestJava {
options.compilerArgs.addAll(listOf(
"--add-exports=java.base/sun.security.util=ALL-UNNAMED",
"--add-exports=java.base/sun.security.x509=ALL-UNNAMED"
))
}

tasks.withType<Test> {
jvmArgs(
"--add-exports=java.base/sun.security.util=ALL-UNNAMED",
"--add-exports=java.base/sun.security.x509=ALL-UNNAMED"
)
}

tasks.test {
systemProperty("tests.security.manager", "false")

Expand Down Expand Up @@ -239,9 +255,6 @@ dependencies {
testImplementation("junit", "junit" , "4.13.2") {
exclude(group = "org.hamcrest")
}

// Apache 2.0
testImplementation("org.wiremock", "wiremock", "3.9.2")
}

licenseReport {
Expand Down
1 change: 0 additions & 1 deletion java-client/src/main/java/org/opensearch/client/transport/aws/AwsSdk2Transport.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ public class AwsSdk2Transport implements OpenSearchTransport {
private final String signingServiceName;
private final Region signingRegion;
private final JsonpMapper defaultMapper;
@Nonnull
private final AwsSdk2TransportOptions transportOptions;

/**
Expand Down
207 changes: 113 additions & 94 deletions java-client/src/test/java/org/opensearch/client/transport/aws/AwsSdk2TransportTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,39 +8,48 @@

package org.opensearch.client.transport.aws;

import static com.github.tomakehurst.wiremock.client.WireMock.any;
import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl;
import static com.github.tomakehurst.wiremock.client.WireMock.delete;
import static com.github.tomakehurst.wiremock.client.WireMock.okJson;
import static com.github.tomakehurst.wiremock.client.WireMock.put;
import static com.github.tomakehurst.wiremock.client.WireMock.serviceUnavailable;
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathTemplate;
import static com.github.tomakehurst.wiremock.common.ContentTypes.APPLICATION_JSON;
import static com.github.tomakehurst.wiremock.common.ContentTypes.CONTENT_LENGTH;
import static com.github.tomakehurst.wiremock.common.ContentTypes.CONTENT_TYPE;
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
import static org.apache.hc.core5.http.ContentType.APPLICATION_JSON;
import static org.apache.hc.core5.http.HttpHeaders.CONTENT_LENGTH;
import static org.apache.hc.core5.http.HttpHeaders.CONTENT_TYPE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;

import com.github.tomakehurst.wiremock.http.RequestMethod;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import com.github.tomakehurst.wiremock.verification.LoggedRequest;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.stream.Collectors;
import org.apache.hc.core5.function.Supplier;
import org.apache.hc.core5.http.HttpRequest;
import org.apache.hc.core5.http.ProtocolException;
import org.apache.hc.core5.http.URIScheme;
import org.apache.hc.core5.http.impl.BasicEntityDetails;
import org.apache.hc.core5.http.impl.bootstrap.AsyncServerBootstrap;
import org.apache.hc.core5.http.impl.bootstrap.HttpAsyncServer;
import org.apache.hc.core5.http.impl.routing.RequestRouter;
import org.apache.hc.core5.http.message.BasicHttpResponse;
import org.apache.hc.core5.http.nio.AsyncServerExchangeHandler;
import org.apache.hc.core5.http.nio.ssl.BasicClientTlsStrategy;
import org.apache.hc.core5.io.CloseMode;
import org.apache.hc.core5.reactive.ReactiveServerExchangeHandler;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.transport.util.FunnellingHttpsProxy;
import org.opensearch.client.transport.util.GeneratedCertificateSSLContext;
import reactor.core.publisher.Flux;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.http.SdkHttpClient;
import software.amazon.awssdk.http.SdkHttpConfigurationOption;
Expand All @@ -55,9 +64,12 @@
@RunWith(Parameterized.class)
public class AwsSdk2TransportTests {
private static final Region TEST_REGION = Region.AP_SOUTHEAST_2;
private static final String TEST_INDEX = "sample-index1";

@Rule
public final WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort().enableBrowserProxying(true));
private HttpAsyncServer server;
private FunnellingHttpsProxy proxy;

private final ConcurrentLinkedQueue<HttpRequest> receivedRequests = new ConcurrentLinkedQueue<>();
private final SdkHttpClientType sdkHttpClientType;
private final String serviceName;
private final String serviceHostName;
Expand All @@ -84,59 +96,99 @@ public enum SdkHttpClientType {
}

@Before
public void setup() {
stubFor(any(anyUrl()).atPriority(10).willReturn(serviceUnavailable()));

stubFor(
put(urlPathTemplate("/{index}")).atPriority(1)
.willReturn(
okJson("{\"acknowledged\": true,\"shards_acknowledged\": true,\"index\": \"{{request.path.index}}\"}").withTransformers(
"response-template"
public void setup() throws Exception {
server = AsyncServerBootstrap.bootstrap()
.setRequestRouter(
RequestRouter.<Supplier<AsyncServerExchangeHandler>>builder()
.addRoute(
RequestRouter.LOCAL_AUTHORITY,
"/" + TEST_INDEX,
hardcodedJsonHandler(
"PUT",
"{\"acknowledged\": true,\"shards_acknowledged\": true,\"index\": \"" + TEST_INDEX + "\"}"
)
)
)
);

stubFor(delete(urlPathEqualTo("/_search/scroll")).atPriority(1).willReturn(okJson("{\"succeeded\": true,\"num_freed\": 1}")));
.addRoute(
RequestRouter.LOCAL_AUTHORITY,
"/_search/scroll",
hardcodedJsonHandler("DELETE", "{\"succeeded\": true,\"num_freed\": 1}")
)
.addRoute(
RequestRouter.LOCAL_AUTHORITY,
"/_search/point_in_time",
hardcodedJsonHandler("DELETE", "{\"pits\": [{\"pit_id\": \"pit1\", \"successful\": true}]}")
)
.resolveAuthority(RequestRouter.LOCAL_AUTHORITY_RESOLVER)
.build()
)
.setTlsStrategy(new BasicClientTlsStrategy(GeneratedCertificateSSLContext.generate()))
.create();
server.start();
var serverAddress = (InetSocketAddress) server.listen(new InetSocketAddress(0), URIScheme.HTTPS).get().getAddress();
proxy = new FunnellingHttpsProxy(serverAddress.getPort());
}

stubFor(
delete(urlPathEqualTo("/_search/point_in_time")).atPriority(1)
.willReturn(okJson("{\"pits\": [{\"pit_id\": \"pit1\", \"successful\": true}]}"))
private Supplier<AsyncServerExchangeHandler> hardcodedJsonHandler(String method, String json) {
var jsonBytes = json.getBytes(StandardCharsets.UTF_8);
return () -> new ReactiveServerExchangeHandler(
(request, entityDetails, responseChannel, context, requestBody, responseBodyFuture) -> {
receivedRequests.add(request);
if (!request.getMethod().equals(method)) {
responseChannel.sendResponse(new BasicHttpResponse(405), null, context);
return;
}
responseChannel.sendResponse(
new BasicHttpResponse(200),
new BasicEntityDetails(jsonBytes.length, APPLICATION_JSON),
context
);
responseBodyFuture.execute(Flux.just(ByteBuffer.wrap(jsonBytes)));
}
);
}

private OpenSearchClient getTestClient() throws Exception {
AwsSdk2TransportOptions options = AwsSdk2TransportOptions.builder()
@After
public void teardown() {
server.close(CloseMode.IMMEDIATE);
server = null;
proxy.close();
proxy = null;
receivedRequests.clear();
}

private OpenSearchClient getTestClient() throws URISyntaxException {
var options = AwsSdk2TransportOptions.builder()
.setCredentials(() -> AwsBasicCredentials.builder().accessKeyId("test-access-key").secretAccessKey("test-secret-key").build())
.setSigningClock(Clock.fixed(Instant.ofEpochSecond(1673626117), ZoneId.of("UTC"))) // 2023-01-13 16:08:37 +0000
.setResponseCompression(false)
.build();

AttributeMap sdkConfig = AttributeMap.builder().put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true).build();
var sdkConfig = AttributeMap.builder().put(SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES, true).build();

SdkHttpClient sdkHttpClient = null;
SdkAsyncHttpClient sdkAsyncHttpClient = null;
switch (sdkHttpClientType) {
case AWS_CRT:
sdkHttpClient = AwsCrtHttpClient.builder()
.proxyConfiguration(p -> p.scheme("http").host("localhost").port(wireMockRule.port()))
.proxyConfiguration(p -> p.scheme("http").host("localhost").port(proxy.getPort()))
.buildWithDefaults(sdkConfig);
break;
case AWS_CRT_ASYNC:
sdkAsyncHttpClient = AwsCrtAsyncHttpClient.builder()
.proxyConfiguration(p -> p.scheme("http").host("localhost").port(wireMockRule.port()))
.proxyConfiguration(p -> p.scheme("http").host("localhost").port(proxy.getPort()))
.buildWithDefaults(sdkConfig);
break;
case APACHE:
var proxyConfig = software.amazon.awssdk.http.apache.ProxyConfiguration.builder()
.endpoint(new URI("http://localhost:" + wireMockRule.port()))
.endpoint(new URI("http://localhost:" + proxy.getPort()))
.build();
sdkHttpClient = ApacheHttpClient.builder().proxyConfiguration(proxyConfig).buildWithDefaults(sdkConfig);
break;
case NETTY_NIO_ASYNC:
var nettyProxyConfig = software.amazon.awssdk.http.nio.netty.ProxyConfiguration.builder()
.scheme("http")
.host("localhost")
.port(wireMockRule.port())
.port(proxy.getPort())
.build();
sdkAsyncHttpClient = NettyNioAsyncHttpClient.builder().proxyConfiguration(nettyProxyConfig).buildWithDefaults(sdkConfig);
break;
Expand All @@ -153,12 +205,6 @@ private OpenSearchClient getTestClient() throws Exception {
return new OpenSearchClient(transport);
}

private LoggedRequest getReceivedRequest() {
var serveEvents = wireMockRule.getAllServeEvents();
assertEquals(1, serveEvents.size());
return serveEvents.get(0).getRequest();
}

@Test
public void testSigV4PutIndex() throws Exception {
String expectedSignature = null;
Expand Down Expand Up @@ -186,21 +232,7 @@ public void testSigV4PutIndex() throws Exception {
assertEquals("sample-index1", resp.index());
assertEquals(Boolean.TRUE, resp.acknowledged());

var req = getReceivedRequest();

assertEquals(RequestMethod.PUT, req.getMethod());
assertEquals(APPLICATION_JSON, req.getHeader(CONTENT_TYPE));
assertEquals("156", req.getHeader(CONTENT_LENGTH));
assertEquals(serviceHostName, req.getHeader("Host"));
assertEquals("20230113T160837Z", req.getHeader("x-amz-date"));
assertEquals("381bb92a04d397cab611362eb3ac3e075db11ac08272d64763de2279e2b5604d", req.getHeader("x-amz-content-sha256"));
assertEquals(
"AWS4-HMAC-SHA256 Credential=test-access-key/20230113/ap-southeast-2/"
+ serviceName
+ "/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature="
+ expectedSignature,
req.getHeader("Authorization")
);
assertSigV4Request("PUT", 156, "381bb92a04d397cab611362eb3ac3e075db11ac08272d64763de2279e2b5604d", expectedSignature);
}

@Test
Expand All @@ -222,27 +254,7 @@ public void testSigV4ClearScroll() throws Exception {

client.clearScroll();

var req = getReceivedRequest();

assertEquals(RequestMethod.DELETE, req.getMethod());
assertEquals(APPLICATION_JSON, req.getHeader(CONTENT_TYPE));
var contentLength = req.getHeader(CONTENT_LENGTH);
if (sdkHttpClientType != SdkHttpClientType.APACHE) {
assertEquals("2", contentLength);
} else {
// Apache client does not set content-length for DELETE requests
assertNull(contentLength);
}
assertEquals(serviceHostName, req.getHeader("Host"));
assertEquals("20230113T160837Z", req.getHeader("x-amz-date"));
assertEquals("44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", req.getHeader("x-amz-content-sha256"));
assertEquals(
"AWS4-HMAC-SHA256 Credential=test-access-key/20230113/ap-southeast-2/"
+ serviceName
+ "/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature="
+ expectedSignature,
req.getHeader("Authorization")
);
assertSigV4Request("DELETE", 2, "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", expectedSignature);
}

@Test
Expand All @@ -264,26 +276,33 @@ public void testSigV4DeletePit() throws Exception {

client.deletePit(d -> d.pitId(List.of("pit1")));

var req = getReceivedRequest();
assertSigV4Request("DELETE", 19, "daaa6af55a9cfe622f46de69ebc3b4df84703f320b839346b7fb4cf94bdbd766", expectedSignature);
}

assertEquals(RequestMethod.DELETE, req.getMethod());
assertEquals(APPLICATION_JSON, req.getHeader(CONTENT_TYPE));
var contentLength = req.getHeader(CONTENT_LENGTH);
if (sdkHttpClientType != SdkHttpClientType.APACHE) {
assertEquals("19", contentLength);
private void assertSigV4Request(String method, int contentLength, String contentSha256, String expectedSignature)
throws ProtocolException {
assertEquals(1, receivedRequests.size());
var req = receivedRequests.poll();
assertNotNull(req);

assertEquals(method, req.getMethod());
assertEquals(APPLICATION_JSON.getMimeType(), req.getHeader(CONTENT_TYPE).getValue());
var contentLengthHdr = req.getHeader(CONTENT_LENGTH);
if (sdkHttpClientType != SdkHttpClientType.APACHE || !"DELETE".equals(method)) {
assertEquals(String.valueOf(contentLength), contentLengthHdr.getValue());
} else {
// Apache client does not set content-length for DELETE requests
assertNull(contentLength);
assertNull(contentLengthHdr);
}
assertEquals(serviceHostName, req.getHeader("Host"));
assertEquals("20230113T160837Z", req.getHeader("x-amz-date"));
assertEquals("daaa6af55a9cfe622f46de69ebc3b4df84703f320b839346b7fb4cf94bdbd766", req.getHeader("x-amz-content-sha256"));
assertEquals(serviceHostName, req.getHeader("Host").getValue());
assertEquals("20230113T160837Z", req.getHeader("x-amz-date").getValue());
assertEquals(contentSha256, req.getHeader("x-amz-content-sha256").getValue());
assertEquals(
"AWS4-HMAC-SHA256 Credential=test-access-key/20230113/ap-southeast-2/"
+ serviceName
+ "/aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature="
+ expectedSignature,
req.getHeader("Authorization")
req.getHeader("Authorization").getValue()
);
}
}
Loading

0 comments on commit d837e5c

Please sign in to comment.