diff --git a/cve-suppressions.xml b/cve-suppressions.xml index 233c55b63..8afb49858 100644 --- a/cve-suppressions.xml +++ b/cve-suppressions.xml @@ -36,6 +36,9 @@ CVE-2021-4235 CVE-2022-45688 CVE-2020-8908 + + CVE-2023-20861 + CVE-2023-1370 diff --git a/riptide-core/pom.xml b/riptide-core/pom.xml index 349bd0ddf..9ce38d280 100644 --- a/riptide-core/pom.xml +++ b/riptide-core/pom.xml @@ -70,6 +70,12 @@ 2.3.1 test + + com.squareup.okhttp3 + mockwebserver + 4.10.0 + test + diff --git a/riptide-failsafe/pom.xml b/riptide-failsafe/pom.xml index cd7eb5d10..d4c1a54f5 100644 --- a/riptide-failsafe/pom.xml +++ b/riptide-failsafe/pom.xml @@ -47,6 +47,12 @@ com.github.rest-driver rest-client-driver + + + xmlunit + xmlunit + + org.zalando @@ -58,6 +64,92 @@ riptide-faults test + + com.squareup.okhttp3 + mockwebserver + 4.10.0 + test + + + + com.github.tomakehurst + wiremock + 3.0.0-beta-2 + test + + + jakarta.servlet + jakarta.servlet-api + 5.0.0 + test + + + org.eclipse.jetty + jetty-server + 11.0.14 + test + + + org.eclipse.jetty + jetty-util + 11.0.14 + test + + + org.eclipse.jetty + jetty-http + 11.0.14 + test + + + org.eclipse.jetty + jetty-io + 11.0.14 + test + + + org.eclipse.jetty + jetty-servlet + 11.0.14 + test + + + org.eclipse.jetty + jetty-servlets + 11.0.14 + test + + + org.eclipse.jetty + jetty-xml + 11.0.14 + test + + + org.eclipse.jetty + jetty-webapp + 11.0.14 + test + + + org.eclipse.jetty.http2 + http2-hpack + 11.0.14 + test + + + org.eclipse.jetty.http2 + http2-server + 11.0.14 + test + + + org.eclipse.jetty.http2 + http2-common + 11.0.14 + test + diff --git a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestClientDriver.java b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestClientDriver.java new file mode 100644 index 000000000..7c2c49b9f --- /dev/null +++ b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestClientDriver.java @@ -0,0 +1,177 @@ +package org.zalando.riptide.failsafe; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.restdriver.clientdriver.ClientDriver; +import com.github.restdriver.clientdriver.ClientDriverFactory; +import dev.failsafe.CircuitBreaker; +import dev.failsafe.RetryPolicy; +import lombok.SneakyThrows; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.test.web.client.MockRestServiceServer; +import org.zalando.riptide.Http; +import org.zalando.riptide.Plugin; +import org.zalando.riptide.RequestExecution; +import org.zalando.riptide.httpclient.ApacheClientHttpRequestFactory; +import org.zalando.riptide.idempotency.IdempotencyPredicate; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.time.Duration; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.github.restdriver.clientdriver.RestClientDriver.giveEmptyResponse; +import static com.github.restdriver.clientdriver.RestClientDriver.giveResponseAsBytes; +import static com.github.restdriver.clientdriver.RestClientDriver.onRequestTo; +import static com.google.common.io.Resources.getResource; +import static java.util.concurrent.Executors.newFixedThreadPool; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.stream.IntStream.range; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeout; +import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE; +import static org.springframework.http.HttpStatus.Series.SUCCESSFUL; +import static org.zalando.riptide.Attributes.RETRIES; +import static org.zalando.riptide.Bindings.anySeries; +import static org.zalando.riptide.Bindings.on; +import static org.zalando.riptide.Navigators.series; +import static org.zalando.riptide.Navigators.status; +import static org.zalando.riptide.PassRoute.pass; +import static org.zalando.riptide.Route.call; +import static org.zalando.riptide.failsafe.CheckedPredicateConverter.toCheckedPredicate; +import static org.zalando.riptide.failsafe.RetryRoute.retry; +import static org.zalando.riptide.faults.Predicates.alwaysTrue; +import static org.zalando.riptide.faults.TransientFaults.transientConnectionFaults; +import static org.zalando.riptide.faults.TransientFaults.transientSocketFaults; + +final class HttpMockTestClientDriver { + + private final ClientDriver driver = new ClientDriverFactory().createClientDriver(); + + private final CloseableHttpClient client = HttpClientBuilder.create() + .setDefaultRequestConfig(RequestConfig.custom() + .setSocketTimeout(500) + .build()) + .build(); + + private final AtomicInteger attempt = new AtomicInteger(); + + private final Http unit = Http.builder() + .executor(newFixedThreadPool(2)) // to allow for nested calls + .requestFactory(new ApacheClientHttpRequestFactory(client)) + .baseUrl(driver.getBaseUrl()) + .converter(createJsonConverter()) + .plugin(new Plugin() { + @Override + public RequestExecution aroundNetwork(final RequestExecution execution) { + return arguments -> { + arguments.getAttribute(RETRIES).ifPresent(attempt::set); + return execution.execute(arguments); + }; + } + }) + .plugin(new FailsafePlugin() + .withPolicy(new RetryRequestPolicy( + RetryPolicy.builder() + .handleIf(toCheckedPredicate(transientSocketFaults())) + .handle(RetryException.class) + .handleResultIf(this::isBadGateway) + .withDelay(Duration.ofMillis(500)) + .withMaxRetries(4) + .build()) + .withPredicate(new IdempotencyPredicate())) + .withPolicy(new RetryRequestPolicy( + RetryPolicy.builder() + .handleIf(toCheckedPredicate(transientConnectionFaults())) + .withDelay(Duration.ofMillis(500)) + .withMaxRetries(4) + .build()) + .withPredicate(alwaysTrue())) + .withPolicy(CircuitBreaker.builder() + .withFailureThreshold(5, 10) + .withSuccessThreshold(5) + .withDelay(Duration.ofMinutes(1)) + .build())) + .build(); + + @SneakyThrows + private boolean isBadGateway(@Nullable final ClientHttpResponse response) { + return response != null && response.getStatusCode() == HttpStatus.BAD_GATEWAY; + } + + @AfterEach + void tearDown() throws IOException { + driver.verify(); + client.close(); + } + + @SneakyThrows + @Test + void shouldRetrySuccessfully() { + driver.addExpectation(onRequestTo("/foo"), giveEmptyResponse().after(800, MILLISECONDS)); + driver.addExpectation(onRequestTo("/foo"), giveResponseAsBytes(getResource("contributors.json").openStream(), "application/json")); + + unit.get("/foo") + .call(pass()) + .join(); + } + + + @Test + void shouldRetryUnsuccessfully() { + range(0, 5).forEach(i -> + driver.addExpectation(onRequestTo("/bar"), giveEmptyResponse().after(800, MILLISECONDS))); + + final CompletionException exception = assertThrows(CompletionException.class, + unit.get("/bar").call(pass())::join); + + assertThat(exception.getCause(), is(instanceOf(SocketTimeoutException.class))); + } + + @Test + void shouldRetryExplicitly() { + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse().withStatus(503).withHeader("X-RateLimit-Reset", "1523486068")); + driver.addExpectation(onRequestTo("/baz"), giveEmptyResponse()); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass()), + anySeries().dispatch(status(), + on(SERVICE_UNAVAILABLE).call(retry()))) + .join(); + } + + @Test + void shouldAllowNestedCalls() { + driver.addExpectation(onRequestTo("/foo"), giveEmptyResponse()); + driver.addExpectation(onRequestTo("/bar"), giveEmptyResponse()); + + assertTimeout(Duration.ofSeconds(1), + unit.get("/foo") + .call(call(() -> unit.get("/bar").call(pass()).join()))::join); + } + + + private static MappingJackson2HttpMessageConverter createJsonConverter() { + final MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); + converter.setObjectMapper(createObjectMapper()); + return converter; + } + + private static ObjectMapper createObjectMapper() { + return new ObjectMapper().findAndRegisterModules() + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + } +} diff --git a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestOkHttp.java b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestOkHttp.java new file mode 100644 index 000000000..e4442b963 --- /dev/null +++ b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestOkHttp.java @@ -0,0 +1,209 @@ +package org.zalando.riptide.failsafe; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.failsafe.CircuitBreaker; +import dev.failsafe.RetryPolicy; +import io.micrometer.core.instrument.util.IOUtils; +import lombok.SneakyThrows; +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.zalando.riptide.Http; +import org.zalando.riptide.Plugin; +import org.zalando.riptide.RequestExecution; +import org.zalando.riptide.httpclient.ApacheClientHttpRequestFactory; +import org.zalando.riptide.idempotency.IdempotencyPredicate; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.time.Duration; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.google.common.io.Resources.getResource; +import static java.util.concurrent.Executors.newFixedThreadPool; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.stream.IntStream.range; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeout; +import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE; +import static org.springframework.http.HttpStatus.Series.SUCCESSFUL; +import static org.zalando.riptide.Attributes.RETRIES; +import static org.zalando.riptide.Bindings.anySeries; +import static org.zalando.riptide.Bindings.on; +import static org.zalando.riptide.Navigators.series; +import static org.zalando.riptide.Navigators.status; +import static org.zalando.riptide.PassRoute.pass; +import static org.zalando.riptide.Route.call; +import static org.zalando.riptide.failsafe.CheckedPredicateConverter.toCheckedPredicate; +import static org.zalando.riptide.failsafe.RetryRoute.retry; +import static org.zalando.riptide.faults.Predicates.alwaysTrue; +import static org.zalando.riptide.faults.TransientFaults.transientConnectionFaults; +import static org.zalando.riptide.faults.TransientFaults.transientSocketFaults; + +final class HttpMockTestOkHttp { + + MockWebServer server = new MockWebServer(); + + private final CloseableHttpClient client = HttpClientBuilder.create() + .setDefaultRequestConfig(RequestConfig.custom() + .setSocketTimeout(500) + .build()) + .build(); + + private final AtomicInteger attempt = new AtomicInteger(); + + private final Http unit = Http.builder() + .executor(newFixedThreadPool(2)) // to allow for nested calls + .requestFactory(new ApacheClientHttpRequestFactory(client)) + .baseUrl( "http://localhost:" + server.getPort() ) + .converter(createJsonConverter()) + .plugin(new Plugin() { + @Override + public RequestExecution aroundNetwork(final RequestExecution execution) { + return arguments -> { + arguments.getAttribute(RETRIES).ifPresent(attempt::set); + return execution.execute(arguments); + }; + } + }) + .plugin(new FailsafePlugin() + .withPolicy(new RetryRequestPolicy( + RetryPolicy.builder() + .handleIf(toCheckedPredicate(transientSocketFaults())) + .handle(RetryException.class) + .handleResultIf(this::isBadGateway) + .withDelay(Duration.ofMillis(500)) + .withMaxRetries(4) + .build()) + .withPredicate(new IdempotencyPredicate())) + .withPolicy(new RetryRequestPolicy( + RetryPolicy.builder() + .handleIf(toCheckedPredicate(transientConnectionFaults())) + .withDelay(Duration.ofMillis(500)) + .withMaxRetries(4) + .build()) + .withPredicate(alwaysTrue())) + .withPolicy(CircuitBreaker.builder() + .withFailureThreshold(5, 10) + .withSuccessThreshold(5) + .withDelay(Duration.ofMinutes(1)) + .build())) + .build(); + + @SneakyThrows + private boolean isBadGateway(@Nullable final ClientHttpResponse response) { + return response != null && response.getStatusCode() == HttpStatus.BAD_GATEWAY; + } + + @AfterEach + void tearDown() throws IOException { + //TODO: where server started? + client.close(); + server.shutdown(); + } + + @SneakyThrows + @Test + void shouldRetrySuccessfully() { + server.enqueue(new MockResponse().setResponseCode(200).setBody("Hello") + .setBodyDelay(800, MILLISECONDS) + ); + + server.enqueue(new MockResponse().setResponseCode(200) + .setBody(IOUtils.toString(getResource("contributors.json").openStream())) + .setHeader("Content-Type","application/json" )); + + unit.get("/foo") + .call(pass()) + .join(); + + //can't invoke verify so verifying in every test + assertEquals(2, server.getRequestCount()); + } + + + @Test + void shouldRetryUnsuccessfully() { + range(0, 5).forEach(i -> + server.enqueue(new MockResponse().setResponseCode(200).setBody("Hello").throttleBody(4, 800, MILLISECONDS) )); + + final CompletionException exception = assertThrows(CompletionException.class, + unit.get("/bar").call(pass())::join); + + assertThat(exception.getCause(), is(instanceOf(SocketTimeoutException.class))); + assertEquals(5, server.getRequestCount()); + } + + @Test + void shouldRetryExplicitly() { + + server.enqueue(new MockResponse().setResponseCode(503).setHeader("X-RateLimit-Reset", "1523486068")); + server.enqueue(new MockResponse().setResponseCode(204)); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass()), + anySeries().dispatch(status(), + on(SERVICE_UNAVAILABLE).call(retry()))) + .join(); + + assertEquals(2, server.getRequestCount()); + } + + @SneakyThrows + @Test + void shouldAllowNestedCalls() { + final Dispatcher dispatcher = new Dispatcher() { + + @Override + public MockResponse dispatch (RecordedRequest request) throws InterruptedException { + + //not needed here, just to demonstrate dispatch logic + switch (request.getPath()) { + case "/foo": + return new MockResponse().setResponseCode(204); + case "/bar": + return new MockResponse().setResponseCode(204); + } + return new MockResponse().setResponseCode(404); + } + }; + server.setDispatcher(dispatcher); + + assertTimeout(Duration.ofSeconds(1), + unit.get("/foo") + .call(call(() -> unit.get("/bar").call(pass()).join()))::join); + + assertEquals(2, server.getRequestCount()); + assertEquals("/foo", server.takeRequest().getPath()); + assertEquals("/bar", server.takeRequest().getPath()); + } + + + private static MappingJackson2HttpMessageConverter createJsonConverter() { + final MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); + converter.setObjectMapper(createObjectMapper()); + return converter; + } + + private static ObjectMapper createObjectMapper() { + return new ObjectMapper().findAndRegisterModules() + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + } +} diff --git a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestSpring.java b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestSpring.java new file mode 100644 index 000000000..1caaf04a5 --- /dev/null +++ b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestSpring.java @@ -0,0 +1,181 @@ +package org.zalando.riptide.failsafe; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.failsafe.CircuitBreaker; +import dev.failsafe.RetryPolicy; +import lombok.SneakyThrows; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.test.web.client.MockRestServiceServer; +import org.zalando.riptide.Http; +import org.zalando.riptide.Plugin; +import org.zalando.riptide.RequestExecution; +import org.zalando.riptide.idempotency.IdempotencyPredicate; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.time.Duration; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.google.common.io.Resources.getResource; +import static java.util.concurrent.Executors.newFixedThreadPool; +import static java.util.stream.IntStream.range; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeout; +import static org.springframework.http.HttpStatus.NO_CONTENT; +import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE; +import static org.springframework.http.HttpStatus.Series.SUCCESSFUL; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; +import static org.zalando.riptide.Attributes.RETRIES; +import static org.zalando.riptide.Bindings.anySeries; +import static org.zalando.riptide.Bindings.on; +import static org.zalando.riptide.Navigators.series; +import static org.zalando.riptide.Navigators.status; +import static org.zalando.riptide.PassRoute.pass; +import static org.zalando.riptide.Route.call; +import static org.zalando.riptide.failsafe.CheckedPredicateConverter.toCheckedPredicate; +import static org.zalando.riptide.failsafe.RetryRoute.retry; +import static org.zalando.riptide.faults.Predicates.alwaysTrue; +import static org.zalando.riptide.faults.TransientFaults.transientConnectionFaults; +import static org.zalando.riptide.faults.TransientFaults.transientSocketFaults; + +final class HttpMockTestSpring { + + private final MockSetup mockSetup = new MockSetup(); + private final MockRestServiceServer server = mockSetup.getServer(); + + private final AtomicInteger attempt = new AtomicInteger(); + + private final Http unit = mockSetup.getRestBuilder(newFixedThreadPool(2)) // to allow for nested calls + .converter(createJsonConverter()) + .plugin(new Plugin() { + @Override + public RequestExecution aroundNetwork(final RequestExecution execution) { + return arguments -> { + arguments.getAttribute(RETRIES).ifPresent(attempt::set); + return execution.execute(arguments); + }; + } + }) + .plugin(new FailsafePlugin() + .withPolicy(new RetryRequestPolicy( + RetryPolicy.builder() + .handleIf(toCheckedPredicate(transientSocketFaults())) + .handle(RetryException.class) + .handleResultIf(this::isBadGateway) + .withDelay(Duration.ofMillis(500)) + .withMaxRetries(4) + .build()) + .withPredicate(new IdempotencyPredicate())) + .withPolicy(new RetryRequestPolicy( + RetryPolicy.builder() + .handleIf(toCheckedPredicate(transientConnectionFaults())) + .withDelay(Duration.ofMillis(500)) + .withMaxRetries(4) + .build()) + .withPredicate(alwaysTrue())) + .withPolicy(CircuitBreaker.builder() + .withFailureThreshold(5, 10) + .withSuccessThreshold(5) + .withDelay(Duration.ofMinutes(1)) + .build())) + .build(); + + @SneakyThrows + private boolean isBadGateway(@Nullable final ClientHttpResponse response) { + return response != null && response.getStatusCode() == HttpStatus.BAD_GATEWAY; + } + + @AfterEach + void tearDown() throws IOException { + try { + server.verify(); + } finally { + server.reset(); + } + } + + @SneakyThrows + @Test + void shouldRetrySuccessfully() { + server.expect(requestTo(mockSetup.getBaseUrl() + "/foo")).andRespond((r) -> + { + // no way to pass own request factory, see MockRestServiceServer 290, + // so we need to throw SocketTimeoutException explicitly + throw new java.net.SocketTimeoutException(); + }); + server.expect(requestTo(mockSetup.getBaseUrl() + "/foo")) + .andRespond(withSuccess(getResource("contributors.json").openStream().readAllBytes(), MediaType.APPLICATION_JSON)); + + unit.get("/foo") + .call(pass()) + .join(); + } + + + @Test + void shouldRetryUnsuccessfully() { + range(0, 5).forEach(i -> + server.expect(requestTo(mockSetup.getBaseUrl() + "/bar")).andRespond((r) -> + { + throw new java.net.SocketTimeoutException(); + }) + ); + + final CompletionException exception = assertThrows(CompletionException.class, + unit.get("/bar").call(pass())::join); + + assertThat(exception.getCause(), is(instanceOf(SocketTimeoutException.class))); + } + + @Test + void shouldRetryExplicitly() { + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.set("X-RateLimit-Reset", "1523486068"); + + server.expect(requestTo(mockSetup.getBaseUrl() + "/baz")).andRespond(withStatus(SERVICE_UNAVAILABLE).headers(httpHeaders)); + server.expect(requestTo(mockSetup.getBaseUrl() + "/baz")).andRespond(withStatus(NO_CONTENT)); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass()), + anySeries().dispatch(status(), + on(SERVICE_UNAVAILABLE).call(retry()))) + .join(); + } + + @Test + void shouldAllowNestedCalls() { + server.expect(requestTo(mockSetup.getBaseUrl() + "/foo")).andRespond(withStatus(NO_CONTENT)); + server.expect(requestTo(mockSetup.getBaseUrl() + "/bar")).andRespond(withStatus(NO_CONTENT)); + + assertTimeout(Duration.ofSeconds(1), + unit.get("/foo") + .call(call(() -> unit.get("/bar").call(pass()).join()))::join); + } + + + private static MappingJackson2HttpMessageConverter createJsonConverter() { + final MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); + converter.setObjectMapper(createObjectMapper()); + return converter; + } + + private static ObjectMapper createObjectMapper() { + return new ObjectMapper().findAndRegisterModules() + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + } +} diff --git a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestWireMock.java b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestWireMock.java new file mode 100644 index 000000000..b591fb9e1 --- /dev/null +++ b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/HttpMockTestWireMock.java @@ -0,0 +1,248 @@ +package org.zalando.riptide.failsafe; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.http.Request; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.github.tomakehurst.wiremock.matching.MatchResult; +import com.github.tomakehurst.wiremock.matching.RequestMatcher; +import dev.failsafe.CircuitBreaker; +import dev.failsafe.RetryPolicy; +import lombok.SneakyThrows; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.zalando.riptide.Http; +import org.zalando.riptide.Plugin; +import org.zalando.riptide.RequestExecution; +import org.zalando.riptide.httpclient.ApacheClientHttpRequestFactory; +import org.zalando.riptide.idempotency.IdempotencyPredicate; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.time.Duration; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.ok; +import static com.github.tomakehurst.wiremock.client.WireMock.status; +import static com.github.tomakehurst.wiremock.matching.RequestPatternBuilder.newRequestPattern; +import static com.google.common.io.Resources.getResource; +import static java.util.concurrent.Executors.newFixedThreadPool; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeout; +import static org.springframework.http.HttpStatus.NO_CONTENT; +import static org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE; +import static org.springframework.http.HttpStatus.Series.SUCCESSFUL; +import static org.zalando.riptide.Attributes.RETRIES; +import static org.zalando.riptide.Bindings.anySeries; +import static org.zalando.riptide.Bindings.on; +import static org.zalando.riptide.Navigators.series; +import static org.zalando.riptide.PassRoute.pass; +import static org.zalando.riptide.Route.call; +import static org.zalando.riptide.failsafe.CheckedPredicateConverter.toCheckedPredicate; +import static org.zalando.riptide.failsafe.RetryRoute.retry; +import static org.zalando.riptide.faults.Predicates.alwaysTrue; +import static org.zalando.riptide.faults.TransientFaults.transientConnectionFaults; +import static org.zalando.riptide.faults.TransientFaults.transientSocketFaults; + +@WireMockTest +final class HttpMockTestWireMock { + + private final CloseableHttpClient client = HttpClientBuilder.create() + .setDefaultRequestConfig(RequestConfig.custom() + .setSocketTimeout(500) + .build()) + .build(); + + private final AtomicInteger attempt = new AtomicInteger(); + + private Http buildHttp(WireMockRuntimeInfo wmRuntimeInfo) { + return Http.builder() + .executor(newFixedThreadPool(2)) // to allow for nested calls + .requestFactory(new ApacheClientHttpRequestFactory(client)) + .baseUrl(wmRuntimeInfo.getHttpBaseUrl()) + .converter(createJsonConverter()) + .plugin(new Plugin() { + @Override + public RequestExecution aroundNetwork(final RequestExecution execution) { + return arguments -> { + arguments.getAttribute(RETRIES).ifPresent(attempt::set); + return execution.execute(arguments); + }; + } + }) + .plugin(new FailsafePlugin() + .withPolicy(new RetryRequestPolicy( + RetryPolicy.builder() + .handleIf(toCheckedPredicate(transientSocketFaults())) + .handle(RetryException.class) + .handleResultIf(this::isBadGateway) + .withDelay(Duration.ofMillis(500)) + .withMaxRetries(4) + .build()) + .withPredicate(new IdempotencyPredicate())) + .withPolicy(new RetryRequestPolicy( + RetryPolicy.builder() + .handleIf(toCheckedPredicate(transientConnectionFaults())) + .withDelay(Duration.ofMillis(500)) + .withMaxRetries(4) + .build()) + .withPredicate(alwaysTrue())) + .withPolicy(CircuitBreaker.builder() + .withFailureThreshold(5, 10) + .withSuccessThreshold(5) + .withDelay(Duration.ofMinutes(1)) + .build())) + .build(); + } + + // workaround to process differently several invocations of the same request + private final class RequestNumberMatcher extends RequestMatcher { + + private AtomicInteger sharedRetriesCount; + private int targetNumber; + + public RequestNumberMatcher(AtomicInteger sharedRetriesCount, int targetNumber) { + this.sharedRetriesCount = sharedRetriesCount; + this.targetNumber = targetNumber; + } + + @Override + public String getName() { + return "request_" + targetNumber; + } + + @Override + public MatchResult match(Request value) { + return sharedRetriesCount.compareAndSet(targetNumber - 1, targetNumber) ? MatchResult.exactMatch() : MatchResult.noMatch(); + } + } + + @SneakyThrows + private boolean isBadGateway(@Nullable final ClientHttpResponse response) { + return response != null && response.getStatusCode() == HttpStatus.BAD_GATEWAY; + } + + @AfterEach + void tearDown() throws IOException { + client.close(); + } + + @SneakyThrows + @Test + void shouldRetrySuccessfully(WireMockRuntimeInfo wmRuntimeInfo) { + var unit = buildHttp(wmRuntimeInfo); + AtomicInteger sharedRetriesCount = new AtomicInteger(); + WireMock wireMock = wmRuntimeInfo.getWireMock(); + + wireMock.register(get("/foo") + .andMatching(new RequestNumberMatcher(sharedRetriesCount, 1)) + .willReturn(status(NO_CONTENT.value()) + .withFixedDelay(800)) + ); + + wireMock.register(get("/foo") + .andMatching(new RequestNumberMatcher(sharedRetriesCount, 2)) + .willReturn(ok() + .withBody(getResource("contributors.json").openStream().readAllBytes()) + .withHeader("Content-Type", "application/json")) + ); + + + unit.get("/foo") + .call(pass()) + .join(); + + assertThat(wireMock.findAllUnmatchedRequests(), empty()); + assertEquals(2, wireMock.find(newRequestPattern().withUrl("/foo")).size()); + } + + + @Test + void shouldRetryUnsuccessfully(WireMockRuntimeInfo wmRuntimeInfo) { + var unit = buildHttp(wmRuntimeInfo); + WireMock wireMock = wmRuntimeInfo.getWireMock(); + + wireMock.register(get("/bar") + .willReturn(status(NO_CONTENT.value()).withFixedDelay(800))); + + final CompletionException exception = assertThrows(CompletionException.class, + unit.get("/bar").call(pass())::join); + + assertThat(exception.getCause(), is(instanceOf(SocketTimeoutException.class))); + assertThat(wireMock.findAllUnmatchedRequests(), empty()); + assertEquals(5, wireMock.find(newRequestPattern().withUrl("/bar")).size()); + } + + @Test + void shouldRetryExplicitly(WireMockRuntimeInfo wmRuntimeInfo) { + var unit = buildHttp(wmRuntimeInfo); + WireMock wireMock = wmRuntimeInfo.getWireMock(); + AtomicInteger sharedRetriesCount = new AtomicInteger(); + + wireMock.register(get("/baz") + .andMatching(new RequestNumberMatcher(sharedRetriesCount, 1)) + .willReturn(status(503) + .withHeader("X-RateLimit-Reset", "1523486068") + )); + + wireMock.register(get("/baz") + .andMatching(new RequestNumberMatcher(sharedRetriesCount, 2)) + .willReturn(status(NO_CONTENT.value())) + ); + + unit.get("/baz") + .dispatch(series(), + on(SUCCESSFUL).call(pass()), + anySeries().dispatch(org.zalando.riptide.Navigators.status(), + on(SERVICE_UNAVAILABLE).call(retry()))) + .join(); + + assertThat(wireMock.findAllUnmatchedRequests(), empty()); + assertEquals(2, wireMock.find(newRequestPattern().withUrl("/baz")).size()); + } + + @Test + void shouldAllowNestedCalls(WireMockRuntimeInfo wmRuntimeInfo) { + var unit = buildHttp(wmRuntimeInfo); + WireMock wireMock = wmRuntimeInfo.getWireMock(); + + wireMock.register(get("/foo").willReturn(status(NO_CONTENT.value()))); + wireMock.register(get("/bar").willReturn(status(NO_CONTENT.value()))); + + assertTimeout(Duration.ofSeconds(1), + unit.get("/foo") + .call(call(() -> unit.get("/bar").call(pass()).join()))::join); + + assertThat(wireMock.findAllUnmatchedRequests(), empty()); + assertEquals(1, wireMock.find(newRequestPattern().withUrl("/foo")).size()); + assertEquals(1, wireMock.find(newRequestPattern().withUrl("/bar")).size()); + } + + + private static MappingJackson2HttpMessageConverter createJsonConverter() { + final MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter(); + converter.setObjectMapper(createObjectMapper()); + return converter; + } + + private static ObjectMapper createObjectMapper() { + return new ObjectMapper().findAndRegisterModules() + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + } +} diff --git a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/MockSetup.java b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/MockSetup.java index 4743b4dac..a4c9aec8e 100644 --- a/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/MockSetup.java +++ b/riptide-failsafe/src/test/java/org/zalando/riptide/failsafe/MockSetup.java @@ -1,6 +1,8 @@ package org.zalando.riptide.failsafe; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Getter; +import org.springframework.http.client.ClientHttpRequestFactory; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.http.converter.StringHttpMessageConverter; import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; @@ -11,6 +13,7 @@ import javax.annotation.Nullable; import java.util.Arrays; import java.util.List; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import static com.google.common.base.MoreObjects.firstNonNull; @@ -27,7 +30,7 @@ private static MappingJackson2HttpMessageConverter createJsonConverter() { return converter; } - + @Getter private final String baseUrl; private final Iterable> converters; private final RestTemplate template; @@ -41,7 +44,8 @@ public MockSetup(final String baseUrl) { this(baseUrl, null); } - private MockSetup(@Nullable final String baseUrl, @Nullable final Iterable> converters) { + private MockSetup(@Nullable final String baseUrl, + @Nullable final Iterable> converters) { this.baseUrl = baseUrl; this.converters = converters; this.template = new RestTemplate(); @@ -52,15 +56,15 @@ public MockRestServiceServer getServer() { return server; } - private Http.ConfigurationStage getRestBuilder() { + public Http.ConfigurationStage getRestBuilder(ExecutorService executorService) { return Http.builder() - .executor(Executors.newSingleThreadExecutor()) + .executor(executorService) .requestFactory(template.getRequestFactory()) .converters(firstNonNull(converters, DEFAULT_CONVERTERS)) .baseUrl(baseUrl); } public Http getRest() { - return getRestBuilder().build(); + return getRestBuilder(Executors.newSingleThreadExecutor()).build(); } } diff --git a/riptide-failsafe/src/test/resources/contributors.json b/riptide-failsafe/src/test/resources/contributors.json new file mode 100644 index 000000000..5a8509954 --- /dev/null +++ b/riptide-failsafe/src/test/resources/contributors.json @@ -0,0 +1,82 @@ +[ + { + "login": "whiskeysierra", + "id": 429981, + "avatar_url": "https://avatars.githubusercontent.com/u/429981?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/whiskeysierra", + "html_url": "https://github.com/whiskeysierra", + "followers_url": "https://api.github.com/users/whiskeysierra/followers", + "following_url": "https://api.github.com/users/whiskeysierra/following{/other_user}", + "gists_url": "https://api.github.com/users/whiskeysierra/gists{/gist_id}", + "starred_url": "https://api.github.com/users/whiskeysierra/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/whiskeysierra/subscriptions", + "organizations_url": "https://api.github.com/users/whiskeysierra/orgs", + "repos_url": "https://api.github.com/users/whiskeysierra/repos", + "events_url": "https://api.github.com/users/whiskeysierra/events{/privacy}", + "received_events_url": "https://api.github.com/users/whiskeysierra/received_events", + "type": "User", + "site_admin": false, + "contributions": 146 + }, + { + "login": "lukasniemeier-zalando", + "id": 10497901, + "avatar_url": "https://avatars.githubusercontent.com/u/10497901?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/lukasniemeier-zalando", + "html_url": "https://github.com/lukasniemeier-zalando", + "followers_url": "https://api.github.com/users/lukasniemeier-zalando/followers", + "following_url": "https://api.github.com/users/lukasniemeier-zalando/following{/other_user}", + "gists_url": "https://api.github.com/users/lukasniemeier-zalando/gists{/gist_id}", + "starred_url": "https://api.github.com/users/lukasniemeier-zalando/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/lukasniemeier-zalando/subscriptions", + "organizations_url": "https://api.github.com/users/lukasniemeier-zalando/orgs", + "repos_url": "https://api.github.com/users/lukasniemeier-zalando/repos", + "events_url": "https://api.github.com/users/lukasniemeier-zalando/events{/privacy}", + "received_events_url": "https://api.github.com/users/lukasniemeier-zalando/received_events", + "type": "User", + "site_admin": false, + "contributions": 21 + }, + { + "login": "ePaul", + "id": 645859, + "avatar_url": "https://avatars.githubusercontent.com/u/645859?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/ePaul", + "html_url": "https://github.com/ePaul", + "followers_url": "https://api.github.com/users/ePaul/followers", + "following_url": "https://api.github.com/users/ePaul/following{/other_user}", + "gists_url": "https://api.github.com/users/ePaul/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ePaul/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ePaul/subscriptions", + "organizations_url": "https://api.github.com/users/ePaul/orgs", + "repos_url": "https://api.github.com/users/ePaul/repos", + "events_url": "https://api.github.com/users/ePaul/events{/privacy}", + "received_events_url": "https://api.github.com/users/ePaul/received_events", + "type": "User", + "site_admin": false, + "contributions": 4 + }, + { + "login": "jhorstmann", + "id": 689138, + "avatar_url": "https://avatars.githubusercontent.com/u/689138?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/jhorstmann", + "html_url": "https://github.com/jhorstmann", + "followers_url": "https://api.github.com/users/jhorstmann/followers", + "following_url": "https://api.github.com/users/jhorstmann/following{/other_user}", + "gists_url": "https://api.github.com/users/jhorstmann/gists{/gist_id}", + "starred_url": "https://api.github.com/users/jhorstmann/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/jhorstmann/subscriptions", + "organizations_url": "https://api.github.com/users/jhorstmann/orgs", + "repos_url": "https://api.github.com/users/jhorstmann/repos", + "events_url": "https://api.github.com/users/jhorstmann/events{/privacy}", + "received_events_url": "https://api.github.com/users/jhorstmann/received_events", + "type": "User", + "site_admin": false, + "contributions": 1 + } +] \ No newline at end of file