Use fixture to test repository-s3 plugin

plugins/repository-gcs/build.gradle

@@ -58,6 +58,12 @@ thirdPartyAudit.excludes = [
         'org.apache.log.Logger',
 ]
 
+forbiddenApisTest {
+  // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+  bundledSignatures -= 'jdk-non-portable'
+  bundledSignatures += 'jdk-internal'
+}
+
 /** A task to start the GoogleCloudStorageFixture which emulates a Google Cloud Storage service **/
 task googleCloudStorageFixture(type: AntFixture) {
     dependsOn compileTestJava

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java

@@ -52,17 +52,16 @@
  */
 public class GoogleCloudStorageFixture {
 
-    @SuppressForbidden(reason = "PathUtils#get is fine - we don't have environment here")
     public static void main(String[] args) throws Exception {
         if (args == null || args.length != 2) {
             throw new IllegalArgumentException("GoogleCloudStorageFixture <working directory> <bucket>");
         }
 
-        final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 43635);
+        final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0);
         final HttpServer httpServer = MockHttpServer.createHttp(socketAddress, 0);
 
         try {
-            final Path workingDirectory = Paths.get(args[0]);
+            final Path workingDirectory = workingDir(args[0]);
             /// Writes the PID of the current Java process in a `pid` file located in the working directory
             writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
 
@@ -86,6 +85,11 @@ public static void main(String[] args) throws Exception {
         }
     }
 
+    @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
+    private static Path workingDir(final String dir) {
+        return Paths.get(dir);
+    }
+
     private static void writeFile(final Path dir, final String fileName, final String content) throws IOException {
         final Path tempPidFile = Files.createTempFile(dir, null, null);
         Files.write(tempPidFile, singleton(content));
@@ -101,7 +105,6 @@ private static String addressToString(final SocketAddress address) {
         }
     }
 
-    @SuppressForbidden(reason = "Use a http server")
     static class ResponseHandler implements HttpHandler {
 
         private final GoogleCloudStorageTestServer storageServer;

plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/10_basic.yml

@@ -13,9 +13,6 @@
     - match:  { nodes.$master.plugins.0.name: repository-gcs  }
 ---
 "Snapshot/Restore with repository-gcs":
-  - skip:
-      version: " - 6.3.0"
-      reason: repository-gcs was not testable through YAML tests until 6.3.0
 
   # Register repository
   - do:
@@ -28,7 +25,15 @@
             client: "integration_test"
 
   - match: { acknowledged: true }
-
+
+  # Get repository
+  - do:
+      snapshot.get_repository:
+        repository: repository
+
+  - match: {repository.settings.bucket : "bucket_test"}
+  - match: {repository.settings.client : "integration_test"}
+
   # Index documents
   - do:
       bulk:
@@ -180,7 +185,3 @@
   - do:
      snapshot.delete_repository:
        repository: repository
-
-
-
-

plugins/repository-s3/build.gradle

@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.test.AntFixture
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -64,9 +66,28 @@ test {
   exclude '**/*CredentialsTests.class'
 }
 
+forbiddenApisTest {
+  // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+  bundledSignatures -= 'jdk-non-portable'
+  bundledSignatures += 'jdk-internal'
+}
+
+/** A task to start the AmazonS3Fixture which emulates a S3 service **/
+task s3Fixture(type: AntFixture) {
+  dependsOn compileTestJava
+  env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
+  executable = new File(project.runtimeJavaHome, 'bin/java')
+  args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, 'bucket_test'
+}
+
 integTestCluster {
-  keystoreSetting 's3.client.default.access_key', 'myaccesskey'
-  keystoreSetting 's3.client.default.secret_key', 'mysecretkey'
+  dependsOn s3Fixture
+
+  keystoreSetting 's3.client.integration_test.access_key', "s3_integration_test_access_key"
+  keystoreSetting 's3.client.integration_test.secret_key', "s3_integration_test_secret_key"
+
+  /* Use a closure on the string to delay evaluation until tests are executed */
+  setting 's3.client.integration_test.endpoint', "http://${ -> s3Fixture.addressAndPort }"
 }
 
 thirdPartyAudit.excludes = [

plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java

@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.repositories.s3;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.mocksocket.MockHttpServer;
+import org.elasticsearch.repositories.s3.AmazonS3TestServer.Response;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+
+/**
+ * {@link AmazonS3Fixture} is a fixture that emulates a S3 service.
+ * <p>
+ * It starts an asynchronous socket server that binds to a random local port. The server parses
+ * HTTP requests and uses a {@link AmazonS3TestServer} to handle them before returning
+ * them to the client as HTTP responses.
+ */
+public class AmazonS3Fixture {
+
+    public static void main(String[] args) throws Exception {
+        if (args == null || args.length != 2) {
+            throw new IllegalArgumentException("AmazonS3Fixture <working directory> <bucket>");
+        }
+
+        final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0);
+        final HttpServer httpServer = MockHttpServer.createHttp(socketAddress, 0);
+
+        try {
+            final Path workingDirectory = workingDir(args[0]);
+            /// Writes the PID of the current Java process in a `pid` file located in the working directory
+            writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
+
+            final String addressAndPort = addressToString(httpServer.getAddress());
+            // Writes the address and port of the http server in a `ports` file located in the working directory
+            writeFile(workingDirectory, "ports", addressAndPort);
+
+            // Emulates S3
+            final String storageUrl = "http://" + addressAndPort;
+            final AmazonS3TestServer storageTestServer = new AmazonS3TestServer(storageUrl);
+            storageTestServer.createBucket(args[1]);
+
+            httpServer.createContext("/", new ResponseHandler(storageTestServer));
+            httpServer.start();
+
+            // Wait to be killed
+            Thread.sleep(Long.MAX_VALUE);
+
+        } finally {
+            httpServer.stop(0);
+        }
+    }
+
+    @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
+    private static Path workingDir(final String dir) {
+        return Paths.get(dir);
+    }
+
+    private static void writeFile(final Path dir, final String fileName, final String content) throws IOException {
+        final Path tempPidFile = Files.createTempFile(dir, null, null);
+        Files.write(tempPidFile, singleton(content));
+        Files.move(tempPidFile, dir.resolve(fileName), StandardCopyOption.ATOMIC_MOVE);
+    }
+
+    private static String addressToString(final SocketAddress address) {
+        final InetSocketAddress inetSocketAddress = (InetSocketAddress) address;
+        if (inetSocketAddress.getAddress() instanceof Inet6Address) {
+            return "[" + inetSocketAddress.getHostString() + "]:" + inetSocketAddress.getPort();
+        } else {
+            return inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort();
+        }
+    }
+
+    static class ResponseHandler implements HttpHandler {
+
+        private final AmazonS3TestServer storageServer;
+
+        private ResponseHandler(final AmazonS3TestServer storageServer) {
+            this.storageServer = storageServer;
+        }
+
+        @Override
+        public void handle(HttpExchange exchange) throws IOException {
+            String method = exchange.getRequestMethod();
+            String path = storageServer.getEndpoint() + exchange.getRequestURI().getRawPath();
+            String query = exchange.getRequestURI().getRawQuery();
+            Map<String, List<String>> headers = exchange.getRequestHeaders();
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            Streams.copy(exchange.getRequestBody(), out);
+
+            final Response storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray());
+
+            Map<String, List<String>> responseHeaders = exchange.getResponseHeaders();
+            responseHeaders.put("Content-Type", singletonList(storageResponse.contentType));
+            storageResponse.headers.forEach((k, v) -> responseHeaders.put(k, singletonList(v)));
+            exchange.sendResponseHeaders(storageResponse.status.getStatus(), storageResponse.body.length);
+            if (storageResponse.body.length > 0) {
+                exchange.getResponseBody().write(storageResponse.body);
+            }
+            exchange.close();
+        }
+    }
+}