Move repository-s3 fixture tests to QA third party project

plugins/repository-s3/build.gradle

@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.test.AntFixture
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -66,28 +64,14 @@ test {
   exclude '**/*CredentialsTests.class'
 }
 
-forbiddenApisTest {
-  // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
-  bundledSignatures -= 'jdk-non-portable'
-  bundledSignatures += 'jdk-internal'
-}
-
-/** A task to start the AmazonS3Fixture which emulates a S3 service **/
-task s3Fixture(type: AntFixture) {
-  dependsOn compileTestJava
-  env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
-  executable = new File(project.runtimeJavaHome, 'bin/java')
-  args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, 'bucket_test'
+check {
+  // also execute the QA tests when testing the plugin
+  dependsOn 'qa:amazon-s3:check'
 }
 
 integTestCluster {
-  dependsOn s3Fixture
-
   keystoreSetting 's3.client.integration_test.access_key', "s3_integration_test_access_key"
   keystoreSetting 's3.client.integration_test.secret_key', "s3_integration_test_secret_key"
-
-  /* Use a closure on the string to delay evaluation until tests are executed */
-  setting 's3.client.integration_test.endpoint', "http://${ -> s3Fixture.addressAndPort }"
 }
 
 thirdPartyAudit.excludes = [

plugins/repository-s3/qa/amazon-s3/build.gradle

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.test.AntFixture
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+    testCompile project(path: ':plugins:repository-s3', configuration: 'runtime')
+}
+
+integTestCluster {
+    plugin ':plugins:repository-s3'
+}
+
+forbiddenApisTest {
+    // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+    bundledSignatures -= 'jdk-non-portable'
+    bundledSignatures += 'jdk-internal'
+}
+
+boolean useFixture = false
+
+String s3AccessKey = System.getenv("amazon_s3_access_key")
+String s3SecretKey = System.getenv("amazon_s3_secret_key")
+String s3Bucket = System.getenv("amazon_s3_bucket")
+String s3BasePath = System.getenv("amazon_s3_base_path")
+
+if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
+    s3AccessKey = 's3_integration_test_access_key'
+    s3SecretKey = 's3_integration_test_secret_key'
+    s3Bucket = 'bucket_test'
+    s3BasePath = 'integration_test'
+    useFixture = true
+}
+
+/** A task to start the AmazonS3Fixture which emulates a S3 service **/
+task s3Fixture(type: AntFixture) {
+    dependsOn compileTestJava
+    env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
+    executable = new File(project.runtimeJavaHome, 'bin/java')
+    args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3Bucket
+}
+
+Map<String, Object> expansions = [
+        'bucket': s3Bucket,
+        'base_path': s3BasePath
+]
+processTestResources {
+    inputs.properties(expansions)
+    MavenFilteringHack.filter(it, expansions)
+}
+
+integTestCluster {
+    keystoreSetting 's3.client.integration_test.access_key', s3AccessKey
+    keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey
+
+    if (useFixture) {
+        dependsOn s3Fixture
+        /* Use a closure on the string to delay evaluation until tests are executed */
+        setting 's3.client.integration_test.endpoint', "http://${-> s3Fixture.addressAndPort}"
+    } else {
+        println "Using an external service to test the repository-s3 plugin"
+    }
+}

plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3RepositoryClientYamlTestSuiteIT.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class AmazonS3RepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public AmazonS3RepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}

plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml

@@ -0,0 +1,183 @@
+# Integration tests for repository-s3
+---
+"Snapshot/Restore with repository-s3":
+
+  # Register repository
+  - do:
+      snapshot.create_repository:
+        repository: repository
+        body:
+          type: s3
+          settings:
+            bucket: ${bucket}
+            client: integration_test
+            base_path: ${base_path}
+            canned_acl: private
+            storage_class: standard
+
+  - match: { acknowledged: true }
+
+  # Get repository
+  - do:
+      snapshot.get_repository:
+        repository: repository
+
+  - match: { repository.settings.bucket : ${bucket} }
+  - match: { repository.settings.client : "integration_test" }
+  - match: { repository.settings.base_path : ${base_path} }
+  - match: { repository.settings.canned_acl : "private" }
+  - match: { repository.settings.storage_class : "standard" }
+  - is_false: repository.settings.access_key
+  - is_false: repository.settings.secret_key
+
+  # Index documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    1
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    2
+          - snapshot: one
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    3
+          - snapshot: one
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Create a first snapshot
+  - do:
+      snapshot.create:
+        repository: repository
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-one }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.include_global_state: true }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.status:
+        repository: repository
+        snapshot: snapshot-one
+
+  - is_true: snapshots
+  - match: { snapshots.0.snapshot: snapshot-one }
+  - match: { snapshots.0.state : SUCCESS }
+
+  # Index more documents
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    4
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    5
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    6
+          - snapshot: two
+          - index:
+              _index: docs
+              _type:  doc
+              _id:    7
+          - snapshot: two
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Create a second snapshot
+  - do:
+      snapshot.create:
+        repository: repository
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - match: { snapshot.snapshot: snapshot-two }
+  - match: { snapshot.state : SUCCESS }
+  - match: { snapshot.shards.failed : 0 }
+
+  - do:
+      snapshot.get:
+        repository: repository
+        snapshot: snapshot-one,snapshot-two
+
+  - is_true: snapshots
+  - match: { snapshots.0.state : SUCCESS }
+  - match: { snapshots.1.state : SUCCESS }
+
+  # Delete the index
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the second snapshot
+  - do:
+      snapshot.restore:
+        repository: repository
+        snapshot: snapshot-two
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 7}
+
+  # Delete the index again
+  - do:
+      indices.delete:
+        index: docs
+
+  # Restore the first snapshot
+  - do:
+      snapshot.restore:
+        repository: repository
+        snapshot: snapshot-one
+        wait_for_completion: true
+
+  - do:
+      count:
+        index: docs
+
+  - match: {count: 3}
+
+  # Remove the snapshots
+  - do:
+      snapshot.delete:
+        repository: repository
+        snapshot: snapshot-two
+
+  - do:
+      snapshot.delete:
+        repository: repository
+        snapshot: snapshot-one
+
+  # Remove our repository
+  - do:
+     snapshot.delete_repository:
+       repository: repository

plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java

@@ -156,7 +156,7 @@ class S3Repository extends BlobStoreRepository {
 
         String bucket = BUCKET_SETTING.get(metadata.settings());
         if (bucket == null) {
-            throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway");
+            throw new RepositoryException(metadata.name(), "No bucket defined for s3 repository");
         }
 
         boolean serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings());