Skip to content

Commit

Permalink
Use ESBlobStoreRepositoryIntegTestCase to test the repository-s3 plugin
Browse files Browse the repository at this point in the history 
The test framework contains a base class for testing blob store
repository implementations, but the S3 plugin does not use it.

This commit adds the S3BlobStoreRepositoryTests class that extends the
base testing class for S3. It also cleans up the S3BlobStoreTests and
S3BlobStoreContainerTests so that they are now based on pure mock S3
clients.

It also removes some usage of socket servers that emulate socket
connections in unit tests. It was added to trigger security exceptions,
but this won't be needed anymore once elastic#29296 will be merged.

closes elastic#16472
  • Loading branch information
@tlrx
tlrx committed Apr 5, 2018
1 parent 9da95ef commit 1e32c08
Show file tree
Hide file tree
Showing 5 changed files with 309 additions and 263 deletions.
3 changes: 1 addition & 2 deletions plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,7 @@ class S3Repository extends BlobStoreRepository {
/**
* Constructs an s3 backed repository
*/
S3Repository(RepositoryMetaData metadata, Settings settings,
NamedXContentRegistry namedXContentRegistry, AwsS3Service s3Service) throws IOException {
S3Repository(RepositoryMetaData metadata, Settings settings, NamedXContentRegistry namedXContentRegistry, AwsS3Service s3Service) {
super(metadata, settings, namedXContentRegistry);

String bucket = BUCKET_SETTING.get(metadata.settings());
Expand Down
338 changes: 150 additions & 188 deletions plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/MockAmazonS3.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,215 +19,177 @@

package org.elasticsearch.repositories.s3;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.AbstractAmazonS3;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import org.elasticsearch.common.io.Streams;

import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.net.InetAddress;
import java.net.Socket;
import java.util.ArrayList;
import java.util.List;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import static org.junit.Assert.assertTrue;
import static java.util.Collections.emptyMap;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

class MockAmazonS3 extends AbstractAmazonS3 {

private final int mockSocketPort;
static final Map<String, Map<String, byte[]>> BUCKETS = new ConcurrentHashMap<>();

private Map<String, InputStream> blobs = new ConcurrentHashMap<>();

// in ESBlobStoreContainerTestCase.java, the maximum
// length of the input data is 100 bytes
private byte[] byteCounter = new byte[100];


MockAmazonS3(int mockSocketPort) {
this.mockSocketPort = mockSocketPort;
private MockAmazonS3() {
}

// Simulate a socket connection to check that SocketAccess.doPrivileged() is used correctly.
// Any method of AmazonS3 might potentially open a socket to the S3 service. Firstly, a call
// to any method of AmazonS3 has to be wrapped by SocketAccess.doPrivileged().
// Secondly, each method on the stack from doPrivileged to opening the socket has to be
// located in a jar that is provided by the plugin.
// Thirdly, a SocketPermission has to be configured in plugin-security.policy.
// By opening a socket in each method of MockAmazonS3 it is ensured that in production AmazonS3
// is able to to open a socket to the S3 Service without causing a SecurityException
private void simulateS3SocketConnection() {
try (Socket socket = new Socket(InetAddress.getByName("127.0.0.1"), mockSocketPort)) {
assertTrue(socket.isConnected()); // NOOP to keep static analysis happy
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}


@Override
public boolean doesBucketExist(String bucket) {
return true;
}

@Override
public boolean doesObjectExist(String bucketName, String objectName) throws AmazonServiceException, SdkClientException {
simulateS3SocketConnection();
return blobs.containsKey(objectName);
}

@Override
public ObjectMetadata getObjectMetadata(
GetObjectMetadataRequest getObjectMetadataRequest)
throws AmazonClientException, AmazonServiceException {
simulateS3SocketConnection();
String blobName = getObjectMetadataRequest.getKey();

if (!blobs.containsKey(blobName)) {
throw new AmazonS3Exception("[" + blobName + "] does not exist.");
}

return new ObjectMetadata(); // nothing is done with it
}

@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest)
throws AmazonClientException, AmazonServiceException {
simulateS3SocketConnection();
String blobName = putObjectRequest.getKey();

if (blobs.containsKey(blobName)) {
throw new AmazonS3Exception("[" + blobName + "] already exists.");
}

blobs.put(blobName, putObjectRequest.getInputStream());
return new PutObjectResult();
}

@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
throws AmazonClientException, AmazonServiceException {
simulateS3SocketConnection();
// in ESBlobStoreContainerTestCase.java, the prefix is empty,
// so the key and blobName are equivalent to each other
String blobName = getObjectRequest.getKey();

if (!blobs.containsKey(blobName)) {
throw new AmazonS3Exception("[" + blobName + "] does not exist.");
}

// the HTTP request attribute is irrelevant for reading
S3ObjectInputStream stream = new S3ObjectInputStream(
blobs.get(blobName), null, false);
S3Object s3Object = new S3Object();
s3Object.setObjectContent(stream);
return s3Object;
}

@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
throws AmazonClientException, AmazonServiceException {
simulateS3SocketConnection();
MockObjectListing list = new MockObjectListing();
list.setTruncated(false);

String blobName;
String prefix = listObjectsRequest.getPrefix();

ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();

for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
blobName = blob.getKey();
S3ObjectSummary objectSummary = new S3ObjectSummary();

if (prefix.isEmpty() || blobName.startsWith(prefix)) {
objectSummary.setKey(blobName);
public static AmazonS3 createClient(final String bucket,
final boolean serverSideEncryption,
final String cannedACL,
final String storageClass) {

final AmazonS3 mockedClient = mock(AmazonS3.class);

// Create or erase the bucket
BUCKETS.put(bucket, new ConcurrentHashMap<>());

// Bucket exists?
when(mockedClient.doesBucketExist(any(String.class))).thenAnswer(invocation -> {
String bucketName = (String) invocation.getArguments()[0];
assertThat(bucketName, equalTo(bucket));
return BUCKETS.containsKey(bucketName);
});

// Blob exists?
when(mockedClient.doesObjectExist(any(String.class), any(String.class))).thenAnswer(invocation -> {
String bucketName = (String) invocation.getArguments()[0];
String objectName = (String) invocation.getArguments()[1];
assertThat(bucketName, equalTo(bucket));
return BUCKETS.getOrDefault(bucketName, emptyMap()).containsKey(objectName);
});

// Write blob
when(mockedClient.putObject(any(PutObjectRequest.class))).thenAnswer(invocation -> {
PutObjectRequest request = (PutObjectRequest) invocation.getArguments()[0];
assertThat(request.getBucketName(), equalTo(bucket));
assertThat(request.getMetadata().getSSEAlgorithm(), serverSideEncryption ? equalTo("AES256") : nullValue());
assertThat(request.getCannedAcl(), notNullValue());
assertThat(request.getCannedAcl().toString(), cannedACL != null ? equalTo(cannedACL) : equalTo("private"));
assertThat(request.getStorageClass(), storageClass != null ? equalTo(storageClass) : equalTo("STANDARD"));

ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(request.getInputStream(), out);
assertThat((long) out.size(), equalTo(request.getMetadata().getContentLength()));

BUCKETS.computeIfAbsent(request.getBucketName(), s -> new ConcurrentHashMap<>()).put(request.getKey(), out.toByteArray());
return null;
});

// Read blob
when(mockedClient.getObject(any(String.class), any(String.class))).thenAnswer(invocation -> {
String bucketName = (String) invocation.getArguments()[0];
String objectName = (String) invocation.getArguments()[1];
assertThat(bucketName, equalTo(bucket));

byte[] blob = BUCKETS.getOrDefault(bucketName, emptyMap()).get(objectName);
if(blob == null){
AmazonS3Exception exception = new AmazonS3Exception("Blob not found");
exception.setStatusCode(404);
throw exception;
}

try {
objectSummary.setSize(getSize(blob.getValue()));
} catch (IOException e) {
throw new AmazonS3Exception("Object listing " +
"failed for blob [" + blob.getKey() + "]");
S3Object response = new S3Object();
response.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(blob), null, false));
return response;
});

// Copy blob
when(mockedClient.copyObject(any(CopyObjectRequest.class))).thenAnswer(invocation -> {
CopyObjectRequest request = (CopyObjectRequest) invocation.getArguments()[0];
assertThat(request.getSourceBucketName(), equalTo(bucket));
assertThat(request.getDestinationBucketName(), equalTo(bucket));

Map<String, byte[]> blobsInBucket = BUCKETS.getOrDefault(bucket, emptyMap());
byte[] blob = blobsInBucket.get(request.getSourceKey());
if(blob != null) {
blobsInBucket.put(request.getDestinationKey(), blob);
} else {
AmazonS3Exception exception = new AmazonS3Exception("Blob not found");
exception.setStatusCode(404);
throw exception;
}
return null;
});

// List BUCKETS
when(mockedClient.listObjects(any(String.class), any(String.class))).thenAnswer(invocation -> {
String bucketName = (String) invocation.getArguments()[0];
String prefix = (String) invocation.getArguments()[1];

assertThat(bucketName, equalTo(bucket));
ObjectListing listing = new ObjectListing();
listing.setBucketName(bucketName);
listing.setPrefix(prefix);
for (Map.Entry<String, byte[]> blob : BUCKETS.getOrDefault(bucketName, emptyMap()).entrySet()) {
if (blob.getKey().startsWith(prefix)) {
S3ObjectSummary summary = new S3ObjectSummary();
summary.setBucketName(bucketName);
summary.setKey(blob.getKey());
summary.setSize(blob.getValue().length);
listing.getObjectSummaries().add(summary);
}

mockObjectSummaries.add(objectSummary);
}
}

list.setObjectSummaries(mockObjectSummaries);
return list;
}

@Override
public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest)
throws AmazonClientException, AmazonServiceException {
simulateS3SocketConnection();
String sourceBlobName = copyObjectRequest.getSourceKey();
String targetBlobName = copyObjectRequest.getDestinationKey();

if (!blobs.containsKey(sourceBlobName)) {
throw new AmazonS3Exception("Source blob [" +
sourceBlobName + "] does not exist.");
}

if (blobs.containsKey(targetBlobName)) {
throw new AmazonS3Exception("Target blob [" +
targetBlobName + "] already exists.");
}

blobs.put(targetBlobName, blobs.get(sourceBlobName));
return new CopyObjectResult(); // nothing is done with it
}

@Override
public void deleteObject(DeleteObjectRequest deleteObjectRequest)
throws AmazonClientException, AmazonServiceException {
simulateS3SocketConnection();
String blobName = deleteObjectRequest.getKey();

if (!blobs.containsKey(blobName)) {
throw new AmazonS3Exception("[" + blobName + "] does not exist.");
}

blobs.remove(blobName);
}

private int getSize(InputStream stream) throws IOException {
int size = stream.read(byteCounter);
stream.reset(); // in case we ever need the size again
return size;
}

private class MockObjectListing extends ObjectListing {
// the objectSummaries attribute in ObjectListing.java
// is read-only, but we need to be able to write to it,
// so we create a mock of it to work around this
private List<S3ObjectSummary> mockObjectSummaries;

@Override
public List<S3ObjectSummary> getObjectSummaries() {
return mockObjectSummaries;
}
return listing;
});

// List next batch of BUCKETS
when(mockedClient.listNextBatchOfObjects(any(ObjectListing.class))).thenAnswer(invocation -> {
ObjectListing objectListing = (ObjectListing) invocation.getArguments()[0];
assertThat(objectListing.getBucketName(), equalTo(bucket));
return new ObjectListing();
});

// Delete blob
doAnswer(invocation -> {
String bucketName = (String) invocation.getArguments()[0];
String objectName = (String) invocation.getArguments()[1];
assertThat(bucketName, equalTo(bucket));

Map<String, byte[]> blobsInBucket = BUCKETS.getOrDefault(bucketName, emptyMap());
if(blobsInBucket.remove(objectName) == null){
AmazonS3Exception exception = new AmazonS3Exception("Blob not found");
exception.setStatusCode(404);
throw exception;
}
return null;
}).when(mockedClient).deleteObject(any(String.class), any(String.class));

// Delete multiple BUCKETS
doAnswer(invocation -> {
DeleteObjectsRequest deleteObjectsRequest = (DeleteObjectsRequest) invocation.getArguments()[0];
assertThat(deleteObjectsRequest.getBucketName(), equalTo(bucket));

Map<String, byte[]> blobsInBucket = BUCKETS.getOrDefault(deleteObjectsRequest.getBucketName(), emptyMap());
for (DeleteObjectsRequest.KeyVersion key : deleteObjectsRequest.getKeys()) {
if(blobsInBucket.remove(key.getKey()) == null){
AmazonS3Exception exception = new AmazonS3Exception("Blob not found");
exception.setStatusCode(404);
throw exception;
}
}
return null;
}).when(mockedClient).deleteObjects(any(DeleteObjectsRequest.class));

private void setObjectSummaries(List<S3ObjectSummary> objectSummaries) {
mockObjectSummaries = objectSummaries;
}
return mockedClient;
}
}
Loading

0 comments on commit 1e32c08

Please sign in to comment.