Skip to content

Commit

Permalink
Fix undeclared dependency used for HTTP constants
Browse files Browse the repository at this point in the history 
Cherry-pick of trinodb/trino#25

PR: prestodb#13125

Co-authored-by: Ariel Weisberg <aweisberg@fb.com>
  • Loading branch information
@electrum @aweisberg
electrum and aweisberg committed Aug 19, 2019
1 parent a742452 commit ba6b071
Show file tree
Hide file tree
Showing 4 changed files with 43 additions and 42 deletions.
12 changes: 6 additions & 6 deletions presto-hive/src/main/java/com/facebook/presto/hive/S3SelectLineRecordReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,14 @@
import static com.google.common.base.Throwables.throwIfInstanceOf;
import static com.google.common.base.Throwables.throwIfUnchecked;
import static java.lang.String.format;
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM;
import static org.apache.hadoop.hive.serde.serdeConstants.LINE_DELIM;
import static org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT;
import static org.apache.http.HttpStatus.SC_BAD_REQUEST;
import static org.apache.http.HttpStatus.SC_FORBIDDEN;
import static org.apache.http.HttpStatus.SC_NOT_FOUND;

@ThreadSafe
public abstract class S3SelectLineRecordReader
Expand Down Expand Up @@ -137,9 +137,9 @@ private int readLine(Text value)
recordsFromS3 = 0;
if (e instanceof AmazonS3Exception) {
switch (((AmazonS3Exception) e).getStatusCode()) {
case SC_FORBIDDEN:
case SC_NOT_FOUND:
case SC_BAD_REQUEST:
case HTTP_FORBIDDEN:
case HTTP_NOT_FOUND:
case HTTP_BAD_REQUEST:
throw new UnrecoverableS3OperationException(selectClient.getBucketName(), selectClient.getKeyName(), e);
}
}
Expand Down
30 changes: 15 additions & 15 deletions presto-hive/src/main/java/com/facebook/presto/hive/s3/PrestoS3FileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,17 +136,16 @@
import static java.lang.Math.max;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.nio.file.Files.createDirectories;
import static java.nio.file.Files.createTempFile;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.fs.FSExceptionMessages.CANNOT_SEEK_PAST_EOF;
import static org.apache.hadoop.fs.FSExceptionMessages.NEGATIVE_SEEK;
import static org.apache.hadoop.fs.FSExceptionMessages.STREAM_IS_CLOSED;
import static org.apache.http.HttpStatus.SC_BAD_REQUEST;
import static org.apache.http.HttpStatus.SC_FORBIDDEN;
import static org.apache.http.HttpStatus.SC_NOT_FOUND;
import static org.apache.http.HttpStatus.SC_REQUESTED_RANGE_NOT_SATISFIABLE;

public class PrestoS3FileSystem
extends FileSystem
Expand All @@ -161,6 +160,7 @@ public class PrestoS3FileSystem
private static final DataSize MAX_SKIP_SIZE = new DataSize(1, MEGABYTE);
private static final String PATH_SEPARATOR = "/";
private static final Duration BACKOFF_MIN_SLEEP = new Duration(1, SECONDS);
private static final int HTTP_RANGE_NOT_SATISFIABLE = 416;

private URI uri;
private Path workingDirectory;
Expand Down Expand Up @@ -595,10 +595,10 @@ private ObjectMetadata getS3ObjectMetadata(Path path, String bucketName, String
STATS.newGetMetadataError();
if (e instanceof AmazonS3Exception) {
switch (((AmazonS3Exception) e).getStatusCode()) {
case SC_NOT_FOUND:
case HTTP_NOT_FOUND:
return null;
case SC_FORBIDDEN:
case SC_BAD_REQUEST:
case HTTP_FORBIDDEN:
case HTTP_BAD_REQUEST:
throw new UnrecoverableS3OperationException(path, e);
}
}
Expand Down Expand Up @@ -859,11 +859,11 @@ public int read(long position, byte[] buffer, int offset, int length)
STATS.newGetObjectError();
if (e instanceof AmazonS3Exception) {
switch (((AmazonS3Exception) e).getStatusCode()) {
case SC_REQUESTED_RANGE_NOT_SATISFIABLE:
case HTTP_RANGE_NOT_SATISFIABLE:
throw new EOFException(CANNOT_SEEK_PAST_EOF);
case SC_FORBIDDEN:
case SC_NOT_FOUND:
case SC_BAD_REQUEST:
case HTTP_FORBIDDEN:
case HTTP_NOT_FOUND:
case HTTP_BAD_REQUEST:
throw new UnrecoverableS3OperationException(path, e);
}
}
Expand Down Expand Up @@ -1023,12 +1023,12 @@ private InputStream openStream(Path path, long start)
STATS.newGetObjectError();
if (e instanceof AmazonS3Exception) {
switch (((AmazonS3Exception) e).getStatusCode()) {
case SC_REQUESTED_RANGE_NOT_SATISFIABLE:
case HTTP_RANGE_NOT_SATISFIABLE:
// ignore request for start past end of object
return new ByteArrayInputStream(new byte[0]);
case SC_FORBIDDEN:
case SC_NOT_FOUND:
case SC_BAD_REQUEST:
case HTTP_FORBIDDEN:
case HTTP_NOT_FOUND:
case HTTP_BAD_REQUEST:
throw new UnrecoverableS3OperationException(path, e);
}
}
Expand Down
10 changes: 5 additions & 5 deletions presto-hive/src/test/java/com/facebook/presto/hive/s3/MockAmazonS3.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,16 @@

import java.util.Date;

import static org.apache.http.HttpStatus.SC_OK;
import static java.net.HttpURLConnection.HTTP_OK;

public class MockAmazonS3
extends AbstractAmazonS3
{
private static final String STANDARD_OBJECT_KEY = "test/standard";
private static final String GLACIER_OBJECT_KEY = "test/glacier";

private int getObjectHttpCode = SC_OK;
private int getObjectMetadataHttpCode = SC_OK;
private int getObjectHttpCode = HTTP_OK;
private int getObjectMetadataHttpCode = HTTP_OK;
private GetObjectMetadataRequest getObjectMetadataRequest;
private CannedAccessControlList acl;
private boolean hasGlacierObjects;
Expand Down Expand Up @@ -72,7 +72,7 @@ public GetObjectMetadataRequest getGetObjectMetadataRequest()
public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest)
{
this.getObjectMetadataRequest = getObjectMetadataRequest;
if (getObjectMetadataHttpCode != SC_OK) {
if (getObjectMetadataHttpCode != HTTP_OK) {
AmazonS3Exception exception = new AmazonS3Exception("Failing getObjectMetadata call with " + getObjectMetadataHttpCode);
exception.setStatusCode(getObjectMetadataHttpCode);
throw exception;
Expand All @@ -83,7 +83,7 @@ public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetada
@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
{
if (getObjectHttpCode != SC_OK) {
if (getObjectHttpCode != HTTP_OK) {
AmazonS3Exception exception = new AmazonS3Exception("Failing getObject call with " + getObjectHttpCode);
exception.setStatusCode(getObjectHttpCode);
throw exception;
Expand Down
33 changes: 17 additions & 16 deletions presto-hive/src/test/java/com/facebook/presto/hive/s3/TestPrestoS3FileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,17 +71,18 @@
import static com.google.common.io.MoreFiles.deleteRecursively;
import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
import static io.airlift.testing.Assertions.assertInstanceOf;
import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.nio.file.Files.createTempDirectory;
import static java.nio.file.Files.createTempFile;
import static org.apache.http.HttpStatus.SC_FORBIDDEN;
import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
import static org.apache.http.HttpStatus.SC_NOT_FOUND;
import static org.apache.http.HttpStatus.SC_REQUESTED_RANGE_NOT_SATISFIABLE;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;

public class TestPrestoS3FileSystem
{
private static final int HTTP_RANGE_NOT_SATISFIABLE = 416;

@Test
public void testStaticCredentials()
throws Exception
Expand Down Expand Up @@ -190,7 +191,7 @@ public void testReadRetryCounters()
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
int maxRetries = 2;
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectHttpErrorCode(SC_INTERNAL_SERVER_ERROR);
s3.setGetObjectHttpErrorCode(HTTP_INTERNAL_ERROR);
Configuration configuration = new Configuration();
configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
configuration.set(S3_MAX_RETRY_TIME, "5s");
Expand All @@ -202,7 +203,7 @@ public void testReadRetryCounters()
}
catch (Throwable expected) {
assertInstanceOf(expected, AmazonS3Exception.class);
assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR);
assertEquals(((AmazonS3Exception) expected).getStatusCode(), HTTP_INTERNAL_ERROR);
assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries);
assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries);
}
Expand All @@ -216,7 +217,7 @@ public void testGetMetadataRetryCounter()
int maxRetries = 2;
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectMetadataHttpCode(SC_INTERNAL_SERVER_ERROR);
s3.setGetObjectMetadataHttpCode(HTTP_INTERNAL_ERROR);
Configuration configuration = new Configuration();
configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
configuration.set(S3_MAX_RETRY_TIME, "5s");
Expand All @@ -227,19 +228,19 @@ public void testGetMetadataRetryCounter()
}
catch (Throwable expected) {
assertInstanceOf(expected, AmazonS3Exception.class);
assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR);
assertEquals(((AmazonS3Exception) expected).getStatusCode(), HTTP_INTERNAL_ERROR);
assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetMetadataRetries().getTotalCount(), maxRetries);
}
}

@SuppressWarnings("ResultOfMethodCallIgnored")
@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + SC_NOT_FOUND + ".*")
@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + HTTP_NOT_FOUND + ".*")
public void testReadNotFound()
throws Exception
{
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectHttpErrorCode(SC_NOT_FOUND);
s3.setGetObjectHttpErrorCode(HTTP_NOT_FOUND);
fs.initialize(new URI("s3n://test-bucket/"), new Configuration());
fs.setS3Client(s3);
try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
Expand All @@ -249,13 +250,13 @@ public void testReadNotFound()
}

@SuppressWarnings("ResultOfMethodCallIgnored")
@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + SC_FORBIDDEN + ".*")
@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObject call with " + HTTP_FORBIDDEN + ".*")
public void testReadForbidden()
throws Exception
{
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectHttpErrorCode(SC_FORBIDDEN);
s3.setGetObjectHttpErrorCode(HTTP_FORBIDDEN);
fs.initialize(new URI("s3n://test-bucket/"), new Configuration());
fs.setS3Client(s3);
try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
Expand Down Expand Up @@ -348,7 +349,7 @@ public void testReadRequestRangeNotSatisfiable()
{
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectHttpErrorCode(SC_REQUESTED_RANGE_NOT_SATISFIABLE);
s3.setGetObjectHttpErrorCode(HTTP_RANGE_NOT_SATISFIABLE);
fs.initialize(new URI("s3n://test-bucket/"), new Configuration());
fs.setS3Client(s3);
try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
Expand All @@ -357,13 +358,13 @@ public void testReadRequestRangeNotSatisfiable()
}
}

@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObjectMetadata call with " + SC_FORBIDDEN + ".*")
@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = ".*Failing getObjectMetadata call with " + HTTP_FORBIDDEN + ".*")
public void testGetMetadataForbidden()
throws Exception
{
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectMetadataHttpCode(SC_FORBIDDEN);
s3.setGetObjectMetadataHttpCode(HTTP_FORBIDDEN);
fs.initialize(new URI("s3n://test-bucket/"), new Configuration());
fs.setS3Client(s3);
fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test"));
Expand All @@ -376,7 +377,7 @@ public void testGetMetadataNotFound()
{
try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
MockAmazonS3 s3 = new MockAmazonS3();
s3.setGetObjectMetadataHttpCode(SC_NOT_FOUND);
s3.setGetObjectMetadataHttpCode(HTTP_NOT_FOUND);
fs.initialize(new URI("s3n://test-bucket/"), new Configuration());
fs.setS3Client(s3);
assertEquals(fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test")), null);
Expand Down

0 comments on commit ba6b071

Please sign in to comment.