Move S3Select classes to separate package

trinodb · Sep 23, 2019 · 7524a44 · 7524a44
1 parent e5cdc1f
commit 7524a44
Show file tree

Hide file tree

Showing 15 changed files with 39 additions and 27 deletions.
diff --git a/...e/TestHiveFileSystemS3SelectPushdown.java → ...t/TestHiveFileSystemS3SelectPushdown.java b/...e/TestHiveFileSystemS3SelectPushdown.java → ...t/TestHiveFileSystemS3SelectPushdown.java
@@ -11,8 +11,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
+import io.prestosql.plugin.hive.AbstractTestHiveFileSystemS3;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Parameters;
 

diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/BackgroundHiveSplitLoader.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/BackgroundHiveSplitLoader.java
@@ -79,9 +79,9 @@
 import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE;
 import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_UNKNOWN_ERROR;
 import static io.prestosql.plugin.hive.HiveSessionProperties.isForceLocalScheduling;
-import static io.prestosql.plugin.hive.S3SelectPushdown.shouldEnablePushdownForTable;
 import static io.prestosql.plugin.hive.metastore.MetastoreUtil.getHiveSchema;
 import static io.prestosql.plugin.hive.metastore.MetastoreUtil.getPartitionLocation;
+import static io.prestosql.plugin.hive.s3select.S3SelectPushdown.shouldEnablePushdownForTable;
 import static io.prestosql.plugin.hive.util.ConfigurationUtils.toJobConf;
 import static io.prestosql.plugin.hive.util.HiveFileIterator.NestedDirectoryPolicy.FAIL;
 import static io.prestosql.plugin.hive.util.HiveFileIterator.NestedDirectoryPolicy.IGNORED;

diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/GenericHiveRecordCursor.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/GenericHiveRecordCursor.java
@@ -82,7 +82,7 @@
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
 
-class GenericHiveRecordCursor<K, V extends Writable>
+public class GenericHiveRecordCursor<K, V extends Writable>
         implements RecordCursor
 {
     private final Path path;

diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveModule.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveModule.java
@@ -26,7 +26,8 @@
 import io.prestosql.plugin.hive.parquet.ParquetReaderConfig;
 import io.prestosql.plugin.hive.parquet.ParquetWriterConfig;
 import io.prestosql.plugin.hive.rcfile.RcFilePageSourceFactory;
-import io.prestosql.plugin.hive.s3.PrestoS3ClientFactory;
+import io.prestosql.plugin.hive.s3select.PrestoS3ClientFactory;
+import io.prestosql.plugin.hive.s3select.S3SelectRecordCursorProvider;
 import io.prestosql.spi.connector.ConnectorNodePartitioningProvider;
 import io.prestosql.spi.connector.ConnectorPageSinkProvider;
 import io.prestosql.spi.connector.ConnectorPageSourceProvider;

diff --git a/...plugin/hive/s3/PrestoS3ClientFactory.java → .../hive/s3select/PrestoS3ClientFactory.java b/...plugin/hive/s3/PrestoS3ClientFactory.java → .../hive/s3select/PrestoS3ClientFactory.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive.s3;
+package io.prestosql.plugin.hive.s3select;
 
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
@@ -29,6 +29,9 @@
 import com.amazonaws.services.s3.AmazonS3Client;
 import io.airlift.units.Duration;
 import io.prestosql.plugin.hive.HiveConfig;
+import io.prestosql.plugin.hive.s3.HiveS3Config;
+import io.prestosql.plugin.hive.s3.PrestoS3FileSystemMetricCollector;
+import io.prestosql.plugin.hive.s3.PrestoS3FileSystemStats;
 import org.apache.hadoop.conf.Configuration;
 
 import javax.annotation.concurrent.GuardedBy;

diff --git a/.../plugin/hive/s3/PrestoS3SelectClient.java → ...n/hive/s3select/PrestoS3SelectClient.java b/.../plugin/hive/s3/PrestoS3SelectClient.java → ...n/hive/s3select/PrestoS3SelectClient.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive.s3;
+package io.prestosql.plugin.hive.s3select;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.SelectObjectContentEventVisitor;
@@ -27,7 +27,7 @@
 import static com.amazonaws.services.s3.model.SelectObjectContentEvent.EndEvent;
 import static java.util.Objects.requireNonNull;
 
-public class PrestoS3SelectClient
+class PrestoS3SelectClient
         implements Closeable
 {
     private final AmazonS3 s3Client;

diff --git a/.../plugin/hive/S3SelectCsvRecordReader.java → ...ive/s3select/S3SelectCsvRecordReader.java b/.../plugin/hive/S3SelectCsvRecordReader.java → ...ive/s3select/S3SelectCsvRecordReader.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
 import com.amazonaws.services.s3.model.CSVInput;
 import com.amazonaws.services.s3.model.CSVOutput;
@@ -20,7 +20,7 @@
 import com.amazonaws.services.s3.model.InputSerialization;
 import com.amazonaws.services.s3.model.OutputSerialization;
 import com.amazonaws.services.s3.model.SelectObjectContentRequest;
-import io.prestosql.plugin.hive.s3.PrestoS3ClientFactory;
+import io.prestosql.plugin.hive.HiveConfig;
 import io.prestosql.plugin.hive.s3.PrestoS3FileSystem;
 import io.prestosql.spi.PrestoException;
 import org.apache.hadoop.conf.Configuration;
@@ -36,7 +36,7 @@
 import static org.apache.hadoop.hive.serde.serdeConstants.ESCAPE_CHAR;
 import static org.apache.hadoop.hive.serde.serdeConstants.QUOTE_CHAR;
 
-public class S3SelectCsvRecordReader
+class S3SelectCsvRecordReader
         extends S3SelectLineRecordReader
 {
     /*
@@ -49,7 +49,7 @@ public class S3SelectCsvRecordReader
 
     private static final String COMMENTS_CHAR_STR = "\uFDD0";
 
-    S3SelectCsvRecordReader(
+    public S3SelectCsvRecordReader(
             Configuration configuration,
             HiveConfig hiveConfig,
             Path path,

diff --git a/...plugin/hive/S3SelectLineRecordReader.java → ...ve/s3select/S3SelectLineRecordReader.java b/...plugin/hive/S3SelectLineRecordReader.java → ...ve/s3select/S3SelectLineRecordReader.java
@@ -11,16 +11,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.SelectObjectContentRequest;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.Closer;
 import io.airlift.units.Duration;
+import io.prestosql.plugin.hive.HiveConfig;
 import io.prestosql.plugin.hive.s3.HiveS3Config;
-import io.prestosql.plugin.hive.s3.PrestoS3ClientFactory;
-import io.prestosql.plugin.hive.s3.PrestoS3SelectClient;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;

diff --git a/...estosql/plugin/hive/S3SelectPushdown.java → ...lugin/hive/s3select/S3SelectPushdown.java b/...estosql/plugin/hive/S3SelectPushdown.java → ...lugin/hive/s3select/S3SelectPushdown.java
@@ -11,10 +11,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
 import com.google.common.collect.ImmutableSet;
-import io.airlift.log.Logger;
 import io.prestosql.plugin.hive.metastore.Column;
 import io.prestosql.plugin.hive.metastore.Partition;
 import io.prestosql.plugin.hive.metastore.Table;
@@ -53,7 +52,6 @@
  */
 public final class S3SelectPushdown
 {
-    private static final Logger LOG = Logger.get(S3SelectPushdown.class);
     private static final Set<String> SUPPORTED_S3_PREFIXES = ImmutableSet.of("s3://", "s3a://", "s3n://");
     private static final Set<String> SUPPORTED_SERDES = ImmutableSet.of(LazySimpleSerDe.class.getName());
     private static final Set<String> SUPPORTED_INPUT_FORMATS = ImmutableSet.of(TextInputFormat.class.getName());
@@ -128,7 +126,7 @@ private static boolean isS3Storage(String path)
         return SUPPORTED_S3_PREFIXES.stream().anyMatch(path::startsWith);
     }
 
-    static boolean shouldEnablePushdownForTable(ConnectorSession session, Table table, String path, Optional<Partition> optionalPartition)
+    public static boolean shouldEnablePushdownForTable(ConnectorSession session, Table table, String path, Optional<Partition> optionalPartition)
     {
         if (!isS3SelectPushdownEnabled(session)) {
             return false;

diff --git a/...sql/plugin/hive/S3SelectRecordCursor.java → ...n/hive/s3select/S3SelectRecordCursor.java b/...sql/plugin/hive/S3SelectRecordCursor.java → ...n/hive/s3select/S3SelectRecordCursor.java
@@ -11,8 +11,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
+import com.google.common.annotations.VisibleForTesting;
+import io.prestosql.plugin.hive.GenericHiveRecordCursor;
+import io.prestosql.plugin.hive.HiveColumnHandle;
 import io.prestosql.spi.type.TypeManager;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -59,6 +62,7 @@ public S3SelectRecordCursor(
     // since s3select only returns the required column, not the whole columns
     // we need to update the split schema to include only the required columns
     // otherwise, Serde could not deserialize output from s3select to row data correctly
+    @VisibleForTesting
     static Properties updateSplitSchema(Properties splitSchema, List<HiveColumnHandle> columns)
     {
         requireNonNull(splitSchema, "splitSchema is null");

diff --git a/...in/hive/S3SelectRecordCursorProvider.java → ...3select/S3SelectRecordCursorProvider.java b/...in/hive/S3SelectRecordCursorProvider.java → ...3select/S3SelectRecordCursorProvider.java
@@ -11,10 +11,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
 import com.google.common.collect.ImmutableSet;
-import io.prestosql.plugin.hive.s3.PrestoS3ClientFactory;
+import io.prestosql.plugin.hive.HdfsEnvironment;
+import io.prestosql.plugin.hive.HiveColumnHandle;
+import io.prestosql.plugin.hive.HiveConfig;
+import io.prestosql.plugin.hive.HiveRecordCursorProvider;
+import io.prestosql.plugin.hive.IonSqlQueryBuilder;
 import io.prestosql.spi.PrestoException;
 import io.prestosql.spi.connector.ConnectorSession;
 import io.prestosql.spi.connector.RecordCursor;

diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/InternalHiveSplitFactory.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/InternalHiveSplitFactory.java
@@ -20,7 +20,7 @@
 import io.prestosql.plugin.hive.HiveTypeName;
 import io.prestosql.plugin.hive.InternalHiveSplit;
 import io.prestosql.plugin.hive.InternalHiveSplit.InternalHiveBlock;
-import io.prestosql.plugin.hive.S3SelectPushdown;
+import io.prestosql.plugin.hive.s3select.S3SelectPushdown;
 import io.prestosql.spi.HostAddress;
 import io.prestosql.spi.predicate.Domain;
 import io.prestosql.spi.predicate.TupleDomain;

diff --git a/...in/hive/TestS3SelectLineRecordReader.java → ...3select/TestS3SelectLineRecordReader.java b/...in/hive/TestS3SelectLineRecordReader.java → ...3select/TestS3SelectLineRecordReader.java
@@ -11,13 +11,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
 import org.testng.annotations.Test;
 
 import java.io.IOException;
 
-import static io.prestosql.plugin.hive.S3SelectLineRecordReader.UnrecoverableS3OperationException;
+import static io.prestosql.plugin.hive.s3select.S3SelectLineRecordReader.UnrecoverableS3OperationException;
 
 public class TestS3SelectLineRecordReader
 {

diff --git a/...sql/plugin/hive/TestS3SelectPushdown.java → ...n/hive/s3select/TestS3SelectPushdown.java b/...sql/plugin/hive/TestS3SelectPushdown.java → ...n/hive/s3select/TestS3SelectPushdown.java
@@ -11,7 +11,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.JobConf;

diff --git a/...plugin/hive/TestS3SelectRecordCursor.java → ...ve/s3select/TestS3SelectRecordCursor.java b/...plugin/hive/TestS3SelectRecordCursor.java → ...ve/s3select/TestS3SelectRecordCursor.java
@@ -11,9 +11,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.plugin.hive;
+package io.prestosql.plugin.hive.s3select;
 
 import com.google.common.collect.ImmutableList;
+import io.prestosql.plugin.hive.HiveColumnHandle;
+import io.prestosql.plugin.hive.HiveType;
 import io.prestosql.spi.type.StandardTypes;
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -27,7 +29,7 @@
 import static io.prestosql.plugin.hive.HiveColumnHandle.ColumnType.REGULAR;
 import static io.prestosql.plugin.hive.HiveType.HIVE_INT;
 import static io.prestosql.plugin.hive.HiveType.HIVE_STRING;
-import static io.prestosql.plugin.hive.S3SelectRecordCursor.updateSplitSchema;
+import static io.prestosql.plugin.hive.s3select.S3SelectRecordCursor.updateSplitSchema;
 import static io.prestosql.spi.type.TypeSignature.parseTypeSignature;
 import static java.util.Arrays.asList;
 import static java.util.stream.Collectors.joining;