From da099bc94ab1665e16f1d6560571443f0a29907e Mon Sep 17 00:00:00 2001 From: shubham agarwal Date: Wed, 6 May 2015 01:41:10 -0400 Subject: [PATCH 1/7] xmls --- pom.xml | 5 +++-- presto-server/src/main/provisio/presto.xml | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 91d00cd66ac6..4a18c051ccfb 100644 --- a/pom.xml +++ b/pom.xml @@ -62,13 +62,14 @@ presto-spi presto-kafka - presto-cassandra + presto-kinesis + presto-example-http presto-tpch presto-raptor diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml index 4bee85795e8a..f2f5c1e97eb8 100644 --- a/presto-server/src/main/provisio/presto.xml +++ b/presto-server/src/main/provisio/presto.xml @@ -68,6 +68,12 @@ + + + + + + From e459f314dc2f3b877e12a92e56037712f3ab0864 Mon Sep 17 00:00:00 2001 From: shubham agarwal Date: Fri, 8 May 2015 05:48:36 -0400 Subject: [PATCH 2/7] Adding Kinesis Directory --- presto-kinesis/pom.xml | 288 ++++++++++++++++ .../presto/kinesis/KinesisAwsCredentials.java | 43 +++ .../presto/kinesis/KinesisClientManager.java | 53 +++ .../presto/kinesis/KinesisColumnHandle.java | 204 +++++++++++ .../presto/kinesis/KinesisConnector.java | 90 +++++ .../kinesis/KinesisConnectorConfig.java | 130 +++++++ .../kinesis/KinesisConnectorFactory.java | 104 ++++++ .../kinesis/KinesisConnectorModule.java | 90 +++++ .../presto/kinesis/KinesisErrorCode.java | 45 +++ .../kinesis/KinesisFieldValueProvider.java | 47 +++ .../presto/kinesis/KinesisHandleResolver.java | 142 ++++++++ .../KinesisInternalFieldDescription.java | 233 +++++++++++++ .../presto/kinesis/KinesisMetadata.java | 239 +++++++++++++ .../presto/kinesis/KinesisPlugin.java | 71 ++++ .../presto/kinesis/KinesisRecordSet.java | 318 ++++++++++++++++++ .../kinesis/KinesisRecordSetProvider.java | 76 +++++ .../facebook/presto/kinesis/KinesisShard.java | 65 ++++ .../facebook/presto/kinesis/KinesisSplit.java | 126 +++++++ .../presto/kinesis/KinesisSplitManager.java | 115 +++++++ .../kinesis/KinesisStreamDescription.java | 84 +++++ .../KinesisStreamFieldDescription.java | 154 +++++++++ .../kinesis/KinesisStreamFieldGroup.java | 59 ++++ .../KinesisTableDescriptionSupplier.java | 122 +++++++ .../presto/kinesis/KinesisTableHandle.java | 141 ++++++++ .../kinesis/decoder/KinesisDecoderModule.java | 56 +++ .../decoder/KinesisDecoderRegistry.java | 113 +++++++ .../kinesis/decoder/KinesisFieldDecoder.java | 51 +++ .../kinesis/decoder/KinesisRowDecoder.java | 44 +++ .../decoder/csv/CsvKinesisDecoderModule.java | 32 ++ .../decoder/csv/CsvKinesisFieldDecoder.java | 100 ++++++ .../decoder/csv/CsvKinesisRowDecoder.java | 88 +++++ .../dummy/DummyKinesisDecoderModule.java | 32 ++ .../dummy/DummyKinesisFieldDecoder.java | 77 +++++ .../decoder/dummy/DummyKinesisRowDecoder.java | 44 +++ ...CustomDateTimeJsonKinesisFieldDecoder.java | 95 ++++++ .../json/ISO8601JsonKinesisFieldDecoder.java | 101 ++++++ .../json/JsonKinesisDecoderModule.java | 56 +++ .../decoder/json/JsonKinesisFieldDecoder.java | 115 +++++++ .../decoder/json/JsonKinesisRowDecoder.java | 92 +++++ ...ondsSinceEpochJsonKinesisFieldDecoder.java | 101 ++++++ .../json/RFC2822JsonKinesisFieldDecoder.java | 102 ++++++ ...ondsSinceEpochJsonKinesisFieldDecoder.java | 101 ++++++ .../decoder/raw/RawKinesisDecoderModule.java | 31 ++ .../decoder/raw/RawKinesisFieldDecoder.java | 227 +++++++++++++ .../decoder/raw/RawKinesisRowDecoder.java | 54 +++ .../kinesis/TestKinesisConnectorConfig.java | 70 ++++ .../presto/kinesis/TestKinesisPlugin.java | 86 +++++ .../kinesis/TestMinimalFunctionality.java | 176 ++++++++++ .../kinesis/decoder/csv/TestCsvDecoder.java | 80 +++++ .../TestISO8601JsonKinesisFieldDecoder.java | 136 ++++++++ .../kinesis/decoder/json/TestJsonDecoder.java | 135 ++++++++ ...ondsSinceEpochJsonKinesisFieldDecoder.java | 131 ++++++++ .../TestRFC2822JsonKinesisFieldDecoder.java | 132 ++++++++ ...ondsSinceEpochJsonKinesisFieldDecoder.java | 132 ++++++++ .../kinesis/decoder/raw/TestRawDecoder.java | 233 +++++++++++++ .../kinesis/decoder/util/DecoderTestUtil.java | 74 ++++ .../kinesis/util/EmbeddedKinesisStream.java | 79 +++++ .../presto/kinesis/util/TestUtils.java | 71 ++++ .../test/resources/decoder/json/message.json | 80 +++++ 59 files changed, 6366 insertions(+) create mode 100644 presto-kinesis/pom.xml create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisAwsCredentials.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisClientManager.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisColumnHandle.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnector.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorConfig.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorFactory.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorModule.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisErrorCode.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisFieldValueProvider.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisHandleResolver.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisInternalFieldDescription.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisMetadata.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisPlugin.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSet.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSetProvider.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisShard.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplit.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplitManager.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamDescription.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamFieldDescription.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamFieldGroup.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisTableDescriptionSupplier.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisTableHandle.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisDecoderModule.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisDecoderRegistry.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisRowDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisDecoderModule.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisRowDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisDecoderModule.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisRowDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/CustomDateTimeJsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/ISO8601JsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisDecoderModule.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisRowDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/MillisecondsSinceEpochJsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/RFC2822JsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/SecondsSinceEpochJsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisDecoderModule.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisRowDecoder.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestKinesisConnectorConfig.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestKinesisPlugin.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestMinimalFunctionality.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/csv/TestCsvDecoder.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestISO8601JsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestJsonDecoder.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestMillisecondsSinceEpochJsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestRFC2822JsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestSecondsSinceEpochJsonKinesisFieldDecoder.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/raw/TestRawDecoder.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/util/DecoderTestUtil.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/util/EmbeddedKinesisStream.java create mode 100644 presto-kinesis/src/test/java/com/facebook/presto/kinesis/util/TestUtils.java create mode 100644 presto-kinesis/src/test/resources/decoder/json/message.json diff --git a/presto-kinesis/pom.xml b/presto-kinesis/pom.xml new file mode 100644 index 000000000000..92b08591a5b8 --- /dev/null +++ b/presto-kinesis/pom.xml @@ -0,0 +1,288 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.102 + + + + + + + presto-kinesis + Presto - Kinesis Connector + presto-plugin + + + + ${project.parent.basedir} + + + + + + + org.weakref + jmxutils + provided + + + + io.airlift + bootstrap + provided + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + org.slf4j + jul-to-slf4j + + + org.slf4j + jcl-over-slf4j + + + + + + io.airlift + json + provided + + + joda-time + joda-time + + + + + + + joda-time + joda-time + 2.7 + + + + io.airlift + log + provided + + + + io.airlift + configuration + provided + + + + io.airlift + slice + provided + + + + + + com.facebook.presto + presto-spi + provided + + + + com.google.guava + guava + provided + + + + com.google.inject + guice + provided + + + + com.google.inject.extensions + guice-multibindings + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + com.fasterxml.jackson.core + jackson-databind + provided + + + + com.fasterxml.jackson.core + jackson-core + provided + + + + javax.inject + javax.inject + provided + + + + javax.validation + validation-api + provided + + + + com.google.code.findbugs + annotations + provided + + + + net.sf.opencsv + opencsv + + + + + + com.amazonaws + aws-java-sdk-kinesis + 1.9.16 + + + + com.amazonaws + aws-java-sdk-core + 1.9.16 + + + + + + org.testng + testng + test + + + + io.airlift + testing + test + + + + com.facebook.presto + presto-main + test + + + + com.facebook.presto + presto-tpch + test + + + + com.facebook.presto + presto-client + test + + + + com.facebook.presto + presto-tests + test + + + + io.airlift.tpch + tpch + test + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.9 + + + analyze + + analyze-only + + + true + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + **/TestKafkaDistributed.java + + + + + + + + + ci + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + + + diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisAwsCredentials.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisAwsCredentials.java new file mode 100644 index 000000000000..55d60e6f7dac --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisAwsCredentials.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import com.amazonaws.auth.AWSCredentials; +import com.google.inject.Inject; + +public class KinesisAwsCredentials + implements AWSCredentials +{ + private String accessKeyId; + private String secretKey; + + @Inject + public KinesisAwsCredentials(String accessKeyId, String secretKey) + { + this.accessKeyId = accessKeyId; + this.secretKey = secretKey; + } + + @Override + public String getAWSAccessKeyId() + { + return accessKeyId; + } + + @Override + public String getAWSSecretKey() + { + return secretKey; + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisClientManager.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisClientManager.java new file mode 100644 index 000000000000..69c04edb3c9f --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisClientManager.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import javax.inject.Named; + +import com.amazonaws.services.kinesis.AmazonKinesisClient; +import com.amazonaws.services.kinesis.model.DescribeStreamRequest; +import com.google.inject.Inject; + +import io.airlift.log.Logger; + +/** + * + * Creates and manages clients for consumer + * + */ +public class KinesisClientManager +{ + private static final Logger log = Logger.get(KinesisClientManager.class); + private final AmazonKinesisClient client; + private final KinesisAwsCredentials kinesisAwsCredentials; + + @Inject + KinesisClientManager(@Named("connectorId") String connectorId, + KinesisConnectorConfig kinesisConnectorConfig) + { + log.info("Creating new client for Consuner"); + this.kinesisAwsCredentials = new KinesisAwsCredentials(kinesisConnectorConfig.getAccessKey(), kinesisConnectorConfig.getSecretKey()); + this.client = new AmazonKinesisClient(this.kinesisAwsCredentials); + } + + public AmazonKinesisClient getClient() + { + return client; + } + + public DescribeStreamRequest getDescribeStreamRequest() + { + return new DescribeStreamRequest(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisColumnHandle.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisColumnHandle.java new file mode 100644 index 000000000000..b47cb68f1734 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisColumnHandle.java @@ -0,0 +1,204 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.type.Type; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; +import com.google.common.primitives.Ints; + +/** + * Kinesis version of ConnectoColumnHandle. Keeps all the data of the columns of table formed from data received + * from kinesis. + * + */ +public class KinesisColumnHandle + implements ColumnHandle, Comparable +{ + private String connectorId; + private final int ordinalPosition; + + /** + * Column Name + */ + private final String name; + + /** + * Column type + */ + private final Type type; + + /** + * Mapping hint for the decoder. Can be null. + */ + private final String mapping; + + /** + * Data format to use (selects the decoder). Can be null. + */ + private final String dataFormat; + + /** + * Additional format hint for the selected decoder. Selects a decoder subtype (e.g. which timestamp decoder). + */ + private final String formatHint; + + /** + * True if the column should be hidden. + */ + private final boolean hidden; + + /** + * True if the column is internal to the connector and not defined by a topic definition. + */ + private final boolean internal; + + @JsonCreator + public KinesisColumnHandle( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("ordinalPosition") int ordinalPosition, + @JsonProperty("name") String name, + @JsonProperty("type") Type type, + @JsonProperty("mapping") String mapping, + @JsonProperty("dataFormat") String dataFormat, + @JsonProperty("formatHint") String formatHint, + @JsonProperty("hidden") boolean hidden, + @JsonProperty("internal") boolean internal) + { + this.connectorId = checkNotNull(connectorId, "connectorId is null"); + this.ordinalPosition = ordinalPosition; + this.name = checkNotNull(name, "name is null"); + this.type = checkNotNull(type, "type is null"); + this.mapping = mapping; + this.dataFormat = dataFormat; + this.formatHint = formatHint; + this.hidden = hidden; + this.internal = internal; + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public int getOrdinalPosition() + { + return ordinalPosition; + } + + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public Type getType() + { + return type; + } + + @JsonProperty + public String getMapping() + { + return mapping; + } + + @JsonProperty + public String getDataFormat() + { + return dataFormat; + } + + @JsonProperty + public String getFormatHint() + { + return formatHint; + } + + @JsonProperty + public boolean isHidden() + { + return hidden; + } + + @JsonProperty + public boolean isInternal() + { + return internal; + } + + ColumnMetadata getColumnMetadata() + { + return new ColumnMetadata(name, type, ordinalPosition, false, null, hidden); + } + + @Override + public int hashCode() + { + return Objects.hashCode(connectorId, ordinalPosition, name, type, mapping, dataFormat, formatHint, hidden, internal); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KinesisColumnHandle other = (KinesisColumnHandle) obj; + return Objects.equal(this.connectorId, other.connectorId) && + Objects.equal(this.ordinalPosition, other.ordinalPosition) && + Objects.equal(this.name, other.name) && + Objects.equal(this.type, other.type) && + Objects.equal(this.mapping, other.mapping) && + Objects.equal(this.dataFormat, other.dataFormat) && + Objects.equal(this.formatHint, other.formatHint) && + Objects.equal(this.hidden, other.hidden) && + Objects.equal(this.internal, other.internal); + } + + @Override + public int compareTo(KinesisColumnHandle otherHandle) + { + return Ints.compare(this.getOrdinalPosition(), otherHandle.getOrdinalPosition()); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("ordinalPosition", ordinalPosition) + .add("name", name) + .add("type", type) + .add("mapping", mapping) + .add("dataFormat", dataFormat) + .add("formatHint", formatHint) + .add("hidden", hidden) + .add("internal", internal) + .toString(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnector.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnector.java new file mode 100644 index 000000000000..24cd2cbeb2c1 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnector.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.facebook.presto.spi.Connector; +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.ConnectorIndexResolver; +import com.facebook.presto.spi.ConnectorMetadata; +import com.facebook.presto.spi.ConnectorPageSourceProvider; +import com.facebook.presto.spi.ConnectorRecordSetProvider; +import com.facebook.presto.spi.ConnectorRecordSinkProvider; +import com.facebook.presto.spi.ConnectorSplitManager; +import com.google.inject.Inject; + +public class KinesisConnector + implements Connector +{ + private final KinesisMetadata metadata; + private final KinesisSplitManager splitManager; + private final KinesisRecordSetProvider recordSetProvider; + private final KinesisHandleResolver handleResolver; + + @Inject + public KinesisConnector( + KinesisHandleResolver handleResolver, + KinesisMetadata metadata, + KinesisSplitManager splitManager, + KinesisRecordSetProvider recordSetProvider) + { + this.handleResolver = checkNotNull(handleResolver, "handleResolver is null"); + this.metadata = checkNotNull(metadata, "metadata is null"); + this.splitManager = checkNotNull(splitManager, "splitManager is null"); + this.recordSetProvider = checkNotNull(recordSetProvider, "recordSetProvider is null"); + } + + @Override + public ConnectorHandleResolver getHandleResolver() + { + return handleResolver; + } + + @Override + public ConnectorMetadata getMetadata() + { + return metadata; + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return splitManager; + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() + { + throw new UnsupportedOperationException(); + } + + @Override + public ConnectorRecordSetProvider getRecordSetProvider() + { + return recordSetProvider; + } + + @Override + public ConnectorRecordSinkProvider getRecordSinkProvider() + { + throw new UnsupportedOperationException(); + } + + @Override + public ConnectorIndexResolver getIndexResolver() + { + throw new UnsupportedOperationException(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorConfig.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorConfig.java new file mode 100644 index 000000000000..80e3755adf5c --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorConfig.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import io.airlift.configuration.Config; + +import java.io.File; +import java.util.Set; + +import javax.validation.constraints.NotNull; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; + +/** + * This Class handles all the configuration settings that is stored in /etc/catalog/kinesis.properties file + * + */ +public class KinesisConnectorConfig +{ + /** + * Set of tables known to this connector. For each table, a description file may be present in the catalog folder which describes columns for the given topic. + */ + private Set tableNames = ImmutableSet.of(); + + /** + * The schema name to use in the connector. + */ + private String defaultSchema = "default"; + + /** + * Folder holding the JSON description files for Kafka topics. + */ + private File tableDescriptionDir = new File("etc/kinesis/"); + + /** + * Whether internal columns are shown in table metadata or not. Default is no. + */ + private boolean hideInternalColumns = true; + + private String accessKey = null; + + private String secretKey = null; + + @NotNull + public File getTableDescriptionDir() + { + return tableDescriptionDir; + } + + @Config("kinesis.table-description-dir") + public KinesisConnectorConfig setTableDescriptionDir(File tableDescriptionDir) + { + this.tableDescriptionDir = tableDescriptionDir; + return this; + } + + public boolean isHideInternalColumns() + { + return hideInternalColumns; + } + + @Config("kinesis.hide-internal-columns") + public KinesisConnectorConfig setHideInternalColumns(boolean hideInternalColumns) + { + this.hideInternalColumns = hideInternalColumns; + return this; + } + + @NotNull + public Set getTableNames() + { + return tableNames; + } + + @Config("kinesis.table-names") + public KinesisConnectorConfig setTableNames(String tableNames) + { + this.tableNames = ImmutableSet.copyOf(Splitter.on(',').omitEmptyStrings().trimResults().split(tableNames)); + return this; + } + + @NotNull + public String getDefaultSchema() + { + return defaultSchema; + } + + @Config("kinesis.default-schema") + public KinesisConnectorConfig setDefaultSchema(String defaultSchema) + { + this.defaultSchema = defaultSchema; + return this; + } + + @Config("kinesis.access-key") + public KinesisConnectorConfig setAccessKey(String accessKey) + { + this.accessKey = accessKey; + return this; + } + + public String getAccessKey() + { + return this.accessKey; + } + + @Config("kinesis.secret-key") + public KinesisConnectorConfig setSecretKey(String secretKey) + { + this.secretKey = secretKey; + return this; + } + + public String getSecretKey() + { + return this.secretKey; + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorFactory.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorFactory.java new file mode 100644 index 000000000000..a9602adfb0b6 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorFactory.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.Preconditions.checkNotNull; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.json.JsonModule; + +import java.util.Map; +import java.util.Optional; + +import com.facebook.presto.spi.Connector; +import com.facebook.presto.spi.ConnectorFactory; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.base.Supplier; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Scopes; +import com.google.inject.TypeLiteral; +import com.google.inject.Module; +import com.google.inject.name.Names; + +/** + * + * This factory class creates the KinesisConnector during server start and binds all the dependency + * by calling create() method. + */ +public class KinesisConnectorFactory + implements ConnectorFactory +{ + private TypeManager typeManager; + private Optional>> tableDescriptionSupplier = Optional.empty(); + private Map optionalConfig = ImmutableMap.of(); + + KinesisConnectorFactory(TypeManager typeManager, + Optional>> tableDescriptionSupplier, + Map optionalConfig) + { + this.typeManager = checkNotNull(typeManager, "typeManager is null"); + this.optionalConfig = checkNotNull(optionalConfig, "optionalConfig is null"); + this.tableDescriptionSupplier = checkNotNull(tableDescriptionSupplier, "tableDescriptionSupplier is null"); + } + + @Override + public String getName() + { + return "kinesis"; + } + + @Override + public Connector create(String connectorId, Map config) + { + checkNotNull(connectorId, "connectorId is null"); + checkNotNull(config, "config is null"); + + try { + Bootstrap app = new Bootstrap( + new JsonModule(), + new KinesisConnectorModule(), + new Module() + { + @Override + public void configure(Binder binder) + { + binder.bindConstant().annotatedWith(Names.named("connectorId")).to(connectorId); + binder.bind(TypeManager.class).toInstance(typeManager); + + if (tableDescriptionSupplier.isPresent()) { + binder.bind(new TypeLiteral>>() {}).toInstance(tableDescriptionSupplier.get()); + } + else { + binder.bind(new TypeLiteral>>() {}).to(KinesisTableDescriptionSupplier.class).in(Scopes.SINGLETON); + } + } + } + ); + + Injector injector = app.strictConfig() + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .setOptionalConfigurationProperties(optionalConfig) + .initialize(); + + return injector.getInstance(KinesisConnector.class); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorModule.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorModule.java new file mode 100644 index 000000000000..647bd4dc7e88 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisConnectorModule.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static io.airlift.configuration.ConfigurationModule.bindConfig; +import static io.airlift.json.JsonBinder.jsonBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; + +import javax.inject.Inject; + +import com.facebook.presto.kinesis.decoder.KinesisDecoderModule; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; +import com.google.inject.multibindings.Multibinder; + +public class KinesisConnectorModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(KinesisConnector.class).in(Scopes.SINGLETON); + + binder.bind(KinesisHandleResolver.class).in(Scopes.SINGLETON); + binder.bind(KinesisMetadata.class).in(Scopes.SINGLETON); + binder.bind(KinesisSplitManager.class).in(Scopes.SINGLETON); + binder.bind(KinesisRecordSetProvider.class).in(Scopes.SINGLETON); + + binder.bind(KinesisClientManager.class).in(Scopes.SINGLETON); + + bindConfig(binder).to(KinesisConnectorConfig.class); + + jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); + jsonCodecBinder(binder).bindJsonCodec(KinesisStreamDescription.class); + + binder.install(new KinesisDecoderModule()); + + for (KinesisInternalFieldDescription internalFieldDescription : KinesisInternalFieldDescription.getInternalFields()) { + bindInternalColumn(binder, internalFieldDescription); + } + } + + private static void bindInternalColumn(Binder binder, KinesisInternalFieldDescription fieldDescription) + { + Multibinder fieldDescriptionBinder = Multibinder.newSetBinder(binder, KinesisInternalFieldDescription.class); + fieldDescriptionBinder.addBinding().toInstance(fieldDescription); + } + + public static final class TypeDeserializer + extends FromStringDeserializer + { + private static final long serialVersionUID = 1L; + + private final TypeManager typeManager; + + @Inject + public TypeDeserializer(TypeManager typeManager) + { + super(Type.class); + this.typeManager = checkNotNull(typeManager, "typeManager is null"); + } + + @Override + protected Type _deserialize(String value, DeserializationContext context) + { + Type type = typeManager.getType(parseTypeSignature(value)); + checkArgument(type != null, "Unknown type %s", value); + return type; + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisErrorCode.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisErrorCode.java new file mode 100644 index 000000000000..e810394b3263 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisErrorCode.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import com.facebook.presto.spi.ErrorCode; +import com.facebook.presto.spi.ErrorCodeSupplier; + +/** + * Kinesis connector specific error codes. + */ +public enum KinesisErrorCode + implements ErrorCodeSupplier +{ + // Connectors can use error codes starting at EXTERNAL + + /** + * A requested data conversion is not supported. + */ + KINESIS_CONVERSION_NOT_SUPPORTED(0x0200_0000), + KINESIS_SPLIT_ERROR(0x0200_0001); + + private final ErrorCode errorCode; + + KinesisErrorCode(int code) + { + errorCode = new ErrorCode(code, name()); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisFieldValueProvider.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisFieldValueProvider.java new file mode 100644 index 000000000000..5ee1acd7f84e --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisFieldValueProvider.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import io.airlift.slice.Slice; + +import com.facebook.presto.spi.PrestoException; + +import static com.facebook.presto.kinesis.KinesisErrorCode.KINESIS_CONVERSION_NOT_SUPPORTED; + +public abstract class KinesisFieldValueProvider +{ + public abstract boolean accept(KinesisColumnHandle columnHanle); + + public boolean getBoolean() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to boolean not supported"); + } + + public long getLong() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to long not supported"); + } + + public double getDouble() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to double not supported"); + } + + public Slice getSlice() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to Slice not supported"); + } + + public abstract boolean isNull(); +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisHandleResolver.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisHandleResolver.java new file mode 100644 index 000000000000..00a6d70b7442 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisHandleResolver.java @@ -0,0 +1,142 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import javax.inject.Inject; + +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.ConnectorIndexHandle; +import com.facebook.presto.spi.ConnectorInsertTableHandle; +import com.facebook.presto.spi.ConnectorOutputTableHandle; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.google.inject.name.Named; + +public class KinesisHandleResolver + implements ConnectorHandleResolver +{ + private final String connectorId; + + @Inject + KinesisHandleResolver(@Named("connectorId") String connectorId, + KinesisConnectorConfig kinesisConnectorConfig) + { + this.connectorId = checkNotNull(connectorId, "connectorId is null"); + checkNotNull(kinesisConnectorConfig, "kinesisConfig is null"); + } + + @Override + public boolean canHandle(ConnectorTableHandle tableHandle) + { + return tableHandle != null && tableHandle instanceof KinesisTableHandle && connectorId.equals(((KinesisTableHandle) tableHandle).getConnectorId()); + } + + @Override + public boolean canHandle(ColumnHandle columnHandle) + { + return columnHandle != null && columnHandle instanceof KinesisColumnHandle && connectorId.equals(((KinesisColumnHandle) columnHandle).getConnectorId()); + } + + @Override + public boolean canHandle(ConnectorSplit split) + { + return split != null && split instanceof KinesisSplit && connectorId.equals(((KinesisSplit) split).getConnectorId()); + } + + @Override + public boolean canHandle(ConnectorIndexHandle indexHandle) + { + return false; + } + + @Override + public boolean canHandle(ConnectorOutputTableHandle tableHandle) + { + return false; + } + + @Override + public boolean canHandle(ConnectorInsertTableHandle tableHandle) + { + return false; + } + + @Override + public Class getTableHandleClass() + { + return KinesisTableHandle.class; + } + + @Override + public Class getColumnHandleClass() + { + return KinesisColumnHandle.class; + } + + @Override + public Class getIndexHandleClass() + { + throw new UnsupportedOperationException(); + } + + @Override + public Class getSplitClass() + { + return KinesisSplit.class; + } + + @Override + public Class getOutputTableHandleClass() + { + throw new UnsupportedOperationException(); + } + + @Override + public Class getInsertTableHandleClass() + { + throw new UnsupportedOperationException(); + } + + KinesisTableHandle convertTableHandle(ConnectorTableHandle tableHandle) + { + checkNotNull(tableHandle, "tableHandle is null"); + checkArgument(tableHandle instanceof KinesisTableHandle, "tableHandle is not an instance of KinesisTableHandle"); + KinesisTableHandle kinesisTableHandle = (KinesisTableHandle) tableHandle; + checkArgument(kinesisTableHandle.getConnectorId().equals(connectorId), "tableHandle is not for this connector"); + + return kinesisTableHandle; + } + + KinesisColumnHandle convertColumnHandle(ColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + checkArgument(columnHandle instanceof KinesisColumnHandle, "columnHandle is not an instance of KinesisColumnHandle"); + KinesisColumnHandle kinesisColumnHandle = (KinesisColumnHandle) columnHandle; + checkArgument(kinesisColumnHandle.getConnectorId().equals(connectorId), "columnHandle is not for this connector"); + return kinesisColumnHandle; + } + + KinesisSplit convertSplit(ConnectorSplit split) + { + checkNotNull(split, "split is null"); + checkArgument(split instanceof KinesisSplit, "split is not an instance of KinesisSplit"); + KinesisSplit kinesisSplit = (KinesisSplit) split; + checkArgument(kinesisSplit.getConnectorId().equals(connectorId), "split is not for this connector"); + return kinesisSplit; + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisInternalFieldDescription.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisInternalFieldDescription.java new file mode 100644 index 000000000000..9f79369c2bbe --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisInternalFieldDescription.java @@ -0,0 +1,233 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Strings.isNullOrEmpty; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.util.Set; + +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableSet; + +public class KinesisInternalFieldDescription +{ + public static final KinesisInternalFieldDescription SHARD_ID_FIELD = new KinesisInternalFieldDescription("_shard_id", VarcharType.VARCHAR, "Shard Id"); + + public static final KinesisInternalFieldDescription SHARD_SEQUENCE_ID_FIELD = new KinesisInternalFieldDescription("_shard_sequence_id", VarcharType.VARCHAR, "sequence id of message within the shard"); + + public static final KinesisInternalFieldDescription SEGMENT_START_FIELD = new KinesisInternalFieldDescription("_segment_start", VarcharType.VARCHAR, "Segment start sequence id"); + + public static final KinesisInternalFieldDescription SEGMENT_END_FIELD = new KinesisInternalFieldDescription("_segment_end", VarcharType.VARCHAR, "Segment end sequence id"); + + public static final KinesisInternalFieldDescription SEGMENT_COUNT_FIELD = new KinesisInternalFieldDescription("_segment_count", BigintType.BIGINT, "Running message count per segment"); + + public static final KinesisInternalFieldDescription MESSAGE_VALID_FIELD = new KinesisInternalFieldDescription("_message_valid", BooleanType.BOOLEAN, "Message data is valid"); + + public static final KinesisInternalFieldDescription MESSAGE_FIELD = new KinesisInternalFieldDescription("_message", VarcharType.VARCHAR, "Message text"); + + public static final KinesisInternalFieldDescription MESSAGE_LENGTH_FIELD = new KinesisInternalFieldDescription("_message_length", BigintType.BIGINT, "Total number of message bytes"); + + public static final KinesisInternalFieldDescription PARTITION_KEY_FIELD = new KinesisInternalFieldDescription("_partition_key", VarcharType.VARCHAR, "Key text"); + + public static Set getInternalFields() + { + return ImmutableSet.of(SHARD_ID_FIELD, SHARD_SEQUENCE_ID_FIELD, + SEGMENT_START_FIELD, SEGMENT_END_FIELD, SEGMENT_COUNT_FIELD, + PARTITION_KEY_FIELD, MESSAGE_FIELD, MESSAGE_VALID_FIELD, MESSAGE_LENGTH_FIELD); + } + + private final String name; + private final Type type; + private final String comment; + + KinesisInternalFieldDescription( + String name, + Type type, + String comment) + { + checkArgument(!isNullOrEmpty(name), "name is null or is empty"); + this.name = name; + this.type = checkNotNull(type, "type is null"); + this.comment = checkNotNull(comment, "comment is null"); + } + + public String getName() + { + return name; + } + + public Type getType() + { + return type; + } + + KinesisColumnHandle getColumnHandle(String connectorId, int index, boolean hidden) + { + return new KinesisColumnHandle(connectorId, + index, + getName(), + getType(), + null, + null, + null, + hidden, + true); + } + + ColumnMetadata getColumnMetadata(int index, boolean hidden) + { + return new ColumnMetadata(name, type, index, false, comment, hidden); + } + + public KinesisFieldValueProvider forBooleanValue(boolean value) + { + return new BooleanKinesisFieldValueProvider(value); + } + + public KinesisFieldValueProvider forLongValue(long value) + { + return new LongKinesisFieldValueProvider(value); + } + + public KinesisFieldValueProvider forByteValue(byte[] value) + { + return new BytesKinesisFieldValueProvider(value); + } + + @Override + public int hashCode() + { + return Objects.hashCode(name, type); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KinesisInternalFieldDescription other = (KinesisInternalFieldDescription) obj; + return Objects.equal(this.name, other.name) && + Objects.equal(this.type, other.type); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("name", name) + .add("type", type) + .toString(); + } + + public class BooleanKinesisFieldValueProvider + extends KinesisFieldValueProvider + { + private final boolean value; + + private BooleanKinesisFieldValueProvider(boolean value) + { + this.value = value; + } + + @Override + public boolean accept(KinesisColumnHandle columnHandle) + { + return columnHandle.getName().equals(name); + } + + @Override + public boolean getBoolean() + { + return value; + } + + @Override + public boolean isNull() + { + return false; + } + } + + public class LongKinesisFieldValueProvider + extends KinesisFieldValueProvider + { + private final long value; + + private LongKinesisFieldValueProvider(long value) + { + this.value = value; + } + + @Override + public boolean accept(KinesisColumnHandle columnHandle) + { + return columnHandle.getName().equals(name); + } + + @Override + public long getLong() + { + return value; + } + + @Override + public boolean isNull() + { + return false; + } + } + + public class BytesKinesisFieldValueProvider + extends KinesisFieldValueProvider + { + private final byte[] value; + private BytesKinesisFieldValueProvider(byte[] value) + { + this.value = value; + } + + @Override + public boolean accept(KinesisColumnHandle columnHandle) + { + return columnHandle.getName().equals(name); + } + + @Override + public Slice getSlice() + { + return isNull() ? Slices.EMPTY_SLICE : Slices.wrappedBuffer(value); + } + + @Override + public boolean isNull() + { + return value == null || value.length == 0; + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisMetadata.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisMetadata.java new file mode 100644 index 000000000000..f37ee1267f7b --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisMetadata.java @@ -0,0 +1,239 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.Preconditions.checkNotNull; +import io.airlift.log.Logger; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.facebook.presto.kinesis.decoder.dummy.DummyKinesisRowDecoder; +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.ReadOnlyConnectorMetadata; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.SchemaTablePrefix; +import com.facebook.presto.spi.TableNotFoundException; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Inject; +import com.google.inject.name.Named; + +public class KinesisMetadata + extends ReadOnlyConnectorMetadata +{ + private static final Logger log = Logger.get(KinesisMetadata.class); + + private final String connectorId; + private final KinesisConnectorConfig kinesisConnectorConfig; + private final KinesisHandleResolver handleResolver; + + private final Supplier> kinesisTableDescriptionSupplier; + private final Set internalFieldDescriptions; + + @Inject + KinesisMetadata(@Named("connectorId") String connectorId, + KinesisConnectorConfig kinesisConnectorConfig, + KinesisHandleResolver handleResolver, + Supplier> kinesisTableDescriptionSupplier, + Set internalFieldDescriptions) + { + this.connectorId = checkNotNull(connectorId, "connectorId is null"); + this.kinesisConnectorConfig = checkNotNull(kinesisConnectorConfig, "kinesisConfig is null"); + this.handleResolver = checkNotNull(handleResolver, "handleResolver is null"); + + log.debug("Loading kinesis table definitions from %s", kinesisConnectorConfig.getTableDescriptionDir().getAbsolutePath()); + + this.kinesisTableDescriptionSupplier = Suppliers.memoize(kinesisTableDescriptionSupplier); + this.internalFieldDescriptions = checkNotNull(internalFieldDescriptions, "internalFieldDescriptions is null"); + } + + @Override + public List listSchemaNames(ConnectorSession session) + { + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (SchemaTableName tableName : getDefinedTables().keySet()) { + builder.add(tableName.getSchemaName()); + } + return ImmutableList.copyOf(builder.build()); + } + + @Override + public KinesisTableHandle getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) + { + KinesisStreamDescription table = getDefinedTables().get(schemaTableName); + if (table == null) { + throw new TableNotFoundException(schemaTableName); + } + + return new KinesisTableHandle(connectorId, + schemaTableName.getSchemaName(), + schemaTableName.getTableName(), + table.getStreamName(), + getDataFormat(table.getMessage())); + } + + @Override + public ConnectorTableMetadata getTableMetadata(ConnectorTableHandle tableHandle) + { + KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle); + return getTableMetadata(kinesisTableHandle.toSchemaTableName()); + } + + @Override + public List listTables(ConnectorSession session, String schemaNameOrNull) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (SchemaTableName tableName : getDefinedTables().keySet()) { + if (schemaNameOrNull == null || tableName.getSchemaName().equals(schemaNameOrNull)) { + builder.add(tableName); + } + } + + return builder.build(); + } + + @Override + public ColumnHandle getSampleWeightColumnHandle(ConnectorTableHandle tableHandle) + { + return null; + } + + @SuppressWarnings("ValueOfIncrementOrDecrementUsed") + @Override + public Map getColumnHandles(ConnectorTableHandle tableHandle) + { + KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle); + + KinesisStreamDescription kinesisStreamDescription = getDefinedTables().get(kinesisTableHandle.toSchemaTableName()); + if (kinesisStreamDescription == null) { + throw new TableNotFoundException(kinesisTableHandle.toSchemaTableName()); + } + + ImmutableMap.Builder columnHandles = ImmutableMap.builder(); + + int index = 0; + /* KinesisStreamFieldGroup key = kinesisStreamDescription.getPartitionKey(); + if (key != null) { + List fields = key.getFields(); + if (fields != null) { + for (KinesisStreamFieldDescription kinesisStreamFieldDescription : fields) { + columnHandles.put(kinesisStreamFieldDescription.getName(), kinesisStreamFieldDescription.getColumnHandle(connectorId, true, index++)); + } + } + }*/ + + KinesisStreamFieldGroup message = kinesisStreamDescription.getMessage(); + if (message != null) { + List fields = message.getFields(); + if (fields != null) { + for (KinesisStreamFieldDescription kinesisStreamFieldDescription : fields) { + columnHandles.put(kinesisStreamFieldDescription.getName(), kinesisStreamFieldDescription.getColumnHandle(connectorId, index++)); + } + } + } + + for (KinesisInternalFieldDescription kinesisInternalFieldDescription : internalFieldDescriptions) { + columnHandles.put(kinesisInternalFieldDescription.getName(), kinesisInternalFieldDescription.getColumnHandle(connectorId, index++, kinesisConnectorConfig.isHideInternalColumns())); + } + + return columnHandles.build(); + } + + @SuppressWarnings("ValueOfIncrementOrDecrementUsed") + @Override + public ColumnMetadata getColumnMetadata(ConnectorTableHandle tableHandle, ColumnHandle columnHandle) + { + handleResolver.convertTableHandle(tableHandle); + KinesisColumnHandle kinesisColumnHandle = handleResolver.convertColumnHandle(columnHandle); + + return kinesisColumnHandle.getColumnMetadata(); + } + + @Override + public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + { + checkNotNull(prefix, "prefix is null"); + + ImmutableMap.Builder> columns = ImmutableMap.builder(); + + // what if prefix.getTableName == null + List tableNames = prefix.getSchemaName() == null ? listTables(session, null) : ImmutableList.of(new SchemaTableName(prefix.getSchemaName(), prefix.getTableName())); + + for (SchemaTableName tableName : tableNames) { + ConnectorTableMetadata tableMetadata = getTableMetadata(tableName); + if (tableMetadata != null) { + columns.put(tableName, tableMetadata.getColumns()); + } + } + return columns.build(); + } + + private static String getDataFormat(KinesisStreamFieldGroup fieldGroup) + { + return (fieldGroup == null) ? DummyKinesisRowDecoder.NAME : fieldGroup.getDataFormat(); + } + + @VisibleForTesting + Map getDefinedTables() + { + return kinesisTableDescriptionSupplier.get(); + } + + private ConnectorTableMetadata getTableMetadata(SchemaTableName schemaTableName) + { + KinesisStreamDescription table = getDefinedTables().get(schemaTableName); + if (table == null) { + throw new TableNotFoundException(schemaTableName); + } + + ImmutableList.Builder builder = ImmutableList.builder(); + int index = 0; + + /*KinesisStreamFieldGroup key = table.getPartitionKey(); + if (key != null) { + List fields = key.getFields(); + if (fields != null) { + for (KinesisStreamFieldDescription fieldDescription : fields) { + builder.add(fieldDescription.getColumnMetadata(index++)); + } + } + }*/ + + KinesisStreamFieldGroup message = table.getMessage(); + if (message != null) { + List fields = message.getFields(); + if (fields != null) { + for (KinesisStreamFieldDescription fieldDescription : fields) { + builder.add(fieldDescription.getColumnMetadata(index++)); + } + } + } + + for (KinesisInternalFieldDescription fieldDescription : internalFieldDescriptions) { + builder.add(fieldDescription.getColumnMetadata(index++, kinesisConnectorConfig.isHideInternalColumns())); + } + + return new ConnectorTableMetadata(schemaTableName, builder.build()); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisPlugin.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisPlugin.java new file mode 100644 index 000000000000..36bca3d6f28b --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisPlugin.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import javax.inject.Inject; + +import com.facebook.presto.spi.ConnectorFactory; +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.type.TypeManager; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +/** + * + * Kinesis version of Presto Plugin interface. This class calls getServices method to create KinesisConnectorFactory + * and KinesisConnector during server start, + */ +public class KinesisPlugin + implements Plugin +{ + private TypeManager typeManager; + private Optional>> tableDescriptionSupplier = Optional.empty(); + private Map optionalConfig = ImmutableMap.of(); + + @Override + public synchronized void setOptionalConfig(Map optionalConfig) + { + this.optionalConfig = ImmutableMap.copyOf(checkNotNull(optionalConfig, "optionalConfig is null")); + } + + @Inject + public synchronized void setTypeManager(TypeManager typeManager) + { + this.typeManager = checkNotNull(typeManager, "typeManager is null"); + } + + @VisibleForTesting + public synchronized void setTableDescriptionSupplier(Supplier> tableDescriptionSupplier) + { + this.tableDescriptionSupplier = Optional.of(checkNotNull(tableDescriptionSupplier, "tableDescriptionSupplier is null")); + } + + @Override + public synchronized List getServices(Class type) + { + if (type == ConnectorFactory.class) { + return ImmutableList.of(type.cast(new KinesisConnectorFactory(typeManager, tableDescriptionSupplier, optionalConfig))); + } + return ImmutableList.of(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSet.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSet.java new file mode 100644 index 000000000000..114d697fa5e4 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSet.java @@ -0,0 +1,318 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import io.airlift.log.Logger; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.amazonaws.services.kinesis.model.DescribeStreamRequest; +import com.amazonaws.services.kinesis.model.GetRecordsRequest; +import com.amazonaws.services.kinesis.model.GetRecordsResult; +import com.amazonaws.services.kinesis.model.GetShardIteratorRequest; +import com.amazonaws.services.kinesis.model.GetShardIteratorResult; +import com.amazonaws.services.kinesis.model.Record; +import com.amazonaws.services.kinesis.model.ResourceNotFoundException; +import com.amazonaws.services.kinesis.model.StreamDescription; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.kinesis.decoder.KinesisRowDecoder; +import com.facebook.presto.spi.RecordCursor; +import com.facebook.presto.spi.RecordSet; +import com.facebook.presto.spi.type.Type; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +public class KinesisRecordSet + implements RecordSet +{ + private static final Logger log = Logger.get(KinesisRecordSet.class); + + private static final byte [] EMPTY_BYTE_ARRAY = new byte [0]; + + private final KinesisSplit split; + private final KinesisClientManager clientManager; + + private final KinesisRowDecoder messageDecoder; + private final Map> messageFieldDecoders; + + private final List columnHandles; + private final List columnTypes; + + private final Set globalInternalFieldValueProviders; + + KinesisRecordSet(KinesisSplit split, + KinesisClientManager clientManager, + List columnHandles, + KinesisRowDecoder messageDecoder, + Map> messageFieldDecoders) + { + this.split = checkNotNull(split, "split is null"); + this.globalInternalFieldValueProviders = ImmutableSet.of( + KinesisInternalFieldDescription.SHARD_ID_FIELD.forByteValue(split.getShardId().getBytes()), + KinesisInternalFieldDescription.SEGMENT_START_FIELD.forByteValue(split.getStart().getBytes())); + + this.clientManager = checkNotNull(clientManager, "clientManager is null"); + + this.messageDecoder = checkNotNull(messageDecoder, "rowDecoder is null"); + this.messageFieldDecoders = checkNotNull(messageFieldDecoders, "messageFieldDecoders is null"); + + this.columnHandles = checkNotNull(columnHandles, "columnHandles is null"); + + ImmutableList.Builder typeBuilder = ImmutableList.builder(); + + for (KinesisColumnHandle handle : columnHandles) { + typeBuilder.add(handle.getType()); + } + + this.columnTypes = typeBuilder.build(); + } + @Override + public List getColumnTypes() + { + return columnTypes; + } + + @Override + public RecordCursor cursor() + { + return new KinesisRecordCursor(); + } + + public class KinesisRecordCursor + implements RecordCursor + { + private long totalBytes; + private long totalMessages; + + private String shardIterator; + private KinesisFieldValueProvider[] fieldValueProviders; + private List kinesisRecords; + private Iterator listIterator; + private GetRecordsRequest getRecordsRequest; + private GetRecordsResult getRecordsResult; + + @Override + public long getTotalBytes() + { + return totalBytes; + } + + @Override + public long getCompletedBytes() + { + return totalBytes; + } + + @Override + public long getReadTimeNanos() + { + return 0; + } + + @Override + public Type getType(int field) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + return columnHandles.get(field).getType(); + } + + @Override + public boolean advanceNextPosition() + { + try { + if (shardIterator == null) { + getIterator(); + if (getKinesisRecords() == false) { + log.debug("No more records in shard to read."); + return false; + } + } + + while (true) { + log.debug("Reading data from shardIterator %s", shardIterator); + while (listIterator.hasNext()) { + return nextRow(); + } + + shardIterator = getRecordsResult.getNextShardIterator(); + + if (shardIterator == null) { + log.debug("Shard closed"); + return false; + } + else { + if (getKinesisRecords() == false) { + log.debug("No more records in shard to read."); + return false; + } + } + } + } + + catch (ResourceNotFoundException e) { + log.debug("Stream %s not active.", split.getStreamName()); + return false; + } + } + + private String checkStreamStatus(String streamName) + { + DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); + describeStreamRequest.setStreamName(streamName); + + StreamDescription streamDescription = clientManager.getClient().describeStream(describeStreamRequest).getStreamDescription(); + return streamDescription.getStreamStatus(); + } + + private boolean getKinesisRecords() + throws ResourceNotFoundException + { + getRecordsRequest = new GetRecordsRequest(); + getRecordsRequest.setShardIterator(shardIterator); + getRecordsRequest.setLimit(25); + + if (checkStreamStatus(split.getStreamName()).equals("ACTIVE") == false) { + throw new ResourceNotFoundException("Stream not Active"); + } + + getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest); + kinesisRecords = getRecordsResult.getRecords(); + if (kinesisRecords.isEmpty()) { + return false; + } + listIterator = kinesisRecords.iterator(); + return true; + } + + private boolean nextRow() + { + Record currentRecord = listIterator.next(); + String partitionKey = currentRecord.getPartitionKey(); + log.debug("Reading record with partition key %s", partitionKey); + + byte[] messageData = EMPTY_BYTE_ARRAY; + ByteBuffer message = currentRecord.getData(); + if (message != null) { + messageData = new byte[message.remaining()]; + message.get(messageData); + } + totalBytes += messageData.length; + totalMessages++; + + log.debug("Fetching %d bytes from current record. %d messages read so far", messageData.length, totalMessages); + + Set fieldValueProviders = new HashSet<>(); + + fieldValueProviders.addAll(globalInternalFieldValueProviders); + fieldValueProviders.add(KinesisInternalFieldDescription.SEGMENT_COUNT_FIELD.forLongValue(totalMessages)); + fieldValueProviders.add(KinesisInternalFieldDescription.SHARD_SEQUENCE_ID_FIELD.forByteValue(currentRecord.getSequenceNumber().getBytes())); + fieldValueProviders.add(KinesisInternalFieldDescription.MESSAGE_FIELD.forByteValue(messageData)); + fieldValueProviders.add(KinesisInternalFieldDescription.MESSAGE_LENGTH_FIELD.forLongValue(messageData.length)); + fieldValueProviders.add(KinesisInternalFieldDescription.MESSAGE_VALID_FIELD.forBooleanValue(messageDecoder.decodeRow(messageData, fieldValueProviders, columnHandles, messageFieldDecoders))); + fieldValueProviders.add(KinesisInternalFieldDescription.PARTITION_KEY_FIELD.forByteValue(partitionKey.getBytes())); + + this.fieldValueProviders = new KinesisFieldValueProvider[columnHandles.size()]; + + for (int i = 0; i < columnHandles.size(); i++) { + for (KinesisFieldValueProvider fieldValueProvider : fieldValueProviders) { + if (fieldValueProvider.accept(columnHandles.get(i))) { + this.fieldValueProviders[i] = fieldValueProvider; + break; + } + } + } + return true; + } + + @Override + public boolean getBoolean(int field) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + + checkFieldType(field, boolean.class); + return isNull(field) ? false : fieldValueProviders[field].getBoolean(); + } + + @Override + public long getLong(int field) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + + checkFieldType(field, long.class); + return isNull(field) ? 0L : fieldValueProviders[field].getLong(); + } + + @Override + public double getDouble(int field) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + + checkFieldType(field, double.class); + return isNull(field) ? 0.0d : fieldValueProviders[field].getDouble(); + } + + @Override + public Slice getSlice(int field) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + + checkFieldType(field, Slice.class); + return isNull(field) ? Slices.EMPTY_SLICE : fieldValueProviders[field].getSlice(); + } + + @Override + public boolean isNull(int field) + { + checkArgument(field < columnHandles.size(), "Invalid field index"); + + return fieldValueProviders[field] == null || fieldValueProviders[field].isNull(); + } + + @Override + public void close() + { + } + + private void checkFieldType(int field, Class expected) + { + Class actual = getType(field).getJavaType(); + checkArgument(actual == expected, "Expected field %s to be type %s but is %s", field, expected, actual); + } + + private void getIterator() + throws ResourceNotFoundException + { + GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest(); + getShardIteratorRequest.setStreamName(split.getStreamName()); + getShardIteratorRequest.setShardId(split.getShardId()); + getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); + + if (checkStreamStatus(split.getStreamName()).equals("ACTIVE") == false) { + throw new ResourceNotFoundException("Stream not Active"); + } + + GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest); + shardIterator = getShardIteratorResult.getShardIterator(); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSetProvider.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSetProvider.java new file mode 100644 index 000000000000..c31e4d1556bf --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSetProvider.java @@ -0,0 +1,76 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.List; + +import com.facebook.presto.kinesis.decoder.KinesisDecoderRegistry; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.kinesis.decoder.KinesisRowDecoder; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorRecordSetProvider; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.RecordSet; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; + +public class KinesisRecordSetProvider + implements ConnectorRecordSetProvider +{ + private final KinesisHandleResolver handleResolver; + private final KinesisClientManager clientManager; + private final KinesisDecoderRegistry registry; + + @Inject + public KinesisRecordSetProvider(KinesisDecoderRegistry registry, + KinesisHandleResolver handleResolver, + KinesisClientManager clientManager) + { + this.registry = checkNotNull(registry, "registry is null"); + this.handleResolver = checkNotNull(handleResolver, "handleResolver is null"); + this.clientManager = checkNotNull(clientManager, "clientManager is null"); + } + + @Override + public RecordSet getRecordSet(ConnectorSplit split, List columns) + { + KinesisSplit kinesisSplit = handleResolver.convertSplit(split); + + ImmutableList.Builder handleBuilder = ImmutableList.builder(); + ImmutableMap.Builder> messageFieldDecoderBuilder = ImmutableMap.builder(); + + KinesisRowDecoder messageDecoder = registry.getRowDecoder(kinesisSplit.getMessageDataFormat()); + + for (ColumnHandle handle : columns) { + KinesisColumnHandle columnHandle = handleResolver.convertColumnHandle(handle); + handleBuilder.add(columnHandle); + + if (!columnHandle.isInternal()) { + KinesisFieldDecoder fieldDecoder = registry.getFieldDecoder(kinesisSplit.getMessageDataFormat(), + columnHandle.getType().getJavaType(), + columnHandle.getDataFormat()); + + messageFieldDecoderBuilder.put(columnHandle, fieldDecoder); + } + } + + ImmutableList handles = handleBuilder.build(); + ImmutableMap> messageFieldDecoders = messageFieldDecoderBuilder.build(); + + return new KinesisRecordSet(kinesisSplit, clientManager, handles, messageDecoder, messageFieldDecoders); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisShard.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisShard.java new file mode 100644 index 000000000000..9b09d25a4a0c --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisShard.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.MoreObjects.toStringHelper; + +import com.amazonaws.services.kinesis.model.Shard; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorPartition; +import com.facebook.presto.spi.TupleDomain; + +public class KinesisShard + implements ConnectorPartition +{ + private final String streamName; + private final Shard shard; + + public KinesisShard(String streamName, Shard shard) + { + this.streamName = streamName; + this.shard = shard; + } + + @Override + public String getPartitionId() + { + return shard.getShardId(); + } + + @Override + public TupleDomain getTupleDomain() + { + return TupleDomain.all(); + } + + public String getStreamName() + { + return streamName; + } + + public Shard getShard() + { + return shard; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("streamName", streamName) + .add("shard", shard) + .toString(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplit.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplit.java new file mode 100644 index 000000000000..950c91c76773 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplit.java @@ -0,0 +1,126 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.List; + +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.HostAddress; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +/** + * + * Kinesis vertion of ConnectorSplit. KinesisConnector fetch the data from kinesis stream and splits the big chunk to multiple split. + * By default, one shard data is one KinesisSplit. + * + */ +public class KinesisSplit + implements ConnectorSplit +{ + private final String connectorId; + private final String streamName; + private final String messageDataFormat; + private final String shardId; + private final String start; + private final String end; + + @JsonCreator + public KinesisSplit( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("streamName") String streamName, + @JsonProperty("messageDataFormat") String messageDataFormat, + @JsonProperty("shardId") String shardId, + @JsonProperty("start") String start, + @JsonProperty("end") String end) + { + this.connectorId = checkNotNull(connectorId, "connector id is null"); + this.streamName = checkNotNull(streamName, "streamName is null"); + this.messageDataFormat = checkNotNull(messageDataFormat, "messageDataFormat is null"); + this.shardId = shardId; + this.start = start; + this.end = end; + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public String getStart() + { + return start; + } + + @JsonProperty + public String getEnd() + { + return end; + } + + @JsonProperty + public String getStreamName() + { + return streamName; + } + + @JsonProperty + public String getMessageDataFormat() + { + return messageDataFormat; + } + + @JsonProperty + public String getShardId() + { + return shardId; + } + + @Override + public boolean isRemotelyAccessible() + { + return true; + } + + @Override + public List getAddresses() + { + return ImmutableList.of(); + } + + @Override + public Object getInfo() + { + return this; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("streamName", streamName) + .add("messageDataFormat", messageDataFormat) + .add("shardId", shardId) + .add("start", start) + .add("end", end) + .toString(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplitManager.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplitManager.java new file mode 100644 index 000000000000..6af64c1e0063 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplitManager.java @@ -0,0 +1,115 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import java.util.ArrayList; +import java.util.List; + +import javax.inject.Named; + +import com.amazonaws.services.kinesis.model.DescribeStreamRequest; +import com.amazonaws.services.kinesis.model.DescribeStreamResult; +import com.amazonaws.services.kinesis.model.Shard; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorPartition; +import com.facebook.presto.spi.ConnectorPartitionResult; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorSplitManager; +import com.facebook.presto.spi.ConnectorSplitSource; +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.FixedSplitSource; +import com.facebook.presto.spi.TupleDomain; +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; + +import static com.google.common.base.Preconditions.checkState; + +/** + * + * Split data chunk from kinesis Stream to multiple small chunks for parallelization and distribution to multiple Presto workers. + * By default, each shard of Kinesis Stream forms one Kinesis Split + * + */ +public class KinesisSplitManager + implements ConnectorSplitManager +{ + private final String connectorId; + private final KinesisHandleResolver handleResolver; + private final KinesisClientManager clientManager; + + @Inject + public KinesisSplitManager(@Named("connectorId") String connectorId, + KinesisHandleResolver handleResolver, + KinesisClientManager clientManager) + { + this.connectorId = connectorId; + this.handleResolver = handleResolver; + this.clientManager = clientManager; + } + + @Override + public ConnectorPartitionResult getPartitions(ConnectorTableHandle tableHandle, TupleDomain tupleDomain) + { + KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle); + + DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest(); + describeStreamRequest.setStreamName(kinesisTableHandle.getStreamName()); + + List shards = new ArrayList<>(); + ImmutableList.Builder builder = ImmutableList.builder(); + String exclusiveStartShardId = null; + do { + describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); + DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest); + shards.addAll(describeStreamResult.getStreamDescription().getShards()); + + for (Shard shard : shards) { + builder.add(new KinesisShard(kinesisTableHandle.getStreamName(), shard)); + } + + if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) { + exclusiveStartShardId = shards.get(shards.size() - 1).getShardId(); + } + else { + exclusiveStartShardId = null; + } + + } while (exclusiveStartShardId != null); + + return new ConnectorPartitionResult(builder.build(), tupleDomain); + } + + @Override + public ConnectorSplitSource getPartitionSplits(ConnectorTableHandle tableHandle, List partitions) + { + KinesisTableHandle kinesisTableHandle = handleResolver.convertTableHandle(tableHandle); + + ImmutableList.Builder builder = ImmutableList.builder(); + + for (ConnectorPartition cp : partitions) { + checkState(cp instanceof KinesisShard, "Found an unkown partition type: %s", cp.getClass().getSimpleName()); + KinesisShard kinesisShard = (KinesisShard) cp; + + KinesisSplit split = new KinesisSplit(connectorId, + kinesisShard.getStreamName(), + kinesisTableHandle.getMessageDataFormat(), + kinesisShard.getPartitionId(), + kinesisShard.getShard().getSequenceNumberRange().getStartingSequenceNumber(), + kinesisShard.getShard().getSequenceNumberRange().getEndingSequenceNumber()); + builder.add(split); + } + + return new FixedSplitSource(connectorId, builder.build()); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamDescription.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamDescription.java new file mode 100644 index 000000000000..6b0787a4174a --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamDescription.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Strings.isNullOrEmpty; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * + * This Class maintains all the details of Kinesis stream like name, fields of data, Presto table stream is mapping to, tables's schema name + * + */ +public class KinesisStreamDescription +{ + private final String tableName; + private final String streamName; + private final String schemaName; + private final KinesisStreamFieldGroup message; + + @JsonCreator + public KinesisStreamDescription( + @JsonProperty("tableName") String tableName, + @JsonProperty("schemaName") String schemaName, + @JsonProperty("streamName") String streamName, + @JsonProperty("message") KinesisStreamFieldGroup message) + { + checkArgument(!isNullOrEmpty(tableName), "tableName is null or is empty"); + this.tableName = tableName; + this.streamName = checkNotNull(streamName, "topicName is null"); + this.schemaName = schemaName; + this.message = message; + } + + @JsonProperty + public String getTableName() + { + return tableName; + } + + @JsonProperty + public String getStreamName() + { + return streamName; + } + + @JsonProperty + public String getSchemaName() + { + return schemaName; + } + + @JsonProperty + public KinesisStreamFieldGroup getMessage() + { + return message; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("tableName", tableName) + .add("streamName", streamName) + .add("schemaName", schemaName) + .add("message", message) + .toString(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamFieldDescription.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamFieldDescription.java new file mode 100644 index 000000000000..efa2f77fe900 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamFieldDescription.java @@ -0,0 +1,154 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Strings.isNullOrEmpty; + +import com.facebook.presto.spi.ColumnMetadata; +import com.facebook.presto.spi.type.Type; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; + +public class KinesisStreamFieldDescription +{ + private final String name; + private final Type type; + private final String mapping; + private final String comment; + private final String dataFormat; + private final String formatHint; + private final boolean hidden; + + @JsonCreator + public KinesisStreamFieldDescription( + @JsonProperty("name") String name, + @JsonProperty("type") Type type, + @JsonProperty("mapping") String mapping, + @JsonProperty("comment") String comment, + @JsonProperty("dataFormat") String dataFormat, + @JsonProperty("formatHint") String formatHint, + @JsonProperty("hidden") boolean hidden) + { + checkArgument(!isNullOrEmpty(name), "name is null or is empty"); + this.name = name; + this.type = checkNotNull(type, "type is null"); + this.mapping = mapping; + this.comment = comment; + this.dataFormat = dataFormat; + this.formatHint = formatHint; + this.hidden = hidden; + } + + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty + public Type getType() + { + return type; + } + + @JsonProperty + public String getMapping() + { + return mapping; + } + + @JsonProperty + public String getComment() + { + return comment; + } + + @JsonProperty + public String getDataFormat() + { + return dataFormat; + } + + @JsonProperty + public String getFormatHint() + { + return formatHint; + } + + @JsonProperty + public boolean isHidden() + { + return hidden; + } + + KinesisColumnHandle getColumnHandle(String connectorId, int index) + { + return new KinesisColumnHandle(connectorId, + index, + getName(), + getType(), + getMapping(), + getDataFormat(), + getFormatHint(), + isHidden(), + false); + } + + ColumnMetadata getColumnMetadata(int index) + { + return new ColumnMetadata(getName(), getType(), index, false, getComment(), isHidden()); + } + + @Override + public int hashCode() + { + return Objects.hashCode(name, type, mapping, dataFormat, formatHint, hidden); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KinesisStreamFieldDescription other = (KinesisStreamFieldDescription) obj; + return Objects.equal(this.name, other.name) && + Objects.equal(this.type, other.type) && + Objects.equal(this.mapping, other.mapping) && + Objects.equal(this.dataFormat, other.dataFormat) && + Objects.equal(this.formatHint, other.formatHint) && + Objects.equal(this.hidden, other.hidden); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("name", name) + .add("type", type) + .add("mapping", mapping) + .add("dataFormat", dataFormat) + .add("formatHint", formatHint) + .add("hidden", hidden) + .toString(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamFieldGroup.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamFieldGroup.java new file mode 100644 index 000000000000..cfadef1d0a8c --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisStreamFieldGroup.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +public class KinesisStreamFieldGroup +{ + private final String dataFormat; + private final List fields; + + @JsonCreator + public KinesisStreamFieldGroup( + @JsonProperty("dataFormat") String dataFormat, + @JsonProperty("fields") List fields) + { + this.dataFormat = checkNotNull(dataFormat, "dataFormat is null"); + this.fields = ImmutableList.copyOf(checkNotNull(fields, "fields is null")); + } + + @JsonProperty + public String getDataFormat() + { + return dataFormat; + } + + @JsonProperty + public List getFields() + { + return fields; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("dataFormat", dataFormat) + .add("fields", fields) + .toString(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisTableDescriptionSupplier.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisTableDescriptionSupplier.java new file mode 100644 index 000000000000..d0a7ceb39ac5 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisTableDescriptionSupplier.java @@ -0,0 +1,122 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import io.airlift.json.JsonCodec; +import io.airlift.log.Logger; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.weakref.jmx.internal.guava.base.Objects; + +import com.facebook.presto.kinesis.decoder.dummy.DummyKinesisRowDecoder; +import com.facebook.presto.spi.SchemaTableName; +import com.google.common.base.Supplier; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; +import com.google.inject.Inject; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Arrays.asList; + +/** + * + * This class get() method reads the table description file stored in Kinesis directory + * and then creates user defined field for Presto Table. + * + */ +public class KinesisTableDescriptionSupplier + implements Supplier> +{ + private static final Logger log = Logger.get(KinesisTableDescriptionSupplier.class); + + public final KinesisConnectorConfig kinesisConnectorConfig; + public final JsonCodec streamDescriptionCodec; + + @Inject + KinesisTableDescriptionSupplier(KinesisConnectorConfig kinesisConnectorConfig, + JsonCodec streamDescriptionCodec) + { + this.kinesisConnectorConfig = checkNotNull(kinesisConnectorConfig, "kinesisConnectorConfig is null"); + this.streamDescriptionCodec = checkNotNull(streamDescriptionCodec, "streamDescriptionCodec is null"); + } + + @Override + public Map get() + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + try { + for (File file : listFiles(kinesisConnectorConfig.getTableDescriptionDir())) { + if (file.isFile() && file.getName().endsWith(".json")) { + KinesisStreamDescription table = streamDescriptionCodec.fromJson(Files.toByteArray(file)); + String schemaName = Objects.firstNonNull(table.getSchemaName(), kinesisConnectorConfig.getDefaultSchema()); + log.debug("Kinesis table %s %s %s", schemaName, table.getTableName(), table); + builder.put(new SchemaTableName(schemaName, table.getTableName()), table); + } + } + + Map tableDefinitions = builder.build(); + + log.debug("Loaded table definitions: %s", tableDefinitions.keySet()); + + builder = ImmutableMap.builder(); + for (String definedTable : kinesisConnectorConfig.getTableNames()) { + SchemaTableName tableName; + try { + tableName = SchemaTableName.valueOf(definedTable); + } + catch (IllegalArgumentException iae) { + tableName = new SchemaTableName(kinesisConnectorConfig.getDefaultSchema(), definedTable); + } + + if (tableDefinitions.containsKey(tableName)) { + KinesisStreamDescription kinesisTable = tableDefinitions.get(tableName); + log.debug("Found Table definition fo %s %s", tableName, kinesisTable); + builder.put(tableName, kinesisTable); + } + else { + // A dummy table definition only supports the internal columns + log.debug("Created dummy Table definition for %s", tableName); + builder.put(tableName, new KinesisStreamDescription(tableName.getTableName(), + tableName.getSchemaName(), + definedTable, + new KinesisStreamFieldGroup(DummyKinesisRowDecoder.NAME, ImmutableList.of()))); + } + } + + return builder.build(); + } + catch (IOException e) { + log.warn(e, "Error: "); + throw Throwables.propagate(e); + } + } + + private static List listFiles(File dir) + { + if ((dir != null) && dir.isDirectory()) { + File[] files = dir.listFiles(); + if (files != null) { + log.debug("Considering files: %s", asList(files)); + return ImmutableList.copyOf(files); + } + } + return ImmutableList.of(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisTableHandle.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisTableHandle.java new file mode 100644 index 000000000000..2e2d8b1740da --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisTableHandle.java @@ -0,0 +1,141 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.facebook.presto.spi.ConnectorTableHandle; +import com.facebook.presto.spi.SchemaTableName; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; + +/** + * + * Class maintains all the properties of Presto Table + * + */ +public class KinesisTableHandle + implements ConnectorTableHandle +{ + /** + * connector id + */ + private final String connectorId; + + /** + * The schema name for this table. Is set through configuration and read + * using {@link KinesisConnectorConfig#getDefaultSchema()}. Usually 'default'. + */ + private final String schemaName; + + /** + * The table name used by presto. + */ + private final String tableName; + + /** + * The stream name that is read from Kinesis + */ + private final String streamName; + + private final String messageDataFormat; + + @JsonCreator + public KinesisTableHandle( + @JsonProperty("connectorId") String connectorId, + @JsonProperty("schemaName") String schemaName, + @JsonProperty("tableName") String tableName, + @JsonProperty("streamName") String streamName, + @JsonProperty("messageDataFormat") String messageDataFormat) + { + this.connectorId = checkNotNull(connectorId, "connectorId is null"); + this.schemaName = checkNotNull(schemaName, "schemaName is null"); + this.tableName = checkNotNull(tableName, "tableName is null"); + this.streamName = checkNotNull(streamName, "topicName is null"); + this.messageDataFormat = checkNotNull(messageDataFormat, "messageDataFormat is null"); + } + + @JsonProperty + public String getConnectorId() + { + return connectorId; + } + + @JsonProperty + public String getSchemaName() + { + return schemaName; + } + + @JsonProperty + public String getTableName() + { + return tableName; + } + + @JsonProperty + public String getStreamName() + { + return streamName; + } + + @JsonProperty + public String getMessageDataFormat() + { + return messageDataFormat; + } + + public SchemaTableName toSchemaTableName() + { + return new SchemaTableName(schemaName, tableName); + } + + @Override + public int hashCode() + { + return Objects.hashCode(connectorId, schemaName, tableName, streamName, messageDataFormat); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KinesisTableHandle other = (KinesisTableHandle) obj; + return Objects.equal(this.connectorId, other.connectorId) + && Objects.equal(this.schemaName, other.schemaName) + && Objects.equal(this.tableName, other.tableName) + && Objects.equal(this.streamName, other.streamName) + && Objects.equal(this.messageDataFormat, other.messageDataFormat); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("connectorId", connectorId) + .add("schemaName", schemaName) + .add("tableName", tableName) + .add("streamName", streamName) + .add("messageDataFormat", messageDataFormat) + .toString(); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisDecoderModule.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisDecoderModule.java new file mode 100644 index 000000000000..416d43ff11c5 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisDecoderModule.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder; + +import com.facebook.presto.kinesis.decoder.csv.CsvKinesisDecoderModule; +import com.facebook.presto.kinesis.decoder.dummy.DummyKinesisDecoderModule; +import com.facebook.presto.kinesis.decoder.json.JsonKinesisDecoderModule; +import com.facebook.presto.kinesis.decoder.raw.RawKinesisDecoderModule; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Scopes; +import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.Multibinder; + +/** + * + * Kinesis Decoder implementation of Injection Module interface. Binds all the Row and column decoder modules. + * + */ +public class KinesisDecoderModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(KinesisDecoderRegistry.class).in(Scopes.SINGLETON); + + binder.install(new DummyKinesisDecoderModule()); + binder.install(new CsvKinesisDecoderModule()); + binder.install(new JsonKinesisDecoderModule()); + binder.install(new RawKinesisDecoderModule()); + } + + public static void bindRowDecoder(Binder binder, Class decoderClass) + { + Multibinder rowDecoderBinder = Multibinder.newSetBinder(binder, KinesisRowDecoder.class); + rowDecoderBinder.addBinding().to(decoderClass).in(Scopes.SINGLETON); + } + + public static void bindFieldDecoder(Binder binder, Class> decoderClass) + { + Multibinder> fieldDecoderBinder = Multibinder.newSetBinder(binder, new TypeLiteral>() {}); + fieldDecoderBinder.addBinding().to(decoderClass).in(Scopes.SINGLETON); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisDecoderRegistry.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisDecoderRegistry.java new file mode 100644 index 000000000000..5189368d5bdd --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisDecoderRegistry.java @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder; + +import static com.facebook.presto.kinesis.decoder.KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; +import io.airlift.log.Logger; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import javax.annotation.Nullable; + +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.SetMultimap; +import com.google.inject.Inject; + +/** + * + * Maintains list of all the row decoders and column decoders. Also returns appropriate decoders. + * + */ +public class KinesisDecoderRegistry +{ + private static final Logger log = Logger.get(KinesisDecoderRegistry.class); + + private final Map rowDecoders; + private final Map, KinesisFieldDecoder>> fieldDecoders; + + @Inject + KinesisDecoderRegistry(Set rowDecoders, + Set> fieldDecoders) + { + checkNotNull(rowDecoders, "rowDecoders is null"); + + ImmutableMap.Builder rowBuilder = ImmutableMap.builder(); + for (KinesisRowDecoder rowDecoder : rowDecoders) { + rowBuilder.put(rowDecoder.getName(), rowDecoder); + } + + this.rowDecoders = rowBuilder.build(); + + Map, KinesisFieldDecoder>> fieldDecoderBuilders = new HashMap<>(); + + for (KinesisFieldDecoder fieldDecoder : fieldDecoders) { + ImmutableSetMultimap.Builder, KinesisFieldDecoder> fieldDecoderBuilder = fieldDecoderBuilders.get(fieldDecoder.getRowDecoderName()); + if (fieldDecoderBuilder == null) { + fieldDecoderBuilder = ImmutableSetMultimap.builder(); + fieldDecoderBuilders.put(fieldDecoder.getRowDecoderName(), fieldDecoderBuilder); + } + + for (Class clazz : fieldDecoder.getJavaTypes()) { + fieldDecoderBuilder.put(clazz, fieldDecoder); + } + } + + ImmutableMap.Builder, KinesisFieldDecoder>> fieldDecoderBuilder = ImmutableMap.builder(); + for (Map.Entry, KinesisFieldDecoder>> entry : fieldDecoderBuilders.entrySet()) { + fieldDecoderBuilder.put(entry.getKey(), entry.getValue().build()); + } + + this.fieldDecoders = fieldDecoderBuilder.build(); + log.debug("Field decoders found: %s", this.fieldDecoders); + } + + /** + * Return the specific row decoder for a given data format. + */ + public KinesisRowDecoder getRowDecoder(String dataFormat) + { + checkState(rowDecoders.containsKey(dataFormat), "no row decoder for '%s' found", dataFormat); + return rowDecoders.get(dataFormat); + } + + public KinesisFieldDecoder getFieldDecoder(String rowDataFormat, Class fieldType, @Nullable String fieldDataFormat) + { + checkNotNull(rowDataFormat, "rowDataFormat is null"); + checkNotNull(fieldType, "fieldType is null"); + + checkState(fieldDecoders.containsKey(rowDataFormat), "no field decoders for '%s' found", rowDataFormat); + Set> decoders = fieldDecoders.get(rowDataFormat).get(fieldType); + + ImmutableSet fieldNames = ImmutableSet.of(Objects.firstNonNull(fieldDataFormat, DEFAULT_FIELD_DECODER_NAME), + DEFAULT_FIELD_DECODER_NAME); + + for (String fieldName : fieldNames) { + for (KinesisFieldDecoder decoder : decoders) { + if (fieldName.equals(decoder.getFieldDecoderName())) { + return decoder; + } + } + } + + throw new IllegalStateException(format("No field decoder for %s/%s found!", rowDataFormat, fieldType)); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisFieldDecoder.java new file mode 100644 index 000000000000..b1c0576d7781 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisFieldDecoder.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder; + +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; + +public interface KinesisFieldDecoder +{ + /** + * Default name. Each decoder type *must* have a default decoder as fallback. + */ + String DEFAULT_FIELD_DECODER_NAME = "_default"; + + /** + * Returns the types which the field decoder can process. + */ + Set> getJavaTypes(); + + /** + * Returns the name of the row decoder to which this field decoder belongs. + */ + String getRowDecoderName(); + + /** + * Returns the field decoder specific name. This name will be selected with the {@link com.facebook.presto.kafka.KafkaTopicFieldDescription#dataFormat} value. + */ + String getFieldDecoderName(); + + /** + * Decode a value for the given column handle. + * + * @param value The raw value as generated by the row decoder. + * @param columnHandle The column for which the value is decoded. + * @return A {@link KafkaFieldValueProvider} instance which returns a captured value for this specific column. + */ + KinesisFieldValueProvider decode(T value, KinesisColumnHandle columnHandle); +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisRowDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisRowDecoder.java new file mode 100644 index 000000000000..c2a2f70f0cee --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/KinesisRowDecoder.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; + +public interface KinesisRowDecoder +{ + /** + * Returns the row decoder specific name. + */ + String getName(); + + /** + * Decodes a given set of bytes into field values. + * + * @param data The row data (Kinesis message) to decode. + * @param fieldValueProviders Must be a mutable set. Any field value provider created by this row decoder is put into this set. + * @param columnHandles List of column handles for which field values are required. + * @param fieldDecoders Map from column handles to decoders. This map should be used to look up the field decoder that generates the field value provider for a given column handle. + * @return true if the row was decoded successfully, false if it could not be decoded (was corrupt). TODO - reverse this boolean. + */ + boolean decodeRow( + byte[] data, + Set fieldValueProviders, + List columnHandles, + Map> fieldDecoders); +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisDecoderModule.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisDecoderModule.java new file mode 100644 index 000000000000..681bf8d016da --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisDecoderModule.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.csv; + +import static com.facebook.presto.kinesis.decoder.KinesisDecoderModule.bindFieldDecoder; +import static com.facebook.presto.kinesis.decoder.KinesisDecoderModule.bindRowDecoder; + +import com.google.inject.Binder; +import com.google.inject.Module; + +public class CsvKinesisDecoderModule + implements Module +{ + @Override + public void configure(Binder binder) + { + bindRowDecoder(binder, CsvKinesisRowDecoder.class); + + bindFieldDecoder(binder, CsvKinesisFieldDecoder.class); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisFieldDecoder.java new file mode 100644 index 000000000000..2575cc848849 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisFieldDecoder.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.csv; + +import static com.google.common.base.Preconditions.checkNotNull; +import static io.airlift.slice.Slices.EMPTY_SLICE; +import static io.airlift.slice.Slices.utf8Slice; +import static java.lang.String.format; +import io.airlift.slice.Slice; + +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.google.common.collect.ImmutableSet; + +public class CsvKinesisFieldDecoder + implements KinesisFieldDecoder +{ + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(boolean.class, long.class, double.class, Slice.class); + } + + @Override + public String getRowDecoderName() + { + return CsvKinesisRowDecoder.NAME; + } + + @Override + public String getFieldDecoderName() + { + return KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; + } + + @Override + public KinesisFieldValueProvider decode(final String value, final KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + + return new KinesisFieldValueProvider() + { + @Override + public boolean accept(KinesisColumnHandle handle) + { + return columnHandle.equals(handle); + } + + public boolean isNull() + { + return (value == null) || value.isEmpty(); + } + + @SuppressWarnings("SimplifiableConditionalExpression") + @Override + public boolean getBoolean() + { + return isNull() ? false : Boolean.parseBoolean(value.trim()); + } + + @Override + public long getLong() + { + return isNull() ? 0L : Long.parseLong(value.trim()); + } + + @Override + public double getDouble() + { + return isNull() ? 0.0d : Double.parseDouble(value.trim()); + } + + @Override + public Slice getSlice() + { + return isNull() ? EMPTY_SLICE : utf8Slice(value); + } + }; + } + + @Override + public String toString() + { + return format("FieldDecoder[%s/%s]", getRowDecoderName(), getFieldDecoderName()); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisRowDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisRowDecoder.java new file mode 100644 index 000000000000..ad1244d0e9bf --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/csv/CsvKinesisRowDecoder.java @@ -0,0 +1,88 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.csv; + +import static com.google.common.base.Preconditions.checkState; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import au.com.bytecode.opencsv.CSVParser; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.kinesis.decoder.KinesisRowDecoder; +import com.google.inject.Inject; + +public class CsvKinesisRowDecoder + implements KinesisRowDecoder +{ + public static final String NAME = "csv"; + + private final CSVParser parser = new CSVParser(); + + @Inject + CsvKinesisRowDecoder() + { + } + + @Override + public String getName() + { + return NAME; + } + + @Override + public boolean decodeRow(byte[] data, + Set fieldValueProviders, + List columnHandles, + Map> fieldDecoders) + { + String[] fields; + try { + // TODO - There is no reason why the row can't have a formatHint and it could be used + // to set the charset here. + String line = new String(data, StandardCharsets.UTF_8); + fields = parser.parseLine(line); + } + catch (Exception e) { + return false; + } + + for (KinesisColumnHandle columnHandle : columnHandles) { + if (columnHandle.isInternal()) { + continue; + } + + String mapping = columnHandle.getMapping(); + checkState(mapping != null, "No mapping for column handle %s!", columnHandle); + int columnIndex = Integer.parseInt(mapping); + + if (columnIndex >= fields.length) { + continue; + } + + @SuppressWarnings("unchecked") + KinesisFieldDecoder decoder = (KinesisFieldDecoder) fieldDecoders.get(columnHandle); + + if (decoder != null) { + fieldValueProviders.add(decoder.decode(fields[columnIndex], columnHandle)); + } + } + return true; + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisDecoderModule.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisDecoderModule.java new file mode 100644 index 000000000000..faa99d4135c1 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisDecoderModule.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.dummy; + +import com.google.inject.Binder; +import com.google.inject.Module; + +import static com.facebook.presto.kinesis.decoder.KinesisDecoderModule.bindRowDecoder; +import static com.facebook.presto.kinesis.decoder.KinesisDecoderModule.bindFieldDecoder; + +public class DummyKinesisDecoderModule + implements Module +{ + @Override + public void configure(Binder binder) + { + bindRowDecoder(binder, DummyKinesisRowDecoder.class); + + bindFieldDecoder(binder, DummyKinesisFieldDecoder.class); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisFieldDecoder.java new file mode 100644 index 000000000000..c21128125e26 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisFieldDecoder.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.dummy; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; +import io.airlift.slice.Slice; + +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.PrestoException; +import com.google.common.collect.ImmutableSet; + +import static com.facebook.presto.kinesis.KinesisErrorCode.KINESIS_CONVERSION_NOT_SUPPORTED; + +public class DummyKinesisFieldDecoder + implements KinesisFieldDecoder +{ + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(boolean.class, long.class, double.class, Slice.class); + } + + @Override + public String getRowDecoderName() + { + return DummyKinesisRowDecoder.NAME; + } + + @Override + public String getFieldDecoderName() + { + return KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; + } + + @Override + public String toString() + { + return format("FieldDecoder[%s/%s]", getRowDecoderName(), getFieldDecoderName()); + } + + @Override + public KinesisFieldValueProvider decode(Void value, KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + + return new KinesisFieldValueProvider() + { + @Override + public boolean accept(KinesisColumnHandle handle) + { + return false; + } + + @Override + public boolean isNull() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "is null check not supported"); + } + }; + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisRowDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisRowDecoder.java new file mode 100644 index 000000000000..22135dad7ce9 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/dummy/DummyKinesisRowDecoder.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.dummy; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.kinesis.decoder.KinesisRowDecoder; + +public class DummyKinesisRowDecoder + implements KinesisRowDecoder +{ + public static final String NAME = "dummy"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public boolean decodeRow(byte[] data, + Set fieldValueProviders, + List columnHandles, + Map> fieldDecoders) + { + return true; + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/CustomDateTimeJsonKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/CustomDateTimeJsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..a386bfba899c --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/CustomDateTimeJsonKinesisFieldDecoder.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import java.util.Locale; +import java.util.Set; + +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import io.airlift.slice.Slice; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.spi.PrestoException; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableSet; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.facebook.presto.kinesis.KinesisErrorCode.KINESIS_CONVERSION_NOT_SUPPORTED; + +public class CustomDateTimeJsonKinesisFieldDecoder + extends JsonKinesisFieldDecoder +{ + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(long.class, Slice.class); + } + + @Override + public String getFieldDecoderName() + { + return "custom-date-time"; + } + + @Override + public KinesisFieldValueProvider decode(JsonNode value, KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + checkNotNull(value, "value is null"); + + return new CustomeDateTimeJsonKinesisValueProvider(value, columnHandle); + } + + public static class CustomeDateTimeJsonKinesisValueProvider + extends JsonKinesisValueProvider + { + public CustomeDateTimeJsonKinesisValueProvider(JsonNode value, KinesisColumnHandle columnHandle) + { + super(value, columnHandle); + } + + @Override + public boolean getBoolean() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to boolean not supported"); + } + + @Override + public double getDouble() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to double not supported"); + } + + @Override + public long getLong() + { + if (isNull()) { + return 0L; + } + + if (value.canConvertToLong()) { + return value.asLong(); + } + + checkNotNull(columnHandle.getFormatHint(), "formatHint is null"); + String textValue = value.isValueNode() ? value.asText() : value.toString(); + + DateTimeFormatter formatter = DateTimeFormat.forPattern(columnHandle.getFormatHint()).withLocale(Locale.ENGLISH).withZoneUTC(); + return formatter.parseMillis(textValue); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/ISO8601JsonKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/ISO8601JsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..09f5fdd83953 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/ISO8601JsonKinesisFieldDecoder.java @@ -0,0 +1,101 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.google.common.base.Preconditions.checkNotNull; +import io.airlift.slice.Slice; + +import java.util.Locale; +import java.util.Set; + +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.spi.PrestoException; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; + +import static com.facebook.presto.kinesis.KinesisErrorCode.KINESIS_CONVERSION_NOT_SUPPORTED; + +public class ISO8601JsonKinesisFieldDecoder + extends JsonKinesisFieldDecoder +{ + @VisibleForTesting + static final String NAME = "iso8601"; + + /** + * TODO: configurable time zones and locales + */ + private static final DateTimeFormatter FORMATTER = ISODateTimeFormat.dateTimeParser().withLocale(Locale.ENGLISH).withZoneUTC(); + + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(long.class, Slice.class); + } + + @Override + public String getFieldDecoderName() + { + return NAME; + } + + @Override + public KinesisFieldValueProvider decode(JsonNode value, KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + checkNotNull(value, "value is null"); + + return new ISO8601JsonKinesisFieldValueProvider(value, columnHandle); + } + + public static class ISO8601JsonKinesisFieldValueProvider + extends JsonKinesisValueProvider + { + public ISO8601JsonKinesisFieldValueProvider(JsonNode value, KinesisColumnHandle columnHandle) + { + super(value, columnHandle); + } + + @Override + public boolean getBoolean() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to boolean not supported"); + } + + @Override + public double getDouble() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to double not supported"); + } + + @Override + public long getLong() + { + if (isNull()) { + return 0L; + } + + if (value.canConvertToLong()) { + return value.asLong(); + } + + String textValue = value.isValueNode() ? value.asText() : value.toString(); + return FORMATTER.parseMillis(textValue); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisDecoderModule.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisDecoderModule.java new file mode 100644 index 000000000000..200dac7553ab --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisDecoderModule.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import com.google.inject.Binder; +import com.google.inject.Module; + +import static com.facebook.presto.kinesis.decoder.KinesisDecoderModule.bindFieldDecoder; +import static com.facebook.presto.kinesis.decoder.KinesisDecoderModule.bindRowDecoder; + +/** + * Guice module for the Json decoder module. This is the most mature (best tested) topic decoder. + *

+ * Besides the default field decoder for all the values, it also supports a number of decoders for + * timestamp specific information. These decoders can be selected with the dataFormat field. + *

+ *

    + *
  • iso8601 - decode the value of a json string field as an ISO8601 timestamp; returns a long value which can be mapped to a presto TIMESTAMP.
    • + *
  • rfc2822 - decode the value of a json string field as an RFC 2822 compliant timestamp; returns a long value which can be mapped to a presto TIMESTAMP + * (the twitter sample feed contains timestamps in this format).
    • + *
  • milliseconds-since-epoch - Interpret the value of a json string or number field as a long containing milliseconds since the beginning of the epoch.
    • + *
  • seconds-since-epoch - Interpret the value of a json string or number field as a long containing seconds since the beginning of the epoch.
    • + *
  • custom-date-time - Interpret the value of a json string field according to the {@link org.joda.time.format.DateTimeFormatter} formatting rules + * given using the formatHint field.
    • + *
+ */ +public class JsonKinesisDecoderModule + implements Module +{ + @Override + public void configure(Binder binder) + { + bindRowDecoder(binder, JsonKinesisRowDecoder.class); + + bindFieldDecoder(binder, JsonKinesisFieldDecoder.class); + // TODO: Implement other Json field decoders + /* + bindFieldDecoder(binder, ISO8601JsonKinesisFieldDecoder.class); + bindFieldDecoder(binder, RFC2822JsonKinesisFieldDecoder.class); + bindFieldDecoder(binder, SecondsSinceEpochJsonKinesisFieldDecoder.class); + bindFieldDecoder(binder, MillisecondsSinceEpochJsonKinesisFieldDecoder.class); + bindFieldDecoder(binder, CustomDateTimeJsonKinesisFieldDecoder.class); + */ + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..c3fc5d0fdef0 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisFieldDecoder.java @@ -0,0 +1,115 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.google.common.base.Preconditions.checkNotNull; +import static io.airlift.slice.Slices.EMPTY_SLICE; +import static io.airlift.slice.Slices.utf8Slice; +import static java.lang.String.format; +import io.airlift.slice.Slice; + +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableSet; + +public class JsonKinesisFieldDecoder + implements KinesisFieldDecoder +{ + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(boolean.class, long.class, double.class, Slice.class); + } + + @Override + public final String getRowDecoderName() + { + return JsonKinesisRowDecoder.NAME; + } + + @Override + public String getFieldDecoderName() + { + return KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; + } + + @Override + public KinesisFieldValueProvider decode(JsonNode value, KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + checkNotNull(value, "value is null"); + + return new JsonKinesisValueProvider(value, columnHandle); + } + + @Override + public String toString() + { + return format("FieldDecoder[%s/%s]", getRowDecoderName(), getFieldDecoderName()); + } + + public static class JsonKinesisValueProvider + extends KinesisFieldValueProvider + { + protected final JsonNode value; + protected final KinesisColumnHandle columnHandle; + + public JsonKinesisValueProvider(JsonNode value, KinesisColumnHandle columnHandle) + { + this.value = value; + this.columnHandle = columnHandle; + } + + @Override + public final boolean accept(KinesisColumnHandle columnHandle) + { + return this.columnHandle.equals(columnHandle); + } + + @Override + public final boolean isNull() + { + return value.isMissingNode() || value.isNull(); + } + + @Override + public boolean getBoolean() + { + return value.asBoolean(); + } + + @Override + public long getLong() + { + return value.asLong(); + } + + @Override + public double getDouble() + { + return value.asDouble(); + } + + @Override + public Slice getSlice() + { + String textValue = value.isValueNode() ? value.asText() : value.toString(); + return isNull() ? EMPTY_SLICE : utf8Slice(textValue); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisRowDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisRowDecoder.java new file mode 100644 index 000000000000..40dc7956cf41 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/JsonKinesisRowDecoder.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.google.common.base.Preconditions.checkState; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.kinesis.decoder.KinesisRowDecoder; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.MissingNode; +import com.google.common.base.Splitter; +import com.google.inject.Inject; + +public class JsonKinesisRowDecoder + implements KinesisRowDecoder +{ + public static final String NAME = "json"; + + private final ObjectMapper objectMapper; + + @Inject + public JsonKinesisRowDecoder(ObjectMapper objectMapper) + { + this.objectMapper = objectMapper; + } + + @Override + public String getName() + { + return NAME; + } + + @Override + public boolean decodeRow(byte[] data, Set fieldValueProviders, List columnHandles, Map> fieldDecoders) + { + JsonNode tree; + + try { + tree = objectMapper.readTree(data); + } + catch (Exception e) { + return false; + } + + for (KinesisColumnHandle columnHandle : columnHandles) { + if (columnHandle.isInternal()) { + continue; + } + @SuppressWarnings("unchecked") + KinesisFieldDecoder decoder = (KinesisFieldDecoder) fieldDecoders.get(columnHandle); + + if (decoder != null) { + JsonNode node = locateNode(tree, columnHandle); + fieldValueProviders.add(decoder.decode(node, columnHandle)); + } + } + return true; + } + + private static JsonNode locateNode(JsonNode tree, KinesisColumnHandle columnHandle) + { + String mapping = columnHandle.getMapping(); + checkState(mapping != null, "No mapping for %s", columnHandle.getName()); + + JsonNode currentNode = tree; + for (String pathElement : Splitter.on('/').omitEmptyStrings().split(mapping)) { + if (!currentNode.has(pathElement)) { + return MissingNode.getInstance(); + } + currentNode = currentNode.path(pathElement); + } + return currentNode; + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/MillisecondsSinceEpochJsonKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/MillisecondsSinceEpochJsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..8044f35dec53 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/MillisecondsSinceEpochJsonKinesisFieldDecoder.java @@ -0,0 +1,101 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.google.common.base.Preconditions.checkNotNull; +import io.airlift.slice.Slice; + +import java.util.Locale; +import java.util.Set; + +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.spi.PrestoException; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; + +import static com.facebook.presto.kinesis.KinesisErrorCode.KINESIS_CONVERSION_NOT_SUPPORTED; +import static io.airlift.slice.Slices.EMPTY_SLICE; +import static io.airlift.slice.Slices.utf8Slice; + +public class MillisecondsSinceEpochJsonKinesisFieldDecoder + extends JsonKinesisFieldDecoder +{ + @VisibleForTesting + static final String NAME = "milliseconds-since-epoch"; + + /** + * Todo - configurable time zones and locales. + */ + @VisibleForTesting + static final DateTimeFormatter FORMATTER = ISODateTimeFormat.dateTime().withLocale(Locale.ENGLISH).withZoneUTC(); + + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(long.class, Slice.class); + } + + @Override + public String getFieldDecoderName() + { + return NAME; + } + + @Override + public KinesisFieldValueProvider decode(JsonNode value, KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + checkNotNull(value, "value is null"); + + return new MillisecondsSinceEpochJsonKinesisValueProvider(value, columnHandle); + } + + public static class MillisecondsSinceEpochJsonKinesisValueProvider + extends JsonKinesisValueProvider + { + public MillisecondsSinceEpochJsonKinesisValueProvider(JsonNode value, KinesisColumnHandle columnHandle) + { + super(value, columnHandle); + } + + @Override + public boolean getBoolean() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to boolean not supported"); + } + + @Override + public double getDouble() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to double not supported"); + } + + @Override + public long getLong() + { + return isNull() ? 0L : value.asLong(); + } + + @Override + public Slice getSlice() + { + return isNull() ? EMPTY_SLICE : utf8Slice(FORMATTER.print(value.asLong())); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/RFC2822JsonKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/RFC2822JsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..a1f09e0f7b2a --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/RFC2822JsonKinesisFieldDecoder.java @@ -0,0 +1,102 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.google.common.base.Preconditions.checkNotNull; +import io.airlift.slice.Slice; + +import java.util.Locale; +import java.util.Set; + +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.spi.PrestoException; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; + +import static com.facebook.presto.kinesis.KinesisErrorCode.KINESIS_CONVERSION_NOT_SUPPORTED; + +public class RFC2822JsonKinesisFieldDecoder + extends JsonKinesisFieldDecoder +{ + @VisibleForTesting + static final String NAME = "rfc2822"; + + /** + * Todo - configurable time zones and locales. + */ + @VisibleForTesting + static final DateTimeFormatter FORMATTER = DateTimeFormat.forPattern("EEE MMM dd HH:mm:ss Z yyyy").withLocale(Locale.ENGLISH).withZoneUTC(); + + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(long.class, Slice.class); + } + + @Override + public String getFieldDecoderName() + { + return NAME; + } + + @Override + public KinesisFieldValueProvider decode(JsonNode value, KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + checkNotNull(value, "value is null"); + + return new RFC2822JsonKinesisValueProvider(value, columnHandle); + } + + public static class RFC2822JsonKinesisValueProvider + extends JsonKinesisValueProvider + { + public RFC2822JsonKinesisValueProvider(JsonNode value, KinesisColumnHandle columnHandle) + { + super(value, columnHandle); + } + + @Override + public boolean getBoolean() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to boolean not supported"); + } + + @Override + public double getDouble() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to double not supported"); + } + + @Override + public long getLong() + { + if (isNull()) { + return 0L; + } + + if (value.canConvertToLong()) { + return value.asLong(); + } + + String textValue = value.isValueNode() ? value.asText() : value.toString(); + return FORMATTER.parseMillis(textValue); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/SecondsSinceEpochJsonKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/SecondsSinceEpochJsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..26d5328c25d6 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/json/SecondsSinceEpochJsonKinesisFieldDecoder.java @@ -0,0 +1,101 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.google.common.base.Preconditions.checkNotNull; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.util.Locale; +import java.util.Set; + +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.spi.PrestoException; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; + +import static com.facebook.presto.kinesis.KinesisErrorCode.KINESIS_CONVERSION_NOT_SUPPORTED; +import static io.airlift.slice.Slices.utf8Slice; + +public class SecondsSinceEpochJsonKinesisFieldDecoder + extends JsonKinesisFieldDecoder +{ + @VisibleForTesting + static final String NAME = "seconds-since-epoch"; + + /** + * Todo - configurable time zones and locales. + */ + @VisibleForTesting + static final DateTimeFormatter FORMATTER = ISODateTimeFormat.dateTimeNoMillis().withLocale(Locale.ENGLISH).withZoneUTC(); + + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(long.class, Slice.class); + } + + @Override + public String getFieldDecoderName() + { + return NAME; + } + + @Override + public KinesisFieldValueProvider decode(JsonNode value, KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + checkNotNull(value, "value is null"); + + return new SecondsSinceEpochJsonKinesisValueProvider(value, columnHandle); + } + + public static class SecondsSinceEpochJsonKinesisValueProvider + extends JsonKinesisValueProvider + { + public SecondsSinceEpochJsonKinesisValueProvider(JsonNode value, KinesisColumnHandle columnHandle) + { + super(value, columnHandle); + } + + @Override + public boolean getBoolean() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to boolean not supported"); + } + + @Override + public double getDouble() + { + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, "conversion to double not supported"); + } + + @Override + public long getLong() + { + return isNull() ? 0L : value.asLong() * 1000L; + } + + @Override + public Slice getSlice() + { + return isNull() ? Slices.EMPTY_SLICE : utf8Slice(FORMATTER.print(value.asLong() * 1000L)); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisDecoderModule.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisDecoderModule.java new file mode 100644 index 000000000000..ef41c8423cce --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisDecoderModule.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.raw; + +import com.google.inject.Binder; +import com.google.inject.Module; + +import static com.facebook.presto.kinesis.decoder.KinesisDecoderModule.bindFieldDecoder; +import static com.facebook.presto.kinesis.decoder.KinesisDecoderModule.bindRowDecoder; + +public class RawKinesisDecoderModule + implements Module +{ + @Override + public void configure(Binder binder) + { + bindRowDecoder(binder, RawKinesisRowDecoder.class); + bindFieldDecoder(binder, RawKinesisFieldDecoder.class); + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisFieldDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisFieldDecoder.java new file mode 100644 index 000000000000..211750114e2d --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisFieldDecoder.java @@ -0,0 +1,227 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.raw; + +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.PrestoException; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.format; +import static com.facebook.presto.kinesis.KinesisErrorCode.KINESIS_CONVERSION_NOT_SUPPORTED; + +public class RawKinesisFieldDecoder + implements KinesisFieldDecoder +{ + public enum FieldType + { + BYTE(Byte.SIZE), + SHORT(Short.SIZE), + INT(Integer.SIZE), + LONG(Long.SIZE), + FLOAT(Float.SIZE), + DOUBLE(Double.SIZE); + + private final int size; + + FieldType(int bitSize) + { + this.size = bitSize / 8; + } + + public int getSize() + { + return size; + } + + static FieldType forString(String value) + { + if (value != null) { + for (FieldType fieldType : values()) { + if (value.toUpperCase(Locale.ENGLISH).equals(fieldType.name())) { + return fieldType; + } + } + } + + return null; + } + } + + @Override + public Set> getJavaTypes() + { + return ImmutableSet.>of(boolean.class, long.class, double.class, Slice.class); + } + + @Override + public final String getRowDecoderName() + { + return RawKinesisRowDecoder.NAME; + } + + @Override + public String getFieldDecoderName() + { + return KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; + } + + @Override + public KinesisFieldValueProvider decode(byte[] value, KinesisColumnHandle columnHandle) + { + checkNotNull(columnHandle, "columnHandle is null"); + checkNotNull(value, "value is null"); + + String mapping = columnHandle.getMapping(); + FieldType fieldType = columnHandle.getDataFormat() == null ? FieldType.BYTE : FieldType.forString(columnHandle.getDataFormat()); + + int start = 0; + int end = value.length; + + if (mapping != null) { + List fields = ImmutableList.copyOf(Splitter.on(':').limit(2).split(mapping)); + if (!fields.isEmpty()) { + start = Integer.parseInt(fields.get(0)); + checkState(start >= 0 && start < value.length, "Found start %s, but only 0..%s is legal", start, value.length); + if (fields.size() > 1) { + end = Integer.parseInt(fields.get(1)); + checkState(end > 0 && end <= value.length, "Found end %s, but only 1..%s is legal", end, value.length); + } + } + } + + checkState(start <= end, "Found start %s and end %s. start must be smaller than end", start, end); + + return new RawKinesisValueProvider(ByteBuffer.wrap(value, start, end - start), columnHandle, fieldType); + } + + @Override + public String toString() + { + return format("FieldDecoder[%s/%s]", getRowDecoderName(), getFieldDecoderName()); + } + + public static class RawKinesisValueProvider + extends KinesisFieldValueProvider + { + protected final ByteBuffer value; + protected final KinesisColumnHandle columnHandle; + protected final FieldType fieldType; + protected final int size; + + public RawKinesisValueProvider(ByteBuffer value, KinesisColumnHandle columnHandle, FieldType fieldType) + { + this.columnHandle = checkNotNull(columnHandle, "columnHandle is null"); + this.fieldType = checkNotNull(fieldType, "fieldType is null"); + this.size = value.limit() - value.position(); + checkState(size >= fieldType.getSize(), "minimum byte size is %s, found %s,", fieldType.getSize(), size); + this.value = value; + } + + @Override + public final boolean accept(KinesisColumnHandle columnHandle) + { + return this.columnHandle.equals(columnHandle); + } + + @Override + public final boolean isNull() + { + return size == 0; + } + + @Override + public boolean getBoolean() + { + if (isNull()) { + return false; + } + switch (fieldType) { + case BYTE: + return value.get() != 0; + case SHORT: + return value.getShort() != 0; + case INT: + return value.getInt() != 0; + case LONG: + return value.getLong() != 0; + default: + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, format("conversion %s to boolean not supported", fieldType)); + } + } + + @Override + public long getLong() + { + if (isNull()) { + return 0L; + } + switch (fieldType) { + case BYTE: + return value.get(); + case SHORT: + return value.getShort(); + case INT: + return value.getInt(); + case LONG: + return value.getLong(); + default: + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, format("conversion %s to long not supported", fieldType)); + } + } + + @Override + public double getDouble() + { + if (isNull()) { + return 0.0d; + } + switch (fieldType) { + case FLOAT: + return value.getFloat(); + case DOUBLE: + return value.getDouble(); + default: + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, format("conversion %s to double not supported", fieldType)); + } + } + + @Override + public Slice getSlice() + { + if (isNull()) { + return Slices.EMPTY_SLICE; + } + + if (fieldType == FieldType.BYTE) { + return Slices.wrappedBuffer(value.slice()); + } + + throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, format("conversion %s to Slice not supported", fieldType)); + } + } +} diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisRowDecoder.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisRowDecoder.java new file mode 100644 index 000000000000..055375981676 --- /dev/null +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/decoder/raw/RawKinesisRowDecoder.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.raw; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.kinesis.decoder.KinesisRowDecoder; + +public class RawKinesisRowDecoder + implements KinesisRowDecoder +{ + public static final String NAME = "raw"; + + @Override + public String getName() + { + return NAME; + } + + @Override + public boolean decodeRow(byte[] data, Set fieldValueProviders, List columnHandles, Map> fieldDecoders) + { + for (KinesisColumnHandle columnHandle : columnHandles) { + if (columnHandle.isInternal()) { + continue; + } + + @SuppressWarnings("unchecked") + KinesisFieldDecoder decoder = (KinesisFieldDecoder) fieldDecoders.get(columnHandle); + + if (decoder != null) { + fieldValueProviders.add(decoder.decode(data, columnHandle)); + } + } + + return true; + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestKinesisConnectorConfig.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestKinesisConnectorConfig.java new file mode 100644 index 000000000000..65003e0a1de9 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestKinesisConnectorConfig.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import org.testng.annotations.Parameters; +import org.testng.annotations.Test; + +import com.google.common.collect.ImmutableMap; + +import io.airlift.configuration.testing.ConfigAssertions; + +import java.io.File; +import java.util.Map; + +public class TestKinesisConnectorConfig +{ + @Parameters({ + "kinesis.awsAccessKey", + "kinesis.awsSecretKey" + }) + @Test + public void testDefaults(String accessKey, String secretKey) + { + ConfigAssertions.assertRecordedDefaults(ConfigAssertions.recordDefaults(KinesisConnectorConfig.class) + .setDefaultSchema("default") + .setHideInternalColumns(true) + .setTableNames("") + .setTableDescriptionDir(new File("etc/kinesis/")) + .setAccessKey(null) + .setSecretKey(null)); + } + + @Parameters({ + "kinesis.awsAccessKey", + "kinesis.awsSecretKey" + }) + @Test + public void testExplicitPropertyMappings(String accessKey, String secretKey) + { + Map properties = new ImmutableMap.Builder() + .put("kinesis.table-description-dir", "/var/lib/kinesis") + .put("kinesis.table-names", "table1, table2, table3") + .put("kinesis.default-schema", "kinesis") + .put("kinesis.hide-internal-columns", "false") + .put("kinesis.access-key", accessKey) + .put("kinesis.secret-key", secretKey) + .build(); + + KinesisConnectorConfig expected = new KinesisConnectorConfig() + .setTableDescriptionDir(new File("/var/lib/kinesis")) + .setTableNames("table1, table2, table3") + .setDefaultSchema("kinesis") + .setHideInternalColumns(false) + .setAccessKey(accessKey) + .setSecretKey(secretKey); + + ConfigAssertions.assertFullMapping(properties, expected); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestKinesisPlugin.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestKinesisPlugin.java new file mode 100644 index 000000000000..59807b7f8ab8 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestKinesisPlugin.java @@ -0,0 +1,86 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import java.util.List; + +import org.testng.annotations.Parameters; +import org.testng.annotations.Test; + +import com.facebook.presto.spi.Connector; +import com.facebook.presto.spi.ConnectorFactory; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignature; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +public class TestKinesisPlugin +{ + @Test + public ConnectorFactory testConnectorExists() + { + KinesisPlugin plugin = new KinesisPlugin(); + plugin.setTypeManager(new TestingTypeManager()); + + List factories = plugin.getServices(ConnectorFactory.class); + assertNotNull(factories); + assertEquals(factories.size(), 1); + ConnectorFactory factory = factories.get(0); + assertNotNull(factory); + return factory; + } + + @Parameters({ + "kinesis.awsAccessKey", + "kinesis.awsSecretKey" + }) + @Test + public void testSpinUp(String awsAccessKey, String awsSecretKey) + { + ConnectorFactory factory = testConnectorExists(); + Connector c = factory.create("kinesis.test-connector", ImmutableMap.builder() + .put("kinesis.table-names", "test") + .put("kinesis.hide-internal-columns", "false") + .put("kinesis.access-key", awsAccessKey) + .put("kinesis.secret-key", awsSecretKey) + .build()); + assertNotNull(c); + } + + private static class TestingTypeManager + implements TypeManager + { + @Override + public Type getType(TypeSignature signature) + { + return null; + } + + @Override + public Type getParameterizedType(String baseTypeName, List typeParameters, List literalParameters) + { + return null; + } + + @Override + public List getTypes() + { + return ImmutableList.of(); + } + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestMinimalFunctionality.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestMinimalFunctionality.java new file mode 100644 index 000000000000..4100aa8a283d --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/TestMinimalFunctionality.java @@ -0,0 +1,176 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis; + +import io.airlift.log.Logger; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Parameters; +import org.testng.annotations.Test; + +import com.amazonaws.services.kinesis.model.DescribeStreamRequest; +import com.amazonaws.services.kinesis.model.PutRecordsRequest; +import com.amazonaws.services.kinesis.model.PutRecordsRequestEntry; +import com.amazonaws.services.kinesis.model.StreamDescription; +import com.facebook.presto.Session; +import com.facebook.presto.kinesis.util.EmbeddedKinesisStream; +import com.facebook.presto.kinesis.util.TestUtils; +import com.facebook.presto.metadata.QualifiedTableName; +import com.facebook.presto.metadata.TableHandle; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.tests.StandaloneQueryRunner; +import com.google.common.collect.ImmutableMap; + +import static com.facebook.presto.spi.type.TimeZoneKey.UTC_KEY; +import static java.util.Locale.ENGLISH; +import static org.testng.Assert.assertTrue; +import static com.facebook.presto.kinesis.util.TestUtils.createEmptyStreamDescription; +import static org.testng.Assert.assertEquals; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +@Test(singleThreaded = true) +public class TestMinimalFunctionality +{ + private static final Logger log = Logger.get(TestMinimalFunctionality.class); + + private static final Session SESSION = Session.builder() + .setUser("user") + .setSource("source") + .setCatalog("kinesis") + .setSchema("default") + .setTimeZoneKey(UTC_KEY) + .setLocale(ENGLISH) + .build(); + + private EmbeddedKinesisStream embeddedKinesisStream; + private String streamName; + private StandaloneQueryRunner queryRunner; + + @Parameters({ + "kinesis.awsAccessKey", + "kinesis.awsSecretKey" + }) + @BeforeClass + public void start(String accessKey, String secretKey) + throws Exception + { + embeddedKinesisStream = new EmbeddedKinesisStream(accessKey, secretKey); + } + + @AfterClass + public void stop() + throws Exception + { + embeddedKinesisStream.close(); + } + + @Parameters({ + "kinesis.awsAccessKey", + "kinesis.awsSecretKey" + }) + @BeforeMethod + public void spinUp(String accessKey, String secretKey) + throws Exception + { + streamName = "test_" + UUID.randomUUID().toString().replaceAll("-", "_"); + embeddedKinesisStream.createStream(2, streamName); + this.queryRunner = new StandaloneQueryRunner(SESSION); + TestUtils.installKinesisPlugin(embeddedKinesisStream, queryRunner, + ImmutableMap.builder(). + put(createEmptyStreamDescription(streamName, new SchemaTableName("default", streamName))).build(), + accessKey, secretKey); + } + + private String checkStreamStatus(String streamName) + { + DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); + describeStreamRequest.setStreamName(streamName); + + StreamDescription streamDescription = embeddedKinesisStream.getKinesisClient().describeStream(describeStreamRequest).getStreamDescription(); + return streamDescription.getStreamStatus(); + } + + private void createMessages(String streamName, int count) + throws Exception + { + PutRecordsRequest putRecordsRequest = new PutRecordsRequest(); + putRecordsRequest.setStreamName(streamName); + List putRecordsRequestEntryList = new ArrayList<>(); + for (int i = 0; i < count; i++) { + PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry(); + putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes())); + putRecordsRequestEntry.setPartitionKey(Long.toString(i)); + putRecordsRequestEntryList.add(putRecordsRequestEntry); + } + + while (checkStreamStatus(streamName).equals("ACTIVE") == false) { + log.debug("Waiting for Stream %s to become active. CurrentStatus : %s", streamName, checkStreamStatus(streamName)); + MILLISECONDS.sleep(1000); + } + putRecordsRequest.setRecords(putRecordsRequestEntryList); + embeddedKinesisStream.getKinesisClient().putRecords(putRecordsRequest); + } + + @Test + public void testStreamExists() + throws Exception + { + QualifiedTableName name = new QualifiedTableName("kinesis", "default", streamName); + Optional handle = queryRunner.getServer().getMetadata().getTableHandle(SESSION, name); + assertTrue(handle.isPresent()); + } + + @Test + public void testStreamHasData() + throws Exception + { + MaterializedResult result = queryRunner.execute("Select count(1) from " + streamName); + + MaterializedResult expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) + .row(0) + .build(); + + assertEquals(result, expected); + + int count = 500; + createMessages(streamName, count); + + result = queryRunner.execute("SELECT count(1) from " + streamName); + + expected = MaterializedResult.resultBuilder(SESSION, BigintType.BIGINT) + .row(count) + .build(); + + assertEquals(result, expected); + } + + @AfterMethod + public void tearDown() + throws Exception + { + embeddedKinesisStream.delteStream(streamName); + queryRunner.close(); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/csv/TestCsvDecoder.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/csv/TestCsvDecoder.java new file mode 100644 index 000000000000..da3d6c6106e5 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/csv/TestCsvDecoder.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.csv; + +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkValue; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.testng.annotations.Test; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class TestCsvDecoder +{ + private static final CsvKinesisFieldDecoder DEFAULT_FIELD_DECODER = new CsvKinesisFieldDecoder(); + + private static Map> buildMap(List columns) + { + ImmutableMap.Builder> map = ImmutableMap.builder(); + for (KinesisColumnHandle column : columns) { + map.put(column, DEFAULT_FIELD_DECODER); + } + return map.build(); + } + + @Test + public void testSimple() + { + String csv = "\"row 1\",row2,\"row3\",100,\"200\",300,4.5"; + + CsvKinesisRowDecoder rowDecoder = new CsvKinesisRowDecoder(); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "0", null, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "1", null, null, false, false); + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "2", null, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "3", null, null, false, false); + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BigintType.BIGINT, "4", null, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", BigintType.BIGINT, "5", null, null, false, false); + KinesisColumnHandle row7 = new KinesisColumnHandle("", 6, "row7", DoubleType.DOUBLE, "6", null, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6, row7); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(csv.getBytes(StandardCharsets.UTF_8), providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + checkValue(providers, row1, "row 1"); + checkValue(providers, row2, "row2"); + checkValue(providers, row3, "row3"); + checkValue(providers, row4, 100); + checkValue(providers, row5, 200); + checkValue(providers, row6, 300); + checkValue(providers, row7, 4.5d); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestISO8601JsonKinesisFieldDecoder.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestISO8601JsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..bede0f721694 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestISO8601JsonKinesisFieldDecoder.java @@ -0,0 +1,136 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkValue; +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkIsNull; +import static com.facebook.presto.kinesis.decoder.KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import io.airlift.json.ObjectMapperProvider; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; +import org.testng.annotations.Test; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class TestISO8601JsonKinesisFieldDecoder +{ + private static final Map DECODERS = ImmutableMap.of(DEFAULT_FIELD_DECODER_NAME, new JsonKinesisFieldDecoder(), + ISO8601JsonKinesisFieldDecoder.NAME, new ISO8601JsonKinesisFieldDecoder()); + + private static final ObjectMapperProvider PROVIDER = new ObjectMapperProvider(); + + private static final DateTimeFormatter PRINTER = ISODateTimeFormat.dateTime().withLocale(Locale.ENGLISH).withZoneUTC(); + + private static Map> buildMap(List columns) + { + ImmutableMap.Builder> map = ImmutableMap.builder(); + for (KinesisColumnHandle column : columns) { + map.put(column, DECODERS.get(column.getDataFormat())); + } + return map.build(); + } + + @Test + public void testBasicFormatting() + throws Exception + { + long now = System.currentTimeMillis(); + String nowString = PRINTER.print(now); + + byte[] json = format("{\"a_number\":%d,\"a_string\":\"%s\"}", now, nowString).getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false); + + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); + + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + // sanity checks + checkValue(providers, row1, now); + checkValue(providers, row2, nowString); + + // number parsed as number --> as is + checkValue(providers, row3, now); + // string parsed as number --> parse text, convert to timestamp + checkValue(providers, row4, now); + + // number parsed as string --> parse text, convert to timestamp, turn into string + checkValue(providers, row5, Long.toString(now)); + + // string parsed as string --> as is + checkValue(providers, row6, nowString); + } + + @Test + public void testNullValues() + throws Exception + { + byte[] json = "{}".getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false); + + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); + + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", ISO8601JsonKinesisFieldDecoder.NAME, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + // sanity checks + checkIsNull(providers, row1); + checkIsNull(providers, row2); + checkIsNull(providers, row3); + checkIsNull(providers, row4); + checkIsNull(providers, row5); + checkIsNull(providers, row6); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestJsonDecoder.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestJsonDecoder.java new file mode 100644 index 000000000000..50b6af74f195 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestJsonDecoder.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkIsNull; +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkValue; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.testng.annotations.Test; + +import io.airlift.json.ObjectMapperProvider; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.ByteStreams; + +public class TestJsonDecoder +{ + private static final JsonKinesisFieldDecoder DEFAULT_FIELD_DECODER = new JsonKinesisFieldDecoder(); + private static final ObjectMapperProvider PROVIDER = new ObjectMapperProvider(); + + private static Map> buildMap(List columns) + { + ImmutableMap.Builder> map = ImmutableMap.builder(); + for (KinesisColumnHandle column : columns) { + map.put(column, DEFAULT_FIELD_DECODER); + } + return map.build(); + } + + @Test + public void testSimple() + throws Exception + { + byte[] json = ByteStreams.toByteArray(TestJsonDecoder.class.getResourceAsStream("/decoder/json/message.json")); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "source", null, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "user/screen_name", null, null, false, false); + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "id", null, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "user/statuses_count", null, null, false, false); + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", BooleanType.BOOLEAN, "user/geo_enabled", null, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + checkValue(providers, row1, "twitterfeed"); + checkValue(providers, row2, "EKentuckyNews"); + checkValue(providers, row3, 493857959588286460L); + checkValue(providers, row4, 7630); + checkValue(providers, row5, true); + } + + @Test + public void testNonExistent() + throws Exception + { + byte[] json = "{}".getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "very/deep/varchar", null, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "no_bigint", null, null, false, false); + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", DoubleType.DOUBLE, "double/is_missing", null, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BooleanType.BOOLEAN, "hello", null, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + checkIsNull(providers, row1); + checkIsNull(providers, row2); + checkIsNull(providers, row3); + checkIsNull(providers, row4); + } + + @Test + public void testStringNumber() + throws Exception + { + byte[] json = "{\"a_number\":481516,\"a_string\":\"2342\"}".getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, "a_number", null, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "a_number", null, null, false, false); + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "a_string", null, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", null, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + checkValue(providers, row1, "481516"); + checkValue(providers, row2, 481516); + checkValue(providers, row3, "2342"); + checkValue(providers, row4, 2342); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestMillisecondsSinceEpochJsonKinesisFieldDecoder.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestMillisecondsSinceEpochJsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..83ae29942b47 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestMillisecondsSinceEpochJsonKinesisFieldDecoder.java @@ -0,0 +1,131 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkIsNull; +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkValue; +import static com.facebook.presto.kinesis.decoder.KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import io.airlift.json.ObjectMapperProvider; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.testng.annotations.Test; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class TestMillisecondsSinceEpochJsonKinesisFieldDecoder +{ + private static final Map DECODERS = ImmutableMap.of(DEFAULT_FIELD_DECODER_NAME, new JsonKinesisFieldDecoder(), + MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, new MillisecondsSinceEpochJsonKinesisFieldDecoder()); + + private static final ObjectMapperProvider PROVIDER = new ObjectMapperProvider(); + + private static Map> buildMap(List columns) + { + ImmutableMap.Builder> map = ImmutableMap.builder(); + for (KinesisColumnHandle column : columns) { + map.put(column, DECODERS.get(column.getDataFormat())); + } + return map.build(); + } + + @Test + public void testBasicFormatting() + throws Exception + { + long now = System.currentTimeMillis(); + String nowString = MillisecondsSinceEpochJsonKinesisFieldDecoder.FORMATTER.print(now); + + byte[] json = format("{\"a_number\":%d,\"a_string\":\"%d\"}", now, now).getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false); + + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + // sanity checks + checkValue(providers, row1, now); + checkValue(providers, row2, Long.toString(now)); + + // number parsed as number --> return as time stamp (millis) + checkValue(providers, row3, now); + // string parsed as number --> parse text, convert to timestamp + checkValue(providers, row4, now); + + // number parsed as string --> parse text, convert to timestamp, turn into string + checkValue(providers, row5, nowString); + + // string parsed as string --> parse text, convert to timestamp, turn into string + checkValue(providers, row6, nowString); + } + + @Test + public void testNullValues() + throws Exception + { + byte[] json = "{}".getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false); + + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", MillisecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + // sanity checks + checkIsNull(providers, row1); + checkIsNull(providers, row2); + checkIsNull(providers, row3); + checkIsNull(providers, row4); + checkIsNull(providers, row5); + checkIsNull(providers, row6); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestRFC2822JsonKinesisFieldDecoder.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestRFC2822JsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..a46e03b94077 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestRFC2822JsonKinesisFieldDecoder.java @@ -0,0 +1,132 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.facebook.presto.kinesis.decoder.KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; +import static com.facebook.presto.kinesis.decoder.json.RFC2822JsonKinesisFieldDecoder.FORMATTER; +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkIsNull; +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkValue; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import io.airlift.json.ObjectMapperProvider; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.testng.annotations.Test; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class TestRFC2822JsonKinesisFieldDecoder +{ + private static final Map DECODERS = ImmutableMap.of(DEFAULT_FIELD_DECODER_NAME, new JsonKinesisFieldDecoder(), + RFC2822JsonKinesisFieldDecoder.NAME, new RFC2822JsonKinesisFieldDecoder()); + + private static final ObjectMapperProvider PROVIDER = new ObjectMapperProvider(); + + private static Map> map(List columns) + { + ImmutableMap.Builder> map = ImmutableMap.builder(); + for (KinesisColumnHandle column : columns) { + map.put(column, DECODERS.get(column.getDataFormat())); + } + return map.build(); + } + + @Test + public void testBasicFormatting() + throws Exception + { + long now = (System.currentTimeMillis() / 1000) * 1000; // rfc2822 is second granularity + String nowString = FORMATTER.print(now); + + byte[] json = format("{\"a_number\":%d,\"a_string\":\"%s\"}", now, nowString).getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false); + + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false); + + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, map(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + // sanity checks + checkValue(providers, row1, now); + checkValue(providers, row2, nowString); + + // number parsed as number --> as is + checkValue(providers, row3, now); + // string parsed as number --> parse text, convert to timestamp + checkValue(providers, row4, now); + + // number parsed as string --> parse text, convert to timestamp, turn into string + checkValue(providers, row5, Long.toString(now)); + + // string parsed as string --> as is + checkValue(providers, row6, nowString); + } + + @Test + public void testNullValues() + throws Exception + { + byte[] json = "{}".getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false); + + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false); + + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", RFC2822JsonKinesisFieldDecoder.NAME, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, map(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + // sanity checks + checkIsNull(providers, row1); + checkIsNull(providers, row2); + checkIsNull(providers, row3); + checkIsNull(providers, row4); + checkIsNull(providers, row5); + checkIsNull(providers, row6); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestSecondsSinceEpochJsonKinesisFieldDecoder.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestSecondsSinceEpochJsonKinesisFieldDecoder.java new file mode 100644 index 000000000000..e1fce66c6e56 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/json/TestSecondsSinceEpochJsonKinesisFieldDecoder.java @@ -0,0 +1,132 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.json; + +import static com.facebook.presto.kinesis.decoder.KinesisFieldDecoder.DEFAULT_FIELD_DECODER_NAME; +import static com.facebook.presto.kinesis.decoder.json.SecondsSinceEpochJsonKinesisFieldDecoder.FORMATTER; +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkIsNull; +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkValue; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; +import io.airlift.json.ObjectMapperProvider; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.testng.annotations.Test; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class TestSecondsSinceEpochJsonKinesisFieldDecoder +{ + private static final Map DECODERS = ImmutableMap.of(DEFAULT_FIELD_DECODER_NAME, new JsonKinesisFieldDecoder(), + SecondsSinceEpochJsonKinesisFieldDecoder.NAME, new SecondsSinceEpochJsonKinesisFieldDecoder()); + + private static final ObjectMapperProvider PROVIDER = new ObjectMapperProvider(); + + private static Map> buildMap(List columns) + { + ImmutableMap.Builder> map = ImmutableMap.builder(); + for (KinesisColumnHandle column : columns) { + map.put(column, DECODERS.get(column.getDataFormat())); + } + return map.build(); + } + + @Test + public void testBasicFormatting() + throws Exception + { + long now = System.currentTimeMillis() / 1000; // SecondsSinceEpoch is second granularity + String nowString = FORMATTER.print(now * 1000); + + byte[] json = format("{\"a_number\":%d,\"a_string\":\"%d\"}", now, now).getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false); + + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + // sanity checks + checkValue(providers, row1, now); + checkValue(providers, row2, Long.toString(now)); + + // number parsed as number --> return as time stamp (millis) + checkValue(providers, row3, now * 1000); + // string parsed as number --> parse text, convert to timestamp + checkValue(providers, row4, now * 1000); + + // number parsed as string --> parse text, convert to timestamp, turn into string + checkValue(providers, row5, nowString); + + // string parsed as string --> parse text, convert to timestamp, turn into string + checkValue(providers, row6, nowString); + } + + @Test + public void testNullValues() + throws Exception + { + byte[] json = "{}".getBytes(StandardCharsets.UTF_8); + + JsonKinesisRowDecoder rowDecoder = new JsonKinesisRowDecoder(PROVIDER.get()); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "a_number", DEFAULT_FIELD_DECODER_NAME, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "a_string", DEFAULT_FIELD_DECODER_NAME, null, false, false); + + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "a_number", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + KinesisColumnHandle row6 = new KinesisColumnHandle("", 5, "row6", VarcharType.VARCHAR, "a_string", SecondsSinceEpochJsonKinesisFieldDecoder.NAME, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5, row6); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(json, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + // sanity checks + checkIsNull(providers, row1); + checkIsNull(providers, row2); + checkIsNull(providers, row3); + checkIsNull(providers, row4); + checkIsNull(providers, row5); + checkIsNull(providers, row6); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/raw/TestRawDecoder.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/raw/TestRawDecoder.java new file mode 100644 index 000000000000..d1510dc653db --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/raw/TestRawDecoder.java @@ -0,0 +1,233 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.raw; + +import static com.facebook.presto.kinesis.decoder.util.DecoderTestUtil.checkValue; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.testng.annotations.Test; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; +import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; +import com.facebook.presto.spi.type.BigintType; +import com.facebook.presto.spi.type.BooleanType; +import com.facebook.presto.spi.type.VarcharType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class TestRawDecoder +{ + private static final RawKinesisFieldDecoder DEFAULT_FIELD_DECODER = new RawKinesisFieldDecoder(); + + private static Map> buildMap(List columns) + { + ImmutableMap.Builder> map = ImmutableMap.builder(); + for (KinesisColumnHandle column : columns) { + map.put(column, DEFAULT_FIELD_DECODER); + } + return map.build(); + } + + @Test + public void testSimple() + { + ByteBuffer buf = ByteBuffer.allocate(100); + buf.putLong(4815162342L); // 0 - 7 + buf.putInt(12345678); // 8 - 11 + buf.putShort((short) 4567); // 12 - 13 + buf.put((byte) 123); // 14 + buf.put("Ich bin zwei Oeltanks".getBytes(StandardCharsets.UTF_8)); // 15+ + + byte[] row = new byte[buf.position()]; + System.arraycopy(buf.array(), 0, row, 0, buf.position()); + + RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder(); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", BigintType.BIGINT, "0", "LONG", null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", BigintType.BIGINT, "8", "INT", null, false, false); + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", BigintType.BIGINT, "12", "SHORT", null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", BigintType.BIGINT, "14", "BYTE", null, false, false); + KinesisColumnHandle row5 = new KinesisColumnHandle("", 4, "row5", VarcharType.VARCHAR, "15", null, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4, row5); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + checkValue(providers, row1, 4815162342L); + checkValue(providers, row2, 12345678); + checkValue(providers, row3, 4567); + checkValue(providers, row4, 123); + checkValue(providers, row5, "Ich bin zwei Oeltanks"); + } + + @Test + public void testFixedWithString() + { + String str = "Ich bin zwei Oeltanks"; + byte[] row = str.getBytes(StandardCharsets.UTF_8); + + RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder(); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, null, null, null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "0", null, null, false, false); + KinesisColumnHandle row3 = new KinesisColumnHandle("", 2, "row3", VarcharType.VARCHAR, "0:4", null, null, false, false); + KinesisColumnHandle row4 = new KinesisColumnHandle("", 3, "row4", VarcharType.VARCHAR, "5:8", null, null, false, false); + + List columns = ImmutableList.of(row1, row2, row3, row4); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + checkValue(providers, row1, str); + checkValue(providers, row2, str); + // these only work for single byte encodings... + checkValue(providers, row3, str.substring(0, 4)); + checkValue(providers, row4, str.substring(5, 8)); + } + + @SuppressWarnings("NumericCastThatLosesPrecision") + @Test + public void testFloatStuff() + { + ByteBuffer buf = ByteBuffer.allocate(100); + buf.putDouble(Math.PI); + buf.putFloat((float) Math.E); + buf.putDouble(Math.E); + + byte[] row = new byte[buf.position()]; + System.arraycopy(buf.array(), 0, row, 0, buf.position()); + + RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder(); + KinesisColumnHandle row1 = new KinesisColumnHandle("", 0, "row1", VarcharType.VARCHAR, null, "DOUBLE", null, false, false); + KinesisColumnHandle row2 = new KinesisColumnHandle("", 1, "row2", VarcharType.VARCHAR, "8", "FLOAT", null, false, false); + + List columns = ImmutableList.of(row1, row2); + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + checkValue(providers, row1, Math.PI); + checkValue(providers, row2, Math.E); + } + + @Test + public void testBooleanStuff() + { + ByteBuffer buf = ByteBuffer.allocate(100); + buf.put((byte) 127); // offset 0 + buf.putLong(0); // offset 1 + buf.put((byte) 126); // offset 9 + buf.putLong(1); // offset 10 + + buf.put((byte) 125); // offset 18 + buf.putInt(0); // offset 19 + buf.put((byte) 124); // offset 23 + buf.putInt(1); // offset 24 + + buf.put((byte) 123); // offset 28 + buf.putShort((short) 0); // offset 29 + buf.put((byte) 122); // offset 31 + buf.putShort((short) 1); // offset 32 + + buf.put((byte) 121); // offset 34 + buf.put((byte) 0); // offset 35 + buf.put((byte) 120); // offset 36 + buf.put((byte) 1); // offset 37 + + byte[] row = new byte[buf.position()]; + System.arraycopy(buf.array(), 0, row, 0, buf.position()); + + RawKinesisRowDecoder rowDecoder = new RawKinesisRowDecoder(); + KinesisColumnHandle row01 = new KinesisColumnHandle("", 0, "row01", BigintType.BIGINT, "0", "BYTE", null, false, false); + KinesisColumnHandle row02 = new KinesisColumnHandle("", 1, "row02", BooleanType.BOOLEAN, "1", "LONG", null, false, false); + KinesisColumnHandle row03 = new KinesisColumnHandle("", 2, "row03", BigintType.BIGINT, "9", "BYTE", null, false, false); + KinesisColumnHandle row04 = new KinesisColumnHandle("", 3, "row04", BooleanType.BOOLEAN, "10", "LONG", null, false, false); + + KinesisColumnHandle row11 = new KinesisColumnHandle("", 4, "row11", BigintType.BIGINT, "18", "BYTE", null, false, false); + KinesisColumnHandle row12 = new KinesisColumnHandle("", 5, "row12", BooleanType.BOOLEAN, "19", "INT", null, false, false); + KinesisColumnHandle row13 = new KinesisColumnHandle("", 6, "row13", BigintType.BIGINT, "23", "BYTE", null, false, false); + KinesisColumnHandle row14 = new KinesisColumnHandle("", 7, "row14", BooleanType.BOOLEAN, "24", "INT", null, false, false); + + KinesisColumnHandle row21 = new KinesisColumnHandle("", 8, "row21", BigintType.BIGINT, "28", "BYTE", null, false, false); + KinesisColumnHandle row22 = new KinesisColumnHandle("", 9, "row22", BooleanType.BOOLEAN, "29", "SHORT", null, false, false); + KinesisColumnHandle row23 = new KinesisColumnHandle("", 10, "row23", BigintType.BIGINT, "31", "BYTE", null, false, false); + KinesisColumnHandle row24 = new KinesisColumnHandle("", 11, "row24", BooleanType.BOOLEAN, "32", "SHORT", null, false, false); + + KinesisColumnHandle row31 = new KinesisColumnHandle("", 12, "row31", BigintType.BIGINT, "34", "BYTE", null, false, false); + KinesisColumnHandle row32 = new KinesisColumnHandle("", 13, "row32", BooleanType.BOOLEAN, "35", "BYTE", null, false, false); + KinesisColumnHandle row33 = new KinesisColumnHandle("", 14, "row33", BigintType.BIGINT, "36", "BYTE", null, false, false); + KinesisColumnHandle row34 = new KinesisColumnHandle("", 15, "row34", BooleanType.BOOLEAN, "37", "BYTE", null, false, false); + + List columns = ImmutableList.of(row01, + row02, + row03, + row04, + row11, + row12, + row13, + row14, + row21, + row22, + row23, + row24, + row31, + row32, + row33, + row34); + + Set providers = new HashSet<>(); + + boolean valid = rowDecoder.decodeRow(row, providers, columns, buildMap(columns)); + assertTrue(valid); + + assertEquals(providers.size(), columns.size()); + + checkValue(providers, row01, 127); + checkValue(providers, row02, false); + checkValue(providers, row03, 126); + checkValue(providers, row04, true); + + checkValue(providers, row11, 125); + checkValue(providers, row12, false); + checkValue(providers, row13, 124); + checkValue(providers, row14, true); + + checkValue(providers, row21, 123); + checkValue(providers, row22, false); + checkValue(providers, row23, 122); + checkValue(providers, row24, true); + + checkValue(providers, row31, 121); + checkValue(providers, row32, false); + checkValue(providers, row33, 120); + checkValue(providers, row34, true); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/util/DecoderTestUtil.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/util/DecoderTestUtil.java new file mode 100644 index 000000000000..faa118022e1c --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/decoder/util/DecoderTestUtil.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.decoder.util; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.Set; + +import com.facebook.presto.kinesis.KinesisColumnHandle; +import com.facebook.presto.kinesis.KinesisFieldValueProvider; + +public class DecoderTestUtil +{ + private DecoderTestUtil() {} + + private static KinesisFieldValueProvider findValueProvider(Set providers, KinesisColumnHandle handle) + { + for (KinesisFieldValueProvider provider : providers) { + if (provider.accept(handle)) { + return provider; + } + } + return null; + } + + public static void checkValue(Set providers, KinesisColumnHandle handle, String value) + { + KinesisFieldValueProvider provider = findValueProvider(providers, handle); + assertNotNull(provider); + assertEquals(new String(provider.getSlice().getBytes(), StandardCharsets.UTF_8), value); + } + + public static void checkValue(Set providers, KinesisColumnHandle handle, long value) + { + KinesisFieldValueProvider provider = findValueProvider(providers, handle); + assertNotNull(provider); + assertEquals(provider.getLong(), value); + } + + public static void checkValue(Set providers, KinesisColumnHandle handle, double value) + { + KinesisFieldValueProvider provider = findValueProvider(providers, handle); + assertNotNull(provider); + assertEquals(provider.getDouble(), value, 0.0001); + } + + public static void checkValue(Set providers, KinesisColumnHandle handle, boolean value) + { + KinesisFieldValueProvider provider = findValueProvider(providers, handle); + assertNotNull(provider); + assertEquals(provider.getBoolean(), value); + } + + public static void checkIsNull(Set providers, KinesisColumnHandle handle) + { + KinesisFieldValueProvider provider = findValueProvider(providers, handle); + assertNotNull(provider); + assertTrue(provider.isNull()); + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/util/EmbeddedKinesisStream.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/util/EmbeddedKinesisStream.java new file mode 100644 index 000000000000..b492a3d5f796 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/util/EmbeddedKinesisStream.java @@ -0,0 +1,79 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.util; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; + +import com.amazonaws.services.kinesis.AmazonKinesisClient; +import com.amazonaws.services.kinesis.model.CreateStreamRequest; +import com.amazonaws.services.kinesis.model.DeleteStreamRequest; +import com.facebook.presto.kinesis.KinesisAwsCredentials; + +public class EmbeddedKinesisStream + implements Closeable +{ + private KinesisAwsCredentials awsCredentials; + private AmazonKinesisClient amazonKinesisClient; + private ArrayList streamsCreated = new ArrayList(); + + public EmbeddedKinesisStream(String accessKey, String secretKey) + { + this.awsCredentials = new KinesisAwsCredentials(accessKey, secretKey); + this.amazonKinesisClient = new AmazonKinesisClient(awsCredentials); + } + + @Override + public void close() throws IOException + { + } + + public void createStreams(String... streamNames) + { + createStreams(2, streamNames); + } + + public void createStreams(int shardCount, String... streamNames) + { + for (String streamName : streamNames) { + createStream(shardCount, streamName); + } + } + + public void createStream(int shardCount, String streamName) + { + CreateStreamRequest createStreamRequest = new CreateStreamRequest(); + createStreamRequest.setStreamName(streamName); + createStreamRequest.setShardCount(shardCount); + + amazonKinesisClient.createStream(createStreamRequest); + streamsCreated.add(streamName); + } + + public AmazonKinesisClient getKinesisClient() + { + return amazonKinesisClient; + } + + public void delteStream(String streamName) + { + DeleteStreamRequest deleteStreamRequest = new DeleteStreamRequest(); + deleteStreamRequest.setStreamName(streamName); + amazonKinesisClient.deleteStream(deleteStreamRequest); + if (streamsCreated.contains(streamName)) { + streamsCreated.remove(streamName); + } + } +} diff --git a/presto-kinesis/src/test/java/com/facebook/presto/kinesis/util/TestUtils.java b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/util/TestUtils.java new file mode 100644 index 000000000000..12a5577b0d13 --- /dev/null +++ b/presto-kinesis/src/test/java/com/facebook/presto/kinesis/util/TestUtils.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.kinesis.util; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.AbstractMap; +import java.util.Map; +import java.util.Properties; + +import com.facebook.presto.kinesis.KinesisPlugin; +import com.facebook.presto.kinesis.KinesisStreamDescription; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.testing.QueryRunner; +import com.google.common.base.Joiner; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableMap; + +public class TestUtils +{ + private TestUtils() {} + + public static int findUnusedPort() + throws IOException + { + try (ServerSocket socket = new ServerSocket(0)) { + return socket.getLocalPort(); + } + } + + public static Properties toProperties(Map map) + { + Properties properties = new Properties(); + for (Map.Entry entry : map.entrySet()) { + properties.setProperty(entry.getKey(), entry.getValue()); + } + return properties; + } + + public static void installKinesisPlugin(EmbeddedKinesisStream embeddedKinesisStream, QueryRunner queryRunner, Map streamDescriptions, String accessKey, String secretKey) + { + KinesisPlugin kinesisPlugin = new KinesisPlugin(); + kinesisPlugin.setTableDescriptionSupplier(Suppliers.ofInstance(streamDescriptions)); + queryRunner.installPlugin(kinesisPlugin); + + Map kinesisConfig = ImmutableMap.of( + "kinesis.table-names", Joiner.on(",").join(streamDescriptions.keySet()), + "kinesis.default-schema", "default", + "kinesis.access-key", accessKey, + "kinesis.secret-key", secretKey); + queryRunner.createCatalog("kinesis", "kinesis", kinesisConfig); + } + + public static Map.Entry createEmptyStreamDescription(String streamName, SchemaTableName schemaTableName) + { + return new AbstractMap.SimpleImmutableEntry<>( + schemaTableName, + new KinesisStreamDescription(schemaTableName.getTableName(), schemaTableName.getSchemaName(), streamName, null)); + } +} diff --git a/presto-kinesis/src/test/resources/decoder/json/message.json b/presto-kinesis/src/test/resources/decoder/json/message.json new file mode 100644 index 000000000000..9a0743950cb9 --- /dev/null +++ b/presto-kinesis/src/test/resources/decoder/json/message.json @@ -0,0 +1,80 @@ +{ + "created_at": "Mon Jul 28 20:38:07 +0000 2014", + "id": 493857959588286460, + "id_str": "493857959588286465", + "text": "Lots of Important Preseason Football Dates on the Horizon - EKU Sports: Lots of Important Preseason Football D... http://t.co/F7iz6APFTW", + "source": "twitterfeed", + "truncated": false, + "in_reply_to_status_id": null, + "in_reply_to_status_id_str": null, + "in_reply_to_user_id": null, + "in_reply_to_user_id_str": null, + "in_reply_to_screen_name": null, + "user": { + "id": 98247748, + "id_str": "98247748", + "name": "Eastern KY News", + "screen_name": "EKentuckyNews", + "location": "Eastern Kentucky", + "url": null, + "description": "Your Eastern Kentucky News Source.", + "protected": false, + "verified": false, + "followers_count": 305, + "friends_count": 186, + "listed_count": 8, + "favourites_count": 0, + "statuses_count": 7630, + "created_at": "Mon Dec 21 01:17:22 +0000 2009", + "utc_offset": -14400, + "time_zone": "Eastern Time (US & Canada)", + "geo_enabled": true, + "lang": "en", + "contributors_enabled": false, + "is_translator": false, + "profile_background_color": "C6E2EE", + "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", + "profile_background_tile": false, + "profile_link_color": "1F98C7", + "profile_sidebar_border_color": "C6E2EE", + "profile_sidebar_fill_color": "DAECF4", + "profile_text_color": "8F433C", + "profile_use_background_image": true, + "profile_image_url": "http://pbs.twimg.com/profile_images/1297233295/Kentucky_at_Work_logo_normal.jpeg", + "profile_image_url_https": "https://pbs.twimg.com/profile_images/1297233295/Kentucky_at_Work_logo_normal.jpeg", + "default_profile": false, + "default_profile_image": false, + "following": null, + "follow_request_sent": null, + "notifications": null + }, + "geo": null, + "coordinates": null, + "place": null, + "contributors": null, + "retweet_count": 0, + "favorite_count": 0, + "entities": { + "hashtags": [], + "trends": [], + "urls": [ + { + "url": "http://t.co/F7iz6APFTW", + "expanded_url": "http://bit.ly/1rTEYRM", + "display_url": "bit.ly/1rTEYRM", + "indices": [ + 114, + 136 + ] + } + ], + "user_mentions": [], + "symbols": [] + }, + "favorited": false, + "retweeted": false, + "possibly_sensitive": false, + "filter_level": "medium", + "lang": "en" +} From e3489bc69d8eda697a531439c53c066c5053b811 Mon Sep 17 00:00:00 2001 From: shubham agarwal Date: Fri, 8 May 2015 06:38:29 -0400 Subject: [PATCH 3/7] Changes in POM --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 4a18c051ccfb..9cb3a21f4050 100644 --- a/pom.xml +++ b/pom.xml @@ -63,13 +63,13 @@ presto-spi presto-kafka presto-kinesis - + presto-hive-cdh5 presto-example-http presto-tpch presto-raptor From b2c8eb6f992d801af58f4dc2c3b389377161d288 Mon Sep 17 00:00:00 2001 From: shubham agarwal Date: Tue, 12 May 2015 03:02:14 -0400 Subject: [PATCH 4/7] Kinesis-version updated --- presto-kinesis/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/presto-kinesis/pom.xml b/presto-kinesis/pom.xml index 92b08591a5b8..073fff154274 100644 --- a/presto-kinesis/pom.xml +++ b/presto-kinesis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.102 + 0.103-SNAPSHOT From 71368e14df8d62454c42bb14d5ea0013647cbb6d Mon Sep 17 00:00:00 2001 From: shubham agarwal Date: Fri, 15 May 2015 04:23:20 -0400 Subject: [PATCH 5/7] describe-stream check at Coordinator --- .../presto/kinesis/KinesisRecordSet.java | 64 ++++++------------- .../presto/kinesis/KinesisSplitManager.java | 13 +++- 2 files changed, 29 insertions(+), 48 deletions(-) diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSet.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSet.java index 114d697fa5e4..6ceb548fe481 100644 --- a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSet.java +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisRecordSet.java @@ -26,14 +26,12 @@ import java.util.Map; import java.util.Set; -import com.amazonaws.services.kinesis.model.DescribeStreamRequest; import com.amazonaws.services.kinesis.model.GetRecordsRequest; import com.amazonaws.services.kinesis.model.GetRecordsResult; import com.amazonaws.services.kinesis.model.GetShardIteratorRequest; import com.amazonaws.services.kinesis.model.GetShardIteratorResult; import com.amazonaws.services.kinesis.model.Record; import com.amazonaws.services.kinesis.model.ResourceNotFoundException; -import com.amazonaws.services.kinesis.model.StreamDescription; import com.facebook.presto.kinesis.decoder.KinesisFieldDecoder; import com.facebook.presto.kinesis.decoder.KinesisRowDecoder; import com.facebook.presto.spi.RecordCursor; @@ -139,49 +137,33 @@ public Type getType(int field) @Override public boolean advanceNextPosition() { - try { - if (shardIterator == null) { - getIterator(); - if (getKinesisRecords() == false) { - log.debug("No more records in shard to read."); - return false; - } + if (shardIterator == null) { + getIterator(); + if (getKinesisRecords() == false) { + log.debug("No more records in shard to read."); + return false; } + } - while (true) { - log.debug("Reading data from shardIterator %s", shardIterator); - while (listIterator.hasNext()) { - return nextRow(); - } + while (true) { + log.debug("Reading data from shardIterator %s", shardIterator); + while (listIterator.hasNext()) { + return nextRow(); + } - shardIterator = getRecordsResult.getNextShardIterator(); + shardIterator = getRecordsResult.getNextShardIterator(); - if (shardIterator == null) { - log.debug("Shard closed"); + if (shardIterator == null) { + log.debug("Shard closed"); + return false; + } + else { + if (getKinesisRecords() == false) { + log.debug("No more records in shard to read."); return false; } - else { - if (getKinesisRecords() == false) { - log.debug("No more records in shard to read."); - return false; - } - } } } - - catch (ResourceNotFoundException e) { - log.debug("Stream %s not active.", split.getStreamName()); - return false; - } - } - - private String checkStreamStatus(String streamName) - { - DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest(); - describeStreamRequest.setStreamName(streamName); - - StreamDescription streamDescription = clientManager.getClient().describeStream(describeStreamRequest).getStreamDescription(); - return streamDescription.getStreamStatus(); } private boolean getKinesisRecords() @@ -191,10 +173,6 @@ private boolean getKinesisRecords() getRecordsRequest.setShardIterator(shardIterator); getRecordsRequest.setLimit(25); - if (checkStreamStatus(split.getStreamName()).equals("ACTIVE") == false) { - throw new ResourceNotFoundException("Stream not Active"); - } - getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest); kinesisRecords = getRecordsResult.getRecords(); if (kinesisRecords.isEmpty()) { @@ -307,10 +285,6 @@ private void getIterator() getShardIteratorRequest.setShardId(split.getShardId()); getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON"); - if (checkStreamStatus(split.getStreamName()).equals("ACTIVE") == false) { - throw new ResourceNotFoundException("Stream not Active"); - } - GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest); shardIterator = getShardIteratorResult.getShardIterator(); } diff --git a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplitManager.java b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplitManager.java index 6af64c1e0063..14ac70a2871b 100644 --- a/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplitManager.java +++ b/presto-kinesis/src/main/java/com/facebook/presto/kinesis/KinesisSplitManager.java @@ -20,6 +20,7 @@ import com.amazonaws.services.kinesis.model.DescribeStreamRequest; import com.amazonaws.services.kinesis.model.DescribeStreamResult; +import com.amazonaws.services.kinesis.model.ResourceNotFoundException; import com.amazonaws.services.kinesis.model.Shard; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorPartition; @@ -66,12 +67,18 @@ public ConnectorPartitionResult getPartitions(ConnectorTableHandle tableHandle, DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest(); describeStreamRequest.setStreamName(kinesisTableHandle.getStreamName()); + String exclusiveStartShardId = null; + describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); + DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest); + + String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus(); + while ((streamStatus.equals("ACTIVE") == false) && (streamStatus.equals("UPDATING") == false)) { + throw new ResourceNotFoundException("Stream not Active"); + } + List shards = new ArrayList<>(); ImmutableList.Builder builder = ImmutableList.builder(); - String exclusiveStartShardId = null; do { - describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId); - DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest); shards.addAll(describeStreamResult.getStreamDescription().getShards()); for (Shard shard : shards) { From 32cd4a175be53eb79333f0fd78b1dcb956dd110f Mon Sep 17 00:00:00 2001 From: shubham agarwal Date: Thu, 11 Jun 2015 08:40:37 +0000 Subject: [PATCH 6/7] Adding Kinesis Documentation --- .../sphinx/connector/kinesis-tutorial.rst | 252 +++++++++++++++++ .../src/main/sphinx/connector/kinesis.rst | 267 ++++++++++++++++++ 2 files changed, 519 insertions(+) create mode 100644 presto-docs/src/main/sphinx/connector/kinesis-tutorial.rst create mode 100644 presto-docs/src/main/sphinx/connector/kinesis.rst diff --git a/presto-docs/src/main/sphinx/connector/kinesis-tutorial.rst b/presto-docs/src/main/sphinx/connector/kinesis-tutorial.rst new file mode 100644 index 000000000000..fd9a61c496ae --- /dev/null +++ b/presto-docs/src/main/sphinx/connector/kinesis-tutorial.rst @@ -0,0 +1,252 @@ +=========================== +Kinesis Connector Tutorial +=========================== + +Introduction +============ + +The Kinesis connector for Presto allows access to data from live +Kinesis streams, fetch it and process the data on user query. This +tutorial shows how to set up Kinesis streams, how to create table +description files that back presto tables. + +Set Up +====== + +This tutorial assumes that you already have AWS use account and have +access to Aamazon Kinesis through access keys of root accout or IAM +roles. It will focus on how to creating streams and pushing in twitter +data, and then how to query the data in Presto. + +..note:: + If using terminal, enter access key id and secret key in ~/.aws/credentials file. + Load script can read access keys from this file to load and fetch data from Kinesis. + +For the purpose of showing how to use data from Kinesis stream, this +tutorial use twitter data and push it into kinesis streams + +Step 1: Create Kinesis Stream +----------------------------- + +Create a stream using aws-kinesis cli command ``create-stream``. It is assumed that +aws credentials are saved in ~/.aws/credentials file. (The tutorial is creating a +'twitter_data' stream with 4 shards to put twitter feeds) + +..code-block:: create_stream.py + + aws kinesis create-stream --stream-name twitter_data --shard-count 4 + +Step 2: Setup a live twiter feed to Kinesis Stream +-------------------------------------------------- + +Download loadTweet.py script- + +..code-block:: create_stream.py + + $ curl -o loadTweets.py https://raw.githubusercontent.com/shubham166/kinesis-load-tweets/master/loadTweets.py + +* Create a developer account at https://dev.twitter.com/ and set up an + access and consumer token. + +* Create a ``twitter.properties`` file and put the access and consumer key + and secrets into it: + +..code-block:: create_stream.py + + api_key = + api_secret = + access_token_key = + access_token_secret = + +Step 3: Create twitter_data table in Presto +------------------------------------------- + +In your Presto Installation, add a catalog properties file +``etc/catalog/kinesis.properties`` for the Kinesis connector. +This file lists all the Kinesis streams and access and secret +key to be used to access them. + +.. code-block:: none + + connector.name=kinesis + kinesis.table-names=twitter_data + kinesis.hide-internal-columns-hidden=false + kinesis.access-key= + kinesis.secret-key= + +Now start Presto: + +.. code-block:: none + + $ bin/launcher start + +Start the :doc:`Presto CLI `: + +.. code-block:: none + + $ ./presto --catalog kinesis + +List the tables to verify that things are working: + +.. code-block:: none + + presto:default> SHOW TABLES; + Table + ------------ + twitter_data + (1 row) + +Step 4 : Feed live tweets to the Stream +--------------------------------------- + +Run the loadTweets.py script with required parameters + +.. code-block:: none + + $ python loadPython.py + +'stream-name' parameter is required while aws-credentials parameters are optional. + +* If AWS credentials are already stored in ~/.aws/credentials file, use + +.. code-block:: none + + $ python loadPython.py twitter_data + +* If credentials not present in ~/.aws/credentials file or want to overwrite with new credentials, use + +.. code-block:: none + + $ python loadPython.py twitter_data + +Step 5: Basic data querying +--------------------------- + +Kinesis data is unstructured and it has no metadata to describe the format of +the messages. Without further configuration, the Kinesis connector can access +the data and map it in raw form but there are no actual columns besides the +built-one one: + +.. code-block:: none + + presto:default> DESCRIBE twitter_data; + Column | Type | Null | Partition Key | Comment + -------------------+---------+------+---------------+--------------------------------------------- + _shard_id | varchar | true | false | Shard Id + _shard_sequence_id | varchar | true | false | sequence id of messages within the shard + _segment_start | varchar | true | false | segment start sequence id + _segment_end | varchar | true | false | segment end sequence id + _segment_count | bigint | true | false | Running message coutn per segment + _partition_key | bigint | true | false | Key text + _message | varchar | true | false | Message text + _message_valid | boolean | true | false | Message data is valid + _message_length | bigint | true | false | Total number of message bytes + (9 rows) + + presto:default> SELECT count(*) FROM twitter_data; + _col0 + ------- + 1500 + + presto:default> SELECT _message FROM twitter_data LIMIT 5; + _message + -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {"created_at":"Mon Jun 01 12:13:24 +0000 2015","id":605346403448160256,"id_str":"605346403448160256","text":"the programming talent myth: https:\/\/t.co\/4IqNkyaI2Q\nmuch good sense here, well-stated. Even if i only agree with half","so + {"created_at":"Mon Jun 01 12:14:41 +0000 2015","id":605346728431091712,"id_str":"605346728431091712","text":"RT @CompSciFact: Functional Programming Fundamentals. 13 lectures by @headinthebox http:\/\/t.co\/fHFwSK9uLk","source":"\u003ca + {"created_at":"Mon Jun 01 12:15:14 +0000 2015","id":605346866595803136,"id_str":"605346866595803136","text":"#ConciergeChoice: This Festival offers bold & innovative programming in dance & theatre #TRANSAM\u0083RIQUES http:\/\/ + {"created_at":"Mon Jun 01 12:16:23 +0000 2015","id":605347157097476096,"id_str":"605347157097476096","text":"RT @stefstivala: 'the most exciting part of computer programming' \ud83d\ude36\ud83d\udd2b http:\/\/t.co\/7iWHkKq68k","source": + {"created_at":"Mon Jun 01 12:17:37 +0000 2015","id":605347466121216000,"id_str":"605347466121216000","text":"tonybaroneee comments on \"The programming talent myth\" - http:\/\/t.co\/eQlYnFyyLU","source":"\u003ca href=\"http:\/\/runwher + +The data from Kinesis streams can be queried using Presto but it is not yet in +actual table shape. The raw data is available through the ``_message``columns +but it is not decoded into columns. As the sample data is in JSON format, the +:doc:`/functions/json` built into Presto can be used to slice the data. + +Step 5: Add a topic decription file +----------------------------------- + +The Kinesis connector supports topic description files to turn raw data into +table format. These files are located in the ``etc/kinesis`` folder in the +Presto installation and must end with ``.json``. It is recommended that +the file name matches the table name but this is not necessary. + +Add the following file as ``etc/kinesis/twitter_data.json`` and restart Presto. + +.. code-block:: json + + { + "tableName": "twitter_data", + "schemaName": "default", + "streamName": "twitter_data", + "message": { + "dataFormat": "json", + "fields": [ + { + "name": "created_at", + "mapping": "created_at", + "type": "TIMESTAMP", + "dataFormat": "rfc2822" + }, + { + "name": "id", + "mapping": "id", + "type": "BIGINT" + }, + { + "name": "name", + "mapping": "user/screen_name", + "type": "VARCHAR" + }, + { + "name": "location", + "mapping": "user/location", + "type": "VARCHAR" + }, + { + "name": "tweet", + "mapping": "text", + "type": "VARCHAR" + }, + { + "name": "hashtag", + "mapping": "entities/hashtags", + "type": "VARCHAR" + } + ] + } + } + +Now the table twitter_data has additional columns: + +.. code-block:: none + + presto:default> DESCRIBE twitter_data; + Column | Type | Null | Partition Key | Comment + -------------------+---------+------+---------------+--------------------------------------------- + create_at |timestamp| true | false | + id | bigint | true | false | + name | varchar | true | false | + location | varchar | true | false | + hashtag | varchar | true | false | + _shard_id | varchar | true | false | Shard Id + _shard_sequence_id | varchar | true | false | sequence id of messages within the shard + _segment_start | varchar | true | false | segment start sequence id + _segment_end | varchar | true | false | segment end sequence id + _segment_count | bigint | true | false | Running message coutn per segment + _partition_key | bigint | true | false | Key text + _message | varchar | true | false | Message text + _message_valid | boolean | true | false | Message data is valid + _message_length | bigint | true | false | Total number of message bytes + (15 rows) + + presto:default> select created_at, id, name, location, tweet, hashtag from twitter_data limit 5; + created_at | id | name | location | tweet | hashtag + -------------------------+--------------------+-------------+-------------+-----------------------------------------------------------------------------------------------+--------- + 2015-06-01 08:12:37.000 | 605346208438202368 | sheyi6002 | Dreamland | What are you terrible at? — maths and programming xD. i dunno http://t.co/EurCvBsdL3 | [] + 2015-06-01 08:13:27.000 | 605346415993221121 | jacksondevs | Jackson, MS | The programming talent myth +| [] + | | | | https://t.co/UvQIh5FBhG | + 2015-06-01 08:13:33.000 | 605346443801600000 | nerdreich | | RT @neillyneil: "the most exciting part of computer programming" http://t.co/5aapjXmNZt | [] + 2015-06-01 08:13:54.000 | 605346531894521857 | numb3r23 | London | RT @stefstivala: 'the most exciting part of computer programming' http://t.co/7iWHkKq68k | [] + 2015-06-01 08:15:05.000 | 605346829216022531 | hnbot | | The programming talent myth +| [] + | | | | (Discussion on HN - http://t.co/1ojLRw77Vd) http://t.co/zMZmUSXoXJ | + (5 rows) diff --git a/presto-docs/src/main/sphinx/connector/kinesis.rst b/presto-docs/src/main/sphinx/connector/kinesis.rst new file mode 100644 index 000000000000..79a2eb93b1ed --- /dev/null +++ b/presto-docs/src/main/sphinx/connector/kinesis.rst @@ -0,0 +1,267 @@ +================= +Kinesis Connector +================= + +Kinesis is Amazon’s fully managed cloud-based service for real-time processing +of large, distributed data streams. + +Analogous to Kafka connector, this connector allows the use of Kinesis streams as tables in Presto, +such that each data-blob in kinesis stream is presented as a row in Presto. +Streams can be live: rows will appear as data is pushed into the stream, +and disappear as they are dropped once their time expires. (A message is held up for 24 hours by kinesis streams). + This can result in strange behavior if accessing the same table multiple times in a single query. (eg. performing self join) + +.. note:: + + This connector is Read-Only connector. It can only fetch data from kinesis streams, but can not create streams or push data into the already existing streams. + + Configuration + ------------- + + To configure the Kinesis Connector, create a catalog properties file + ``etc/catalog/kinesis.properties`` with the following contents, + replacing the properties as appropriate. + + .. code-block:: none + + connector.name=kinesis + kinesis.table-names=table1,table2 + kinesis.access-key= + kinesis.secret-key= + +Configuration Properties +------------------------ + +The following configuration properties are available : + +=================================== ========================================================================= +Property Name Description +=================================== ========================================================================= +``kinesis.table-names`` List of all the tables provided by the catalog. +``kinesis.default-schema`` Default schema name for tables. +``kinesis.table-description-dir`` Directory containing table description files. +``kinesis.access-key`` Access key to aws account. +``kinesis.secret-key`` Secret key to aws account. +``kinesis.hide-internal-columns`` Controls whether internal columns are part of the table schema or not. +``kinesis.aws-region`` Aws region to be used to read kinesis stream from. +``kinesis.batch-size`` Maximum number of records to return. Maximum Limit 10000. +``kinesis.fetch-attempts`` Attempts to be made to fetch data from kinesis streams. +``kinesis.sleep-time`` Time till which thread sleep waiting to make next attempt to fetch data. +=================================== ========================================================================= + +``kinesis.table-names`` +^^^^^^^^^^^^^^^^^^^^^^^ + +Comma-separated list of all tables provided by this catalog. A table name +can be unqualified (simple name) and will be put into the default schema +(see below) or qualified with a scheme name (``.``). + +For each table defined here, a table description file (see below) may exist. +If no table description file exists, the table name is used as the stream +name on Kinesis and no data columns are mapped into the table. The table will +contain all internal columns (see below). + +This property is required; there is no defualt and at least one table must be +defined. + +``kinesis.default-schema`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Defines the schema which will contain all tables that were defined without +a qualifying schema name. + +This property is optional; the default is ``default``. + +``kinesis.table-description-dir`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +References a folder within Presto deployment that holds one or more JSON +files (must end wiht ``.json``) which contain table description files. + +This property is optional; the default is ``etc/kinesis``. + +``kinesis.access-key`` +^^^^^^^^^^^^^^^^^^^^^^ + +Defines the access key ID for AWS root account or IAM roles, which is +used to sign programmatic requests to AWS Kinesis. + +This property is required; it must be defined to access Kinesis streams. + +``kinesis.secret-key`` +^^^^^^^^^^^^^^^^^^^^^^ + +Defines the secret key for AWS root account or IAM roles, which together +with Access Key ID, is used to sign programmatic requests to AWS Kinesis. + +This property is required; it must be defined to access Kinesis streams. + +``kinesis.aws-region`` +^^^^^^^^^^^^^^^^^^^^^^ + +Defines AWS Kinesis regional endpoint. Selecting appropriate region may +reduce latency in fetching data. + +This field is optional; The default region is ``us-east-1`` referring to +end point 'kinesis.us-east-1.amazonaws.com'. + +Amazon Kinesis Regions +---------------------- + +For each Amazon Kinesis account, following availabe regions can be used: + +======================= =========================== ========================================= +Region Region Name Endpoint +======================= =========================== ========================================= +``us-east-1`` US East (N. Virginia) kinesis.us-east-1.amazonaws.com +``us-west-1`` US West (N. California) kinesis.us-west-1.amazonaws.com +``us-west-2`` US West (Oregon) kinesis.us-west-2.amazonaws.com +``eu-west-1`` EU (Ireland) kinesis.eu-west-1.amazonaws.com +``eu-central-1`` EU (Frankfurt) kinesis.eu-central-1.amazonaws.com +``ap-southeast-1`` Asia Pacific (Singapore) kinesis.ap-southeast-1.amazonaws.com +``ap-southeast-2`` Asia Pacific (Sydney) kinesis.ap-southeast-2.amazonaws.com +``ap-northeast-1`` Asia Pacific (Tokyo) kinesis.ap-northeast-1.amazonaws.com +======================= =========================== ========================================== + +``kinesis.batch-size`` +^^^^^^^^^^^^^^^^^^^^^^ + +Defines maximum number of records to return in one request to Kinesis Streams. Maximum Limit is +10000 records. If a value greater than 10000 is specified, will throw ``InvalidArgumentException``. + +This field is optional; the default value is ``10000``. + +``kinesis.fetch-attempts`` +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Defines number of failed attempts to be made to fetch data from Kinesis Streams. If first attempt returns +non empty records, then no further attempts are made. + +.. note:: + + It has been found that sometimes ``GetRecordResult`` returns empty records, when shard is not empty. That is why multiple attempts need to be made. + +This field is optional; the default value is ``3``. + +``kinesis.sleep-time`` +^^^^^^^^^^^^^^^^^^^^^^ + +Defines the milliseconds for which thread needs to sleep between get-record-attempts made +to fetch data. + +This field is optional; the defaul value is ``1000`` milliseconds. + +``kinesis.hide-internal-columns`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In addition to the data columns defined in a table description file, the +connector maintains a number of additional columns for each table. If +these columns are hidden, they can still be used in queries but do not +show up in ``DESCRIBE `` or ``SELECT *``. + +This property is optional; the default is ``true``. + +Internal Columns +---------------- + +For each defined table, the connector maintains the following columns: + +======================= ========= =========================================================== +Column name Type Description +======================= ========= =========================================================== +``_shard_id`` VARCHAR ID of the Kinesis stream shard which contains this row. +``_shard_sequence_id`` VARCHAR Sequence id within the Kinesis shard for this row. +``_segment_start`` VARCHAR Lowest sequence id in the segment (inclusive) which contains this row. This sequence id is shrard specific. +``_segment_end`` VARCHAR Highest sequence id in the segment (exclusive) which contains this row. The sequence id is shard specific. If stream is open, then this is not defined. +``_segment_count`` BIGINT Running count of for the current row within the segment. +``_message_valid`` BOOLEAN True if the decoder could decode the message successfully for this row. When false, data columns mapped from the message should be treated as invalid. +``_message`` VARCHAR Message bytes as an UTF-8 encoded string. +``_message_length`` BIGINT Number of bytes in the message. +``_partition_key`` VARCHAR Partition Key bytes as an UTF-8 encoded string. +======================= ========= ============================================================ + +For tables without a table definition file, the ``_message_valid`` column will +always be ``true``. + +Table Definition Files +---------------------- + +Kinesis streams stores data as stream of bytes and leaves it to produceres and +consumers to define how a message should be interpreted. For Presto, this data +must be mapped into columns to allow queries against the data. + +..note:: + + For textual topics that contain JSON data, it is entirely possible to not + use any table definition files, but instead use the Presto + :doc:`/functions/json` to parse the ``_message`` column which contains + the bytes mapped into an UTF-8 string. This is, however, pretty + cumbersome and makes it difficult to write SQL queries. + +A table definition file consists of a JSON definition for a table. The +name of the file can be arbitrary but must end in ``.json``. + +.. code-block:: json + + { + "tableName": ..., + "schemaName": ..., + "streamName": ..., + "message": { + "dataFormat": ..., + "field": [ + ... + ] + } + } + +=============== ========= ============== ======================================================================== +Field Required Type Description +=============== ========= ============== ======================================================================== +``tableName`` required string Presto table name defined by this file. +``schemaName`` optional string Schema which will contain the table. If omitted, the default schema name is used. +``streamName`` required string Kinesis Stream that is mapped. +``message`` optional JSON object Field definitions for data columns mapped to the message itself. +=============== ========= ============== ========================================================================= + +Every message in Kinesis stream can be decoded using the table definition file. +The json object message in table definition file contains two fields: + +=============== ========= ============== ============================= +Field Required Type Description +=============== ========= ============== ============================= +``dataFormat`` required string Selects the decoder for this group of fields. +``fields`` required JSON array A list of field definitions. Each field definition creates a new column in the Presto table. +=============== ========= ============== ============================= + +Each field definition is a JSON object: + +.. code-block:: json + + { + "name": ..., + "type": ..., + "dataFormat": ..., + "mapping": ..., + "formatHint": ..., + "hidden": ..., + "comment": ... + } + +=============== ========= ========= ============================= +Field Required Type Description +=============== ========= ========= ============================= +``name`` required string Name of the column in the Presto table. +``type`` required string Presto type of the column. +``dataFormat`` optional string Selects the column decoder for this field. Default to the default decoder for this row data format and column type. +``mapping`` optional string Mapping information for the column. This is decoder specific, see below. +``formatHint`` optional string Sets a column specific format hint to the column decoder. +``hidden`` optional boolean Hides the column from ``DESCRIBE `` and ``SELECT *``. Defaults to ``false``. +``comment`` optional string Add a column comment which is shown with ``DESCRIBE
``. +=============== ========= ========= ============================= + +There is no limit on field descriptions for message. + +Please refer to `Kafka Connector` _ page for further information about different decoders available. + +.. _Kafka connector: ./kafka.html \ No newline at end of file From f2fa485fe5778469aef26b15edc066e081d2e0c7 Mon Sep 17 00:00:00 2001 From: shubham agarwal Date: Thu, 11 Jun 2015 10:17:06 +0000 Subject: [PATCH 7/7] Update Recordset --- presto-kinesis/pom.xml | 41 +++++++++-- .../presto/kinesis/KinesisClientManager.java | 1 + .../presto/kinesis/KinesisColumnHandle.java | 2 +- .../kinesis/KinesisConnectorConfig.java | 68 +++++++++++++++++++ .../KinesisInternalFieldDescription.java | 4 +- .../presto/kinesis/KinesisMetadata.java | 17 +---- .../presto/kinesis/KinesisRecordSet.java | 28 +++++++- .../kinesis/KinesisRecordSetProvider.java | 7 +- .../KinesisStreamFieldDescription.java | 4 +- .../kinesis/TestKinesisConnectorConfig.java | 16 ++++- .../kinesis/TestMinimalFunctionality.java | 16 ----- 11 files changed, 157 insertions(+), 47 deletions(-) diff --git a/presto-kinesis/pom.xml b/presto-kinesis/pom.xml index 073fff154274..9474583d64b6 100644 --- a/presto-kinesis/pom.xml +++ b/presto-kinesis/pom.xml @@ -5,7 +5,7 @@ com.facebook.presto presto-root - 0.103-SNAPSHOT + 0.107-SNAPSHOT @@ -76,11 +76,11 @@ - + joda-time joda-time - 2.7 - + 2.2 + io.airlift @@ -179,12 +179,24 @@ com.amazonaws aws-java-sdk-kinesis 1.9.16 + + + joda-time + joda-time + + com.amazonaws aws-java-sdk-core 1.9.16 + + + joda-time + joda-time + +