diff --git a/presto-hive-metastore/pom.xml b/presto-hive-metastore/pom.xml index 146ab714dd7cb..8bee3a951d1b6 100644 --- a/presto-hive-metastore/pom.xml +++ b/presto-hive-metastore/pom.xml @@ -71,6 +71,11 @@ slice + + com.fasterxml.jackson.core + jackson-core + + com.fasterxml.jackson.core jackson-databind diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java index 7f389a146752a..01ade1ed4e685 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java @@ -24,15 +24,17 @@ import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.metastore.Table; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; +import static com.facebook.presto.hive.HiveStorageFormat.getHiveStorageFormat; import static com.facebook.presto.hive.metastore.MetastoreUtil.updateStatisticsParameters; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; @@ -44,7 +46,7 @@ public class PartitionMetadata private final List columns; private final Map parameters; - private final Optional storageFormat; + private final StorageFormat storageFormat; private final Optional bucketProperty; private final Map storageParameters; private final Map serdeParameters; @@ -59,7 +61,8 @@ public class PartitionMetadata public PartitionMetadata( @JsonProperty("columns") List columns, @JsonProperty("parameters") Map parameters, - @JsonProperty("storageFormat") Optional storageFormat, + @JsonDeserialize(using = StorageFormatCompatDeserializer.class) + @JsonProperty("storageFormat") StorageFormat storageFormat, @JsonProperty("bucketProperty") Optional bucketProperty, @JsonProperty("storageParameters") Map storageParameters, @JsonProperty("serdeParameters") Map serdeParameters, @@ -71,7 +74,7 @@ public PartitionMetadata( this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null")); - this.storageFormat = requireNonNull(storageFormat, "storageFormat is null"); + this.storageFormat = storageFormat == null ? VIEW_STORAGE_FORMAT : storageFormat; this.bucketProperty = requireNonNull(bucketProperty, "bucketProperty is null"); this.storageParameters = ImmutableMap.copyOf(firstNonNull(storageParameters, ImmutableMap.of())); this.serdeParameters = requireNonNull(serdeParameters, "serdeParameters is null"); @@ -82,6 +85,32 @@ public PartitionMetadata( this.sealedPartition = sealedPartition; } + @Deprecated + public PartitionMetadata( + List columns, + Map parameters, + Optional storageFormat, + Optional bucketProperty, + Map storageParameters, + Map serdeParameters, + Optional externalLocation, + Map columnStatistics, + boolean eligibleToIgnore, + boolean sealedPartition) + { + this( + columns, + parameters, + storageFormat.map(StorageFormat::fromHiveStorageFormat).orElse(VIEW_STORAGE_FORMAT), + bucketProperty, + storageParameters, + serdeParameters, + externalLocation, + columnStatistics, + eligibleToIgnore, + sealedPartition); + } + public PartitionMetadata(Table table, PartitionWithStatistics partitionWithStatistics) { Partition partition = partitionWithStatistics.getPartition(); @@ -90,10 +119,7 @@ public PartitionMetadata(Table table, PartitionWithStatistics partitionWithStati this.columns = partition.getColumns(); this.parameters = updateStatisticsParameters(partition.getParameters(), statistics.getBasicStatistics()); - StorageFormat tableFormat = partition.getStorage().getStorageFormat(); - storageFormat = Arrays.stream(HiveStorageFormat.values()) - .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format))) - .findFirst(); + storageFormat = partition.getStorage().getStorageFormat(); if (table.getTableType().equals(EXTERNAL_TABLE)) { externalLocation = Optional.of(partition.getStorage().getLocation()); @@ -122,8 +148,15 @@ public Map getParameters() return parameters; } - @JsonProperty + @Deprecated + @JsonIgnore public Optional getStorageFormat() + { + return getHiveStorageFormat(storageFormat); + } + + @JsonProperty("storageFormat") + public StorageFormat getPartitionStorageFormat() { return storageFormat; } @@ -188,7 +221,7 @@ public Partition toPartition(String databaseName, String tableName, List values, Storage.builder() .setLocation(externalLocation.orElse(location)) - .setStorageFormat(storageFormat.map(StorageFormat::fromHiveStorageFormat).orElse(VIEW_STORAGE_FORMAT)) + .setStorageFormat(storageFormat) .setBucketProperty(bucketProperty) .setSerdeParameters(serdeParameters) .setParameters(parameters) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/StorageFormatCompatDeserializer.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/StorageFormatCompatDeserializer.java new file mode 100644 index 0000000000000..6ac104686fc41 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/StorageFormatCompatDeserializer.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.hive.metastore.file; + +import com.facebook.presto.hive.HiveStorageFormat; +import com.facebook.presto.hive.metastore.StorageFormat; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +import java.io.IOException; + +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; +import static com.fasterxml.jackson.core.JsonToken.VALUE_STRING; + +public class StorageFormatCompatDeserializer + extends JsonDeserializer +{ + @Override + public StorageFormat deserialize(JsonParser p, DeserializationContext ctxt) + throws IOException, JsonProcessingException + { + // Prior to version 0.271, HiveStorageFormat was used for storage format; + // this deserializer is to ensure backward compatibility + if (p.currentToken() == VALUE_STRING) { + HiveStorageFormat format = p.readValueAs(HiveStorageFormat.class); + return fromHiveStorageFormat(format); + } + return p.readValueAs(StorageFormat.class); + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java index f257efbb1fdc6..f9520ea40d33e 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java @@ -22,15 +22,17 @@ import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.metastore.Table; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; +import static com.facebook.presto.hive.HiveStorageFormat.getHiveStorageFormat; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; import static com.google.common.base.MoreObjects.firstNonNull; @@ -45,7 +47,7 @@ public class TableMetadata private final List partitionColumns; private final Map parameters; - private final Optional storageFormat; + private final StorageFormat storageFormat; private final Optional bucketProperty; private final Map storageParameters; private final Map serdeParameters; @@ -64,7 +66,8 @@ public TableMetadata( @JsonProperty("dataColumns") List dataColumns, @JsonProperty("partitionColumns") List partitionColumns, @JsonProperty("parameters") Map parameters, - @JsonProperty("storageFormat") Optional storageFormat, + @JsonDeserialize(using = StorageFormatCompatDeserializer.class) + @JsonProperty("storageFormat") StorageFormat storageFormat, @JsonProperty("bucketProperty") Optional bucketProperty, @JsonProperty("storageParameters") Map storageParameters, @JsonProperty("serdeParameters") Map serdeParameters, @@ -79,7 +82,7 @@ public TableMetadata( this.partitionColumns = ImmutableList.copyOf(requireNonNull(partitionColumns, "partitionColumns is null")); this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null")); this.storageParameters = ImmutableMap.copyOf(firstNonNull(storageParameters, ImmutableMap.of())); - this.storageFormat = requireNonNull(storageFormat, "storageFormat is null"); + this.storageFormat = storageFormat == null ? VIEW_STORAGE_FORMAT : storageFormat; this.bucketProperty = requireNonNull(bucketProperty, "bucketProperty is null"); this.serdeParameters = requireNonNull(serdeParameters, "serdeParameters is null"); this.externalLocation = requireNonNull(externalLocation, "externalLocation is null"); @@ -96,6 +99,38 @@ public TableMetadata( checkArgument(partitionColumns.isEmpty() || columnStatistics.isEmpty(), "column statistics cannot be set for partitioned table"); } + @Deprecated + public TableMetadata( + String owner, + PrestoTableType tableType, + List dataColumns, + List partitionColumns, + Map parameters, + Optional storageFormat, + Optional bucketProperty, + Map storageParameters, + Map serdeParameters, + Optional externalLocation, + Optional viewOriginalText, + Optional viewExpandedText, + Map columnStatistics) + { + this( + owner, + tableType, + dataColumns, + partitionColumns, + parameters, + storageFormat.map(StorageFormat::fromHiveStorageFormat).orElse(VIEW_STORAGE_FORMAT), + bucketProperty, + storageParameters, + serdeParameters, + externalLocation, + viewOriginalText, + viewExpandedText, + columnStatistics); + } + public TableMetadata(Table table) { this(table, ImmutableMap.of()); @@ -109,10 +144,7 @@ public TableMetadata(Table table, Map columnStatis partitionColumns = table.getPartitionColumns(); parameters = table.getParameters(); - StorageFormat tableFormat = table.getStorage().getStorageFormat(); - storageFormat = Arrays.stream(HiveStorageFormat.values()) - .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format))) - .findFirst(); + storageFormat = table.getStorage().getStorageFormat(); bucketProperty = table.getStorage().getBucketProperty(); storageParameters = table.getStorage().getParameters(); serdeParameters = table.getStorage().getSerdeParameters(); @@ -174,8 +206,15 @@ public Map getParameters() return parameters; } - @JsonProperty + @Deprecated + @JsonIgnore public Optional getStorageFormat() + { + return getHiveStorageFormat(storageFormat); + } + + @JsonProperty("storageFormat") + public StorageFormat getTableStorageFormat() { return storageFormat; } @@ -285,7 +324,7 @@ public Table toTable(String databaseName, String tableName, String location) tableType, Storage.builder() .setLocation(externalLocation.orElse(location)) - .setStorageFormat(storageFormat.map(StorageFormat::fromHiveStorageFormat).orElse(VIEW_STORAGE_FORMAT)) + .setStorageFormat(storageFormat) .setBucketProperty(bucketProperty) .setParameters(storageParameters) .setSerdeParameters(serdeParameters) diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestPartitionMetadata.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestPartitionMetadata.java new file mode 100644 index 0000000000000..c2a29de73aafa --- /dev/null +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestPartitionMetadata.java @@ -0,0 +1,123 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.hive.metastore.file; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.hive.HiveStorageFormat; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.StorageFormat; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Optional; + +import static com.facebook.presto.hive.HiveType.HIVE_STRING; +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testng.Assert.assertEquals; + +public class TestPartitionMetadata +{ + private static final JsonCodec JSON_CODEC = JsonCodec.jsonCodec(PartitionMetadata.class); + private static final String BASE_DIR = "src/test/resources/PR-17368"; + private static final String FILE_NAME_FORMAT = "partition-0.271-%s.json"; + private static final String STORAGE_FORMAT_NOT_EQUALS = "storage format not equals"; + + private static final StorageFormat ORC = fromHiveStorageFormat(HiveStorageFormat.ORC); + private static final StorageFormat PARQUET = fromHiveStorageFormat(HiveStorageFormat.PARQUET); + private static final StorageFormat CUSTOM = StorageFormat.create("serde", "inputFormat", "outputFormat"); + + @Test + public void testAssertPartitionMetadataEquals() + { + assertPartitionMetadataEquals(createPartitionMetadata(null), createPartitionMetadata(null)); + assertPartitionMetadataEquals(createPartitionMetadata(ORC), createPartitionMetadata(ORC)); + assertPartitionMetadataEquals(createPartitionMetadata(PARQUET), createPartitionMetadata(PARQUET)); + assertThatThrownBy(() -> assertPartitionMetadataEquals(createPartitionMetadata(null), createPartitionMetadata(ORC))) + .hasMessageContaining(STORAGE_FORMAT_NOT_EQUALS); + assertThatThrownBy(() -> assertPartitionMetadataEquals(createPartitionMetadata(PARQUET), createPartitionMetadata(ORC))) + .hasMessageContaining(STORAGE_FORMAT_NOT_EQUALS); + } + + @Test + public void testJsonRoundTrip() + { + assertJsonRoundTrip(createPartitionMetadata(null)); + assertJsonRoundTrip(createPartitionMetadata(ORC)); + assertJsonRoundTrip(createPartitionMetadata(PARQUET)); + assertJsonRoundTrip(createPartitionMetadata(CUSTOM)); + } + + @Test + public void testDecodeFromLegacyFile() + throws IOException + { + assertPartitionMetadataEquals(load("null"), createPartitionMetadata(null)); + assertPartitionMetadataEquals(load("orc"), createPartitionMetadata(ORC)); + assertPartitionMetadataEquals(load("parquet"), createPartitionMetadata(PARQUET)); + } + + private static PartitionMetadata load(String tag) + throws IOException + { + return JSON_CODEC.fromBytes(Files.readAllBytes(Paths.get(BASE_DIR, format(FILE_NAME_FORMAT, tag)))); + } + + private static void assertJsonRoundTrip(PartitionMetadata partition) + { + PartitionMetadata decoded = JSON_CODEC.fromJson(JSON_CODEC.toJson(partition)); + assertPartitionMetadataEquals(decoded, partition); + } + + private static void assertPartitionMetadataEquals(PartitionMetadata actual, PartitionMetadata expected) + { + assertEquals(actual.getColumns(), expected.getColumns()); + assertEquals(actual.getParameters(), expected.getParameters()); + assertEquals(actual.getStorageFormat(), expected.getStorageFormat(), STORAGE_FORMAT_NOT_EQUALS); + assertEquals(actual.getBucketProperty(), expected.getBucketProperty()); + assertEquals(actual.getStorageParameters(), expected.getStorageParameters()); + assertEquals(actual.getSerdeParameters(), expected.getSerdeParameters()); + assertEquals(actual.getExternalLocation(), expected.getExternalLocation()); + assertEquals(actual.getColumnStatistics(), expected.getColumnStatistics()); + assertEquals(actual.isEligibleToIgnore(), expected.isEligibleToIgnore()); + assertEquals(actual.isSealedPartition(), expected.isSealedPartition()); + assertEquals(actual.getPartitionStorageFormat(), expected.getPartitionStorageFormat(), STORAGE_FORMAT_NOT_EQUALS); + } + + private static PartitionMetadata createPartitionMetadata(StorageFormat format) + { + return new PartitionMetadata( + ImmutableList.of(column("col1"), column("col2")), + ImmutableMap.of("param1", "value1", "param2", "value2"), + format, + Optional.empty(), + ImmutableMap.of(), + ImmutableMap.of(), + Optional.empty(), + ImmutableMap.of(), + false, + false); + } + + private static Column column(String name) + { + return new Column(name, HIVE_STRING, Optional.of(name), Optional.empty()); + } +} diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestStorageFormatCompatDeserializer.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestStorageFormatCompatDeserializer.java new file mode 100644 index 0000000000000..44aaa47968dce --- /dev/null +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestStorageFormatCompatDeserializer.java @@ -0,0 +1,138 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.hive.metastore.file; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.hive.HiveStorageFormat; +import com.facebook.presto.hive.metastore.StorageFormat; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.testng.annotations.Test; + +import java.util.Objects; + +import static com.facebook.presto.hive.HiveStorageFormat.JSON; +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; + +public class TestStorageFormatCompatDeserializer +{ + @Test + public void testStorageFormatCompatDeserializer() + { + JsonCodec codecV1 = JsonCodec.jsonCodec(ProtocolV1.class); + JsonCodec codecV2 = JsonCodec.jsonCodec(ProtocolV2.class); + + ProtocolV1 v1 = new ProtocolV1(1234, JSON); + ProtocolV2 v2 = new ProtocolV2(1234, fromHiveStorageFormat(JSON)); + + ProtocolV2 v2FromV1 = codecV2.fromJson(codecV1.toJson(v1)); + assertEquals(v2FromV1, v2); + ProtocolV2 v2FromV2 = codecV2.fromJson(codecV2.toJson(v2)); + assertEquals(v2FromV2, v2); + } + + public static class ProtocolV1 + { + private final int id; + private final HiveStorageFormat format; + + @JsonCreator + public ProtocolV1(@JsonProperty("id") int id, @JsonProperty("format") HiveStorageFormat format) + { + this.id = id; + this.format = requireNonNull(format, "format is null"); + } + + @JsonProperty + public int getId() + { + return id; + } + + @JsonProperty + public HiveStorageFormat getFormat() + { + return format; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ProtocolV1 that = (ProtocolV1) o; + return id == that.id && format == that.format; + } + + @Override + public int hashCode() + { + return Objects.hash(id, format); + } + } + + public static class ProtocolV2 + { + private final int id; + private final StorageFormat format; + + @JsonCreator + public ProtocolV2(@JsonProperty("id") int id, + @JsonDeserialize(using = StorageFormatCompatDeserializer.class) + @JsonProperty("format") StorageFormat format) + { + this.id = id; + this.format = requireNonNull(format, "format is null"); + } + + @JsonProperty + public int getId() + { + return id; + } + + @JsonProperty + public StorageFormat getFormat() + { + return format; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ProtocolV2 that = (ProtocolV2) o; + return id == that.id && format.equals(that.format); + } + + @Override + public int hashCode() + { + return Objects.hash(id, format); + } + } +} diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestTableMetadata.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestTableMetadata.java new file mode 100644 index 0000000000000..8ea2d599b9e2a --- /dev/null +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestTableMetadata.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.hive.metastore.file; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.hive.HiveStorageFormat; +import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.StorageFormat; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Optional; + +import static com.facebook.presto.hive.HiveType.HIVE_STRING; +import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testng.Assert.assertEquals; + +public class TestTableMetadata +{ + private static final JsonCodec JSON_CODEC = JsonCodec.jsonCodec(TableMetadata.class); + private static final String BASE_DIR = "src/test/resources/PR-17368"; + private static final String FILE_NAME_FORMAT = "table-0.271-%s.json"; + private static final String STORAGE_FORMAT_NOT_EQUALS = "storage format not equals"; + + private static final StorageFormat ORC = fromHiveStorageFormat(HiveStorageFormat.ORC); + private static final StorageFormat PARQUET = fromHiveStorageFormat(HiveStorageFormat.PARQUET); + private static final StorageFormat CUSTOM = StorageFormat.create("serde", "inputFormat", "outputFormat"); + + @Test + public void testAssertTableMetadataEquals() + { + assertTableMetadataEquals(createTableMetadata(null), createTableMetadata(null)); + assertTableMetadataEquals(createTableMetadata(ORC), createTableMetadata(ORC)); + assertTableMetadataEquals(createTableMetadata(PARQUET), createTableMetadata(PARQUET)); + assertThatThrownBy(() -> assertTableMetadataEquals(createTableMetadata(null), createTableMetadata(ORC))) + .hasMessageContaining(STORAGE_FORMAT_NOT_EQUALS); + assertThatThrownBy(() -> assertTableMetadataEquals(createTableMetadata(PARQUET), createTableMetadata(ORC))) + .hasMessageContaining(STORAGE_FORMAT_NOT_EQUALS); + } + + @Test + public void testJsonRoundTrip() + { + assertJsonRoundTrip(createTableMetadata(null)); + assertJsonRoundTrip(createTableMetadata(ORC)); + assertJsonRoundTrip(createTableMetadata(PARQUET)); + assertJsonRoundTrip(createTableMetadata(CUSTOM)); + } + + @Test + public void testDecodeFromLegacyFile() + throws IOException + { + assertTableMetadataEquals(load("null"), createTableMetadata(null)); + assertTableMetadataEquals(load("orc"), createTableMetadata(ORC)); + assertTableMetadataEquals(load("parquet"), createTableMetadata(PARQUET)); + } + + private static TableMetadata load(String tag) + throws IOException + { + return JSON_CODEC.fromBytes(Files.readAllBytes(Paths.get(BASE_DIR, format(FILE_NAME_FORMAT, tag)))); + } + + private static void assertJsonRoundTrip(TableMetadata table) + { + TableMetadata decoded = JSON_CODEC.fromJson(JSON_CODEC.toJson(table)); + assertTableMetadataEquals(decoded, table); + } + + private static void assertTableMetadataEquals(TableMetadata actual, TableMetadata expected) + { + assertEquals(actual.getOwner(), expected.getOwner()); + assertEquals(actual.getTableType(), expected.getTableType()); + assertEquals(actual.getDataColumns(), expected.getDataColumns()); + assertEquals(actual.getPartitionColumns(), expected.getPartitionColumns()); + assertEquals(actual.getParameters(), expected.getParameters()); + assertEquals(actual.getStorageFormat(), expected.getStorageFormat(), STORAGE_FORMAT_NOT_EQUALS); + assertEquals(actual.getBucketProperty(), expected.getBucketProperty()); + assertEquals(actual.getStorageParameters(), expected.getStorageParameters()); + assertEquals(actual.getSerdeParameters(), expected.getSerdeParameters()); + assertEquals(actual.getExternalLocation(), expected.getExternalLocation()); + assertEquals(actual.getViewOriginalText(), expected.getViewOriginalText()); + assertEquals(actual.getViewExpandedText(), expected.getViewExpandedText()); + assertEquals(actual.getColumnStatistics(), expected.getColumnStatistics()); + assertEquals(actual.getTableStorageFormat(), expected.getTableStorageFormat(), STORAGE_FORMAT_NOT_EQUALS); + } + + private static TableMetadata createTableMetadata(StorageFormat format) + { + return new TableMetadata( + "owner0", + EXTERNAL_TABLE, + ImmutableList.of(column("col1"), column("col2")), + ImmutableList.of(column("part1")), + ImmutableMap.of("param1", "value1", "param2", "value2"), + format, + Optional.empty(), + ImmutableMap.of(), + ImmutableMap.of(), + Optional.of("/tmp/location"), + Optional.empty(), + Optional.empty(), + ImmutableMap.of()); + } + + private static Column column(String name) + { + return new Column(name, HIVE_STRING, Optional.of(name), Optional.empty()); + } +} diff --git a/presto-hive-metastore/src/test/resources/PR-17368/README.md b/presto-hive-metastore/src/test/resources/PR-17368/README.md new file mode 100644 index 0000000000000..d08e0f13c2787 --- /dev/null +++ b/presto-hive-metastore/src/test/resources/PR-17368/README.md @@ -0,0 +1,6 @@ +README + +This folder contains a set of json files, which were generated based on +the classes TableMetadata and PartitionMetadata at the version 0.271. +The files are used for compatibility testing in +[PR-17368](https://github.com/prestodb/presto/pull/17368); diff --git a/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-null.json b/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-null.json new file mode 100644 index 0000000000000..c3d349926d12b --- /dev/null +++ b/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-null.json @@ -0,0 +1,20 @@ +{ + "columns" : [ { + "name" : "col1", + "type" : "string", + "comment" : "col1" + }, { + "name" : "col2", + "type" : "string", + "comment" : "col2" + } ], + "parameters" : { + "param1" : "value1", + "param2" : "value2" + }, + "storageParameters" : { }, + "serdeParameters" : { }, + "columnStatistics" : { }, + "eligibleToIgnore" : false, + "sealedPartition" : false +} \ No newline at end of file diff --git a/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-orc.json b/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-orc.json new file mode 100644 index 0000000000000..ce110dc15fc34 --- /dev/null +++ b/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-orc.json @@ -0,0 +1,21 @@ +{ + "columns" : [ { + "name" : "col1", + "type" : "string", + "comment" : "col1" + }, { + "name" : "col2", + "type" : "string", + "comment" : "col2" + } ], + "parameters" : { + "param1" : "value1", + "param2" : "value2" + }, + "storageFormat" : "ORC", + "storageParameters" : { }, + "serdeParameters" : { }, + "columnStatistics" : { }, + "eligibleToIgnore" : false, + "sealedPartition" : false +} \ No newline at end of file diff --git a/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-parquet.json b/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-parquet.json new file mode 100644 index 0000000000000..22cca5347770b --- /dev/null +++ b/presto-hive-metastore/src/test/resources/PR-17368/partition-0.271-parquet.json @@ -0,0 +1,21 @@ +{ + "columns" : [ { + "name" : "col1", + "type" : "string", + "comment" : "col1" + }, { + "name" : "col2", + "type" : "string", + "comment" : "col2" + } ], + "parameters" : { + "param1" : "value1", + "param2" : "value2" + }, + "storageFormat" : "PARQUET", + "storageParameters" : { }, + "serdeParameters" : { }, + "columnStatistics" : { }, + "eligibleToIgnore" : false, + "sealedPartition" : false +} \ No newline at end of file diff --git a/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-null.json b/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-null.json new file mode 100644 index 0000000000000..97849da11ac94 --- /dev/null +++ b/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-null.json @@ -0,0 +1,26 @@ +{ + "owner" : "owner0", + "tableType" : "EXTERNAL_TABLE", + "dataColumns" : [ { + "name" : "col1", + "type" : "string", + "comment" : "col1" + }, { + "name" : "col2", + "type" : "string", + "comment" : "col2" + } ], + "partitionColumns" : [ { + "name" : "part1", + "type" : "string", + "comment" : "part1" + } ], + "parameters" : { + "param1" : "value1", + "param2" : "value2" + }, + "storageParameters" : { }, + "serdeParameters" : { }, + "externalLocation" : "/tmp/location", + "columnStatistics" : { } +} \ No newline at end of file diff --git a/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-orc.json b/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-orc.json new file mode 100644 index 0000000000000..7755f82249c66 --- /dev/null +++ b/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-orc.json @@ -0,0 +1,27 @@ +{ + "owner" : "owner0", + "tableType" : "EXTERNAL_TABLE", + "dataColumns" : [ { + "name" : "col1", + "type" : "string", + "comment" : "col1" + }, { + "name" : "col2", + "type" : "string", + "comment" : "col2" + } ], + "partitionColumns" : [ { + "name" : "part1", + "type" : "string", + "comment" : "part1" + } ], + "parameters" : { + "param1" : "value1", + "param2" : "value2" + }, + "storageFormat" : "ORC", + "storageParameters" : { }, + "serdeParameters" : { }, + "externalLocation" : "/tmp/location", + "columnStatistics" : { } +} \ No newline at end of file diff --git a/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-parquet.json b/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-parquet.json new file mode 100644 index 0000000000000..ea36ae6b1a0b3 --- /dev/null +++ b/presto-hive-metastore/src/test/resources/PR-17368/table-0.271-parquet.json @@ -0,0 +1,27 @@ +{ + "owner" : "owner0", + "tableType" : "EXTERNAL_TABLE", + "dataColumns" : [ { + "name" : "col1", + "type" : "string", + "comment" : "col1" + }, { + "name" : "col2", + "type" : "string", + "comment" : "col2" + } ], + "partitionColumns" : [ { + "name" : "part1", + "type" : "string", + "comment" : "part1" + } ], + "parameters" : { + "param1" : "value1", + "param2" : "value2" + }, + "storageFormat" : "PARQUET", + "storageParameters" : { }, + "serdeParameters" : { }, + "externalLocation" : "/tmp/location", + "columnStatistics" : { } +} \ No newline at end of file