From 92d46bdc1bda6b33d37d6d20180aea69eb72a061 Mon Sep 17 00:00:00 2001 From: Todd Gao Date: Mon, 7 Mar 2022 10:40:59 +0800 Subject: [PATCH] Generalize storage format in file metastore --- presto-hive-metastore/pom.xml | 5 + .../metastore/file/PartitionMetadata.java | 53 +++++-- .../file/StorageFormatCompatDeserializer.java | 44 ++++++ .../hive/metastore/file/TableMetadata.java | 59 ++++++-- .../metastore/file/TestPartitionMetadata.java | 31 ++-- .../TestStorageFormatCompatDeserializer.java | 138 ++++++++++++++++++ .../metastore/file/TestTableMetadata.java | 31 ++-- 7 files changed, 301 insertions(+), 60 deletions(-) create mode 100644 presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/StorageFormatCompatDeserializer.java create mode 100644 presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestStorageFormatCompatDeserializer.java diff --git a/presto-hive-metastore/pom.xml b/presto-hive-metastore/pom.xml index 146ab714dd7cb..8bee3a951d1b6 100644 --- a/presto-hive-metastore/pom.xml +++ b/presto-hive-metastore/pom.xml @@ -71,6 +71,11 @@ slice + + com.fasterxml.jackson.core + jackson-core + + com.fasterxml.jackson.core jackson-databind diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java index 7f389a146752a..01ade1ed4e685 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/PartitionMetadata.java @@ -24,15 +24,17 @@ import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.metastore.Table; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; +import static com.facebook.presto.hive.HiveStorageFormat.getHiveStorageFormat; import static com.facebook.presto.hive.metastore.MetastoreUtil.updateStatisticsParameters; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; @@ -44,7 +46,7 @@ public class PartitionMetadata private final List columns; private final Map parameters; - private final Optional storageFormat; + private final StorageFormat storageFormat; private final Optional bucketProperty; private final Map storageParameters; private final Map serdeParameters; @@ -59,7 +61,8 @@ public class PartitionMetadata public PartitionMetadata( @JsonProperty("columns") List columns, @JsonProperty("parameters") Map parameters, - @JsonProperty("storageFormat") Optional storageFormat, + @JsonDeserialize(using = StorageFormatCompatDeserializer.class) + @JsonProperty("storageFormat") StorageFormat storageFormat, @JsonProperty("bucketProperty") Optional bucketProperty, @JsonProperty("storageParameters") Map storageParameters, @JsonProperty("serdeParameters") Map serdeParameters, @@ -71,7 +74,7 @@ public PartitionMetadata( this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null")); - this.storageFormat = requireNonNull(storageFormat, "storageFormat is null"); + this.storageFormat = storageFormat == null ? VIEW_STORAGE_FORMAT : storageFormat; this.bucketProperty = requireNonNull(bucketProperty, "bucketProperty is null"); this.storageParameters = ImmutableMap.copyOf(firstNonNull(storageParameters, ImmutableMap.of())); this.serdeParameters = requireNonNull(serdeParameters, "serdeParameters is null"); @@ -82,6 +85,32 @@ public PartitionMetadata( this.sealedPartition = sealedPartition; } + @Deprecated + public PartitionMetadata( + List columns, + Map parameters, + Optional storageFormat, + Optional bucketProperty, + Map storageParameters, + Map serdeParameters, + Optional externalLocation, + Map columnStatistics, + boolean eligibleToIgnore, + boolean sealedPartition) + { + this( + columns, + parameters, + storageFormat.map(StorageFormat::fromHiveStorageFormat).orElse(VIEW_STORAGE_FORMAT), + bucketProperty, + storageParameters, + serdeParameters, + externalLocation, + columnStatistics, + eligibleToIgnore, + sealedPartition); + } + public PartitionMetadata(Table table, PartitionWithStatistics partitionWithStatistics) { Partition partition = partitionWithStatistics.getPartition(); @@ -90,10 +119,7 @@ public PartitionMetadata(Table table, PartitionWithStatistics partitionWithStati this.columns = partition.getColumns(); this.parameters = updateStatisticsParameters(partition.getParameters(), statistics.getBasicStatistics()); - StorageFormat tableFormat = partition.getStorage().getStorageFormat(); - storageFormat = Arrays.stream(HiveStorageFormat.values()) - .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format))) - .findFirst(); + storageFormat = partition.getStorage().getStorageFormat(); if (table.getTableType().equals(EXTERNAL_TABLE)) { externalLocation = Optional.of(partition.getStorage().getLocation()); @@ -122,8 +148,15 @@ public Map getParameters() return parameters; } - @JsonProperty + @Deprecated + @JsonIgnore public Optional getStorageFormat() + { + return getHiveStorageFormat(storageFormat); + } + + @JsonProperty("storageFormat") + public StorageFormat getPartitionStorageFormat() { return storageFormat; } @@ -188,7 +221,7 @@ public Partition toPartition(String databaseName, String tableName, List values, Storage.builder() .setLocation(externalLocation.orElse(location)) - .setStorageFormat(storageFormat.map(StorageFormat::fromHiveStorageFormat).orElse(VIEW_STORAGE_FORMAT)) + .setStorageFormat(storageFormat) .setBucketProperty(bucketProperty) .setSerdeParameters(serdeParameters) .setParameters(parameters) diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/StorageFormatCompatDeserializer.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/StorageFormatCompatDeserializer.java new file mode 100644 index 0000000000000..6ac104686fc41 --- /dev/null +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/StorageFormatCompatDeserializer.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.hive.metastore.file; + +import com.facebook.presto.hive.HiveStorageFormat; +import com.facebook.presto.hive.metastore.StorageFormat; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; + +import java.io.IOException; + +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; +import static com.fasterxml.jackson.core.JsonToken.VALUE_STRING; + +public class StorageFormatCompatDeserializer + extends JsonDeserializer +{ + @Override + public StorageFormat deserialize(JsonParser p, DeserializationContext ctxt) + throws IOException, JsonProcessingException + { + // Prior to version 0.271, HiveStorageFormat was used for storage format; + // this deserializer is to ensure backward compatibility + if (p.currentToken() == VALUE_STRING) { + HiveStorageFormat format = p.readValueAs(HiveStorageFormat.class); + return fromHiveStorageFormat(format); + } + return p.readValueAs(StorageFormat.class); + } +} diff --git a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java index f257efbb1fdc6..f9520ea40d33e 100644 --- a/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java +++ b/presto-hive-metastore/src/main/java/com/facebook/presto/hive/metastore/file/TableMetadata.java @@ -22,15 +22,17 @@ import com.facebook.presto.hive.metastore.StorageFormat; import com.facebook.presto.hive.metastore.Table; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; +import static com.facebook.presto.hive.HiveStorageFormat.getHiveStorageFormat; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; import static com.facebook.presto.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; import static com.google.common.base.MoreObjects.firstNonNull; @@ -45,7 +47,7 @@ public class TableMetadata private final List partitionColumns; private final Map parameters; - private final Optional storageFormat; + private final StorageFormat storageFormat; private final Optional bucketProperty; private final Map storageParameters; private final Map serdeParameters; @@ -64,7 +66,8 @@ public TableMetadata( @JsonProperty("dataColumns") List dataColumns, @JsonProperty("partitionColumns") List partitionColumns, @JsonProperty("parameters") Map parameters, - @JsonProperty("storageFormat") Optional storageFormat, + @JsonDeserialize(using = StorageFormatCompatDeserializer.class) + @JsonProperty("storageFormat") StorageFormat storageFormat, @JsonProperty("bucketProperty") Optional bucketProperty, @JsonProperty("storageParameters") Map storageParameters, @JsonProperty("serdeParameters") Map serdeParameters, @@ -79,7 +82,7 @@ public TableMetadata( this.partitionColumns = ImmutableList.copyOf(requireNonNull(partitionColumns, "partitionColumns is null")); this.parameters = ImmutableMap.copyOf(requireNonNull(parameters, "parameters is null")); this.storageParameters = ImmutableMap.copyOf(firstNonNull(storageParameters, ImmutableMap.of())); - this.storageFormat = requireNonNull(storageFormat, "storageFormat is null"); + this.storageFormat = storageFormat == null ? VIEW_STORAGE_FORMAT : storageFormat; this.bucketProperty = requireNonNull(bucketProperty, "bucketProperty is null"); this.serdeParameters = requireNonNull(serdeParameters, "serdeParameters is null"); this.externalLocation = requireNonNull(externalLocation, "externalLocation is null"); @@ -96,6 +99,38 @@ public TableMetadata( checkArgument(partitionColumns.isEmpty() || columnStatistics.isEmpty(), "column statistics cannot be set for partitioned table"); } + @Deprecated + public TableMetadata( + String owner, + PrestoTableType tableType, + List dataColumns, + List partitionColumns, + Map parameters, + Optional storageFormat, + Optional bucketProperty, + Map storageParameters, + Map serdeParameters, + Optional externalLocation, + Optional viewOriginalText, + Optional viewExpandedText, + Map columnStatistics) + { + this( + owner, + tableType, + dataColumns, + partitionColumns, + parameters, + storageFormat.map(StorageFormat::fromHiveStorageFormat).orElse(VIEW_STORAGE_FORMAT), + bucketProperty, + storageParameters, + serdeParameters, + externalLocation, + viewOriginalText, + viewExpandedText, + columnStatistics); + } + public TableMetadata(Table table) { this(table, ImmutableMap.of()); @@ -109,10 +144,7 @@ public TableMetadata(Table table, Map columnStatis partitionColumns = table.getPartitionColumns(); parameters = table.getParameters(); - StorageFormat tableFormat = table.getStorage().getStorageFormat(); - storageFormat = Arrays.stream(HiveStorageFormat.values()) - .filter(format -> tableFormat.equals(StorageFormat.fromHiveStorageFormat(format))) - .findFirst(); + storageFormat = table.getStorage().getStorageFormat(); bucketProperty = table.getStorage().getBucketProperty(); storageParameters = table.getStorage().getParameters(); serdeParameters = table.getStorage().getSerdeParameters(); @@ -174,8 +206,15 @@ public Map getParameters() return parameters; } - @JsonProperty + @Deprecated + @JsonIgnore public Optional getStorageFormat() + { + return getHiveStorageFormat(storageFormat); + } + + @JsonProperty("storageFormat") + public StorageFormat getTableStorageFormat() { return storageFormat; } @@ -285,7 +324,7 @@ public Table toTable(String databaseName, String tableName, String location) tableType, Storage.builder() .setLocation(externalLocation.orElse(location)) - .setStorageFormat(storageFormat.map(StorageFormat::fromHiveStorageFormat).orElse(VIEW_STORAGE_FORMAT)) + .setStorageFormat(storageFormat) .setBucketProperty(bucketProperty) .setParameters(storageParameters) .setSerdeParameters(serdeParameters) diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestPartitionMetadata.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestPartitionMetadata.java index 8100738b31c09..c2a29de73aafa 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestPartitionMetadata.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestPartitionMetadata.java @@ -17,6 +17,7 @@ import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.hive.HiveStorageFormat; import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.StorageFormat; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -26,9 +27,8 @@ import java.nio.file.Paths; import java.util.Optional; -import static com.facebook.presto.hive.HiveStorageFormat.ORC; -import static com.facebook.presto.hive.HiveStorageFormat.PARQUET; import static com.facebook.presto.hive.HiveType.HIVE_STRING; +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertEquals; @@ -40,6 +40,10 @@ public class TestPartitionMetadata private static final String FILE_NAME_FORMAT = "partition-0.271-%s.json"; private static final String STORAGE_FORMAT_NOT_EQUALS = "storage format not equals"; + private static final StorageFormat ORC = fromHiveStorageFormat(HiveStorageFormat.ORC); + private static final StorageFormat PARQUET = fromHiveStorageFormat(HiveStorageFormat.PARQUET); + private static final StorageFormat CUSTOM = StorageFormat.create("serde", "inputFormat", "outputFormat"); + @Test public void testAssertPartitionMetadataEquals() { @@ -58,10 +62,11 @@ public void testJsonRoundTrip() assertJsonRoundTrip(createPartitionMetadata(null)); assertJsonRoundTrip(createPartitionMetadata(ORC)); assertJsonRoundTrip(createPartitionMetadata(PARQUET)); + assertJsonRoundTrip(createPartitionMetadata(CUSTOM)); } @Test - public void testDecodeFromFile() + public void testDecodeFromLegacyFile() throws IOException { assertPartitionMetadataEquals(load("null"), createPartitionMetadata(null)); @@ -69,27 +74,12 @@ public void testDecodeFromFile() assertPartitionMetadataEquals(load("parquet"), createPartitionMetadata(PARQUET)); } - @Test(enabled = false) - public void testEncodeToFile() - throws IOException - { - dump(createPartitionMetadata(null), "null"); - dump(createPartitionMetadata(ORC), "orc"); - dump(createPartitionMetadata(PARQUET), "parquet"); - } - private static PartitionMetadata load(String tag) throws IOException { return JSON_CODEC.fromBytes(Files.readAllBytes(Paths.get(BASE_DIR, format(FILE_NAME_FORMAT, tag)))); } - private static void dump(PartitionMetadata partition, String tag) - throws IOException - { - Files.write(Paths.get(BASE_DIR, format(FILE_NAME_FORMAT, tag)), JSON_CODEC.toBytes(partition)); - } - private static void assertJsonRoundTrip(PartitionMetadata partition) { PartitionMetadata decoded = JSON_CODEC.fromJson(JSON_CODEC.toJson(partition)); @@ -108,14 +98,15 @@ private static void assertPartitionMetadataEquals(PartitionMetadata actual, Part assertEquals(actual.getColumnStatistics(), expected.getColumnStatistics()); assertEquals(actual.isEligibleToIgnore(), expected.isEligibleToIgnore()); assertEquals(actual.isSealedPartition(), expected.isSealedPartition()); + assertEquals(actual.getPartitionStorageFormat(), expected.getPartitionStorageFormat(), STORAGE_FORMAT_NOT_EQUALS); } - private static PartitionMetadata createPartitionMetadata(HiveStorageFormat format) + private static PartitionMetadata createPartitionMetadata(StorageFormat format) { return new PartitionMetadata( ImmutableList.of(column("col1"), column("col2")), ImmutableMap.of("param1", "value1", "param2", "value2"), - Optional.ofNullable(format), + format, Optional.empty(), ImmutableMap.of(), ImmutableMap.of(), diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestStorageFormatCompatDeserializer.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestStorageFormatCompatDeserializer.java new file mode 100644 index 0000000000000..44aaa47968dce --- /dev/null +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestStorageFormatCompatDeserializer.java @@ -0,0 +1,138 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.hive.metastore.file; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.presto.hive.HiveStorageFormat; +import com.facebook.presto.hive.metastore.StorageFormat; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.testng.annotations.Test; + +import java.util.Objects; + +import static com.facebook.presto.hive.HiveStorageFormat.JSON; +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; +import static java.util.Objects.requireNonNull; +import static org.testng.Assert.assertEquals; + +public class TestStorageFormatCompatDeserializer +{ + @Test + public void testStorageFormatCompatDeserializer() + { + JsonCodec codecV1 = JsonCodec.jsonCodec(ProtocolV1.class); + JsonCodec codecV2 = JsonCodec.jsonCodec(ProtocolV2.class); + + ProtocolV1 v1 = new ProtocolV1(1234, JSON); + ProtocolV2 v2 = new ProtocolV2(1234, fromHiveStorageFormat(JSON)); + + ProtocolV2 v2FromV1 = codecV2.fromJson(codecV1.toJson(v1)); + assertEquals(v2FromV1, v2); + ProtocolV2 v2FromV2 = codecV2.fromJson(codecV2.toJson(v2)); + assertEquals(v2FromV2, v2); + } + + public static class ProtocolV1 + { + private final int id; + private final HiveStorageFormat format; + + @JsonCreator + public ProtocolV1(@JsonProperty("id") int id, @JsonProperty("format") HiveStorageFormat format) + { + this.id = id; + this.format = requireNonNull(format, "format is null"); + } + + @JsonProperty + public int getId() + { + return id; + } + + @JsonProperty + public HiveStorageFormat getFormat() + { + return format; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ProtocolV1 that = (ProtocolV1) o; + return id == that.id && format == that.format; + } + + @Override + public int hashCode() + { + return Objects.hash(id, format); + } + } + + public static class ProtocolV2 + { + private final int id; + private final StorageFormat format; + + @JsonCreator + public ProtocolV2(@JsonProperty("id") int id, + @JsonDeserialize(using = StorageFormatCompatDeserializer.class) + @JsonProperty("format") StorageFormat format) + { + this.id = id; + this.format = requireNonNull(format, "format is null"); + } + + @JsonProperty + public int getId() + { + return id; + } + + @JsonProperty + public StorageFormat getFormat() + { + return format; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ProtocolV2 that = (ProtocolV2) o; + return id == that.id && format.equals(that.format); + } + + @Override + public int hashCode() + { + return Objects.hash(id, format); + } + } +} diff --git a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestTableMetadata.java b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestTableMetadata.java index db557cbfbf32b..8ea2d599b9e2a 100644 --- a/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestTableMetadata.java +++ b/presto-hive-metastore/src/test/java/com/facebook/presto/hive/metastore/file/TestTableMetadata.java @@ -17,6 +17,7 @@ import com.facebook.airlift.json.JsonCodec; import com.facebook.presto.hive.HiveStorageFormat; import com.facebook.presto.hive.metastore.Column; +import com.facebook.presto.hive.metastore.StorageFormat; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; @@ -26,10 +27,9 @@ import java.nio.file.Paths; import java.util.Optional; -import static com.facebook.presto.hive.HiveStorageFormat.ORC; -import static com.facebook.presto.hive.HiveStorageFormat.PARQUET; import static com.facebook.presto.hive.HiveType.HIVE_STRING; import static com.facebook.presto.hive.metastore.PrestoTableType.EXTERNAL_TABLE; +import static com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.testng.Assert.assertEquals; @@ -41,6 +41,10 @@ public class TestTableMetadata private static final String FILE_NAME_FORMAT = "table-0.271-%s.json"; private static final String STORAGE_FORMAT_NOT_EQUALS = "storage format not equals"; + private static final StorageFormat ORC = fromHiveStorageFormat(HiveStorageFormat.ORC); + private static final StorageFormat PARQUET = fromHiveStorageFormat(HiveStorageFormat.PARQUET); + private static final StorageFormat CUSTOM = StorageFormat.create("serde", "inputFormat", "outputFormat"); + @Test public void testAssertTableMetadataEquals() { @@ -59,10 +63,11 @@ public void testJsonRoundTrip() assertJsonRoundTrip(createTableMetadata(null)); assertJsonRoundTrip(createTableMetadata(ORC)); assertJsonRoundTrip(createTableMetadata(PARQUET)); + assertJsonRoundTrip(createTableMetadata(CUSTOM)); } @Test - public void testDecodeFromFile() + public void testDecodeFromLegacyFile() throws IOException { assertTableMetadataEquals(load("null"), createTableMetadata(null)); @@ -70,27 +75,12 @@ public void testDecodeFromFile() assertTableMetadataEquals(load("parquet"), createTableMetadata(PARQUET)); } - @Test(enabled = false) - public void testEncodeToFile() - throws IOException - { - dump(createTableMetadata(null), "null"); - dump(createTableMetadata(ORC), "orc"); - dump(createTableMetadata(PARQUET), "parquet"); - } - private static TableMetadata load(String tag) throws IOException { return JSON_CODEC.fromBytes(Files.readAllBytes(Paths.get(BASE_DIR, format(FILE_NAME_FORMAT, tag)))); } - private static void dump(TableMetadata table, String tag) - throws IOException - { - Files.write(Paths.get(BASE_DIR, format(FILE_NAME_FORMAT, tag)), JSON_CODEC.toBytes(table)); - } - private static void assertJsonRoundTrip(TableMetadata table) { TableMetadata decoded = JSON_CODEC.fromJson(JSON_CODEC.toJson(table)); @@ -112,9 +102,10 @@ private static void assertTableMetadataEquals(TableMetadata actual, TableMetadat assertEquals(actual.getViewOriginalText(), expected.getViewOriginalText()); assertEquals(actual.getViewExpandedText(), expected.getViewExpandedText()); assertEquals(actual.getColumnStatistics(), expected.getColumnStatistics()); + assertEquals(actual.getTableStorageFormat(), expected.getTableStorageFormat(), STORAGE_FORMAT_NOT_EQUALS); } - private static TableMetadata createTableMetadata(HiveStorageFormat format) + private static TableMetadata createTableMetadata(StorageFormat format) { return new TableMetadata( "owner0", @@ -122,7 +113,7 @@ private static TableMetadata createTableMetadata(HiveStorageFormat format) ImmutableList.of(column("col1"), column("col2")), ImmutableList.of(column("part1")), ImmutableMap.of("param1", "value1", "param2", "value2"), - Optional.ofNullable(format), + format, Optional.empty(), ImmutableMap.of(), ImmutableMap.of(),