Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added ability to set table type as configuration options and as table properties on table in Iceberg #5656

Closed
wants to merge 3 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -90,6 +90,7 @@ public class HiveTableOperations
private final Optional<String> owner;
private final Optional<String> location;
private final FileIO fileIo;
private final TableType tableType;

private TableMetadata currentMetadata;
private String currentMetadataLocation;
@@ -98,21 +99,22 @@ public class HiveTableOperations

public HiveTableOperations(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, HdfsContext hdfsContext, HiveIdentity identity, String database, String table)
{
this(new HdfsFileIo(hdfsEnvironment, hdfsContext), metastore, identity, database, table, Optional.empty(), Optional.empty());
this(new HdfsFileIo(hdfsEnvironment, hdfsContext), metastore, identity, database, table, Optional.empty(), Optional.empty(), TableType.EXTERNAL_TABLE);
}

public HiveTableOperations(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, HdfsContext hdfsContext, HiveIdentity identity, String database, String table, String owner, String location)
public HiveTableOperations(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, HdfsContext hdfsContext, HiveIdentity identity, String database, String table, String owner, String location, TableType tableType)
{
this(new HdfsFileIo(hdfsEnvironment, hdfsContext),
metastore,
identity,
database,
table,
Optional.of(requireNonNull(owner, "owner is null")),
Optional.of(requireNonNull(location, "location is null")));
Optional.of(requireNonNull(location, "location is null")),
tableType);
}

private HiveTableOperations(FileIO fileIo, HiveMetastore metastore, HiveIdentity identity, String database, String table, Optional<String> owner, Optional<String> location)
private HiveTableOperations(FileIO fileIo, HiveMetastore metastore, HiveIdentity identity, String database, String table, Optional<String> owner, Optional<String> location, TableType tableType)
{
this.fileIo = requireNonNull(fileIo, "fileIo is null");
this.metastore = requireNonNull(metastore, "metastore is null");
@@ -121,6 +123,7 @@ private HiveTableOperations(FileIO fileIo, HiveMetastore metastore, HiveIdentity
this.tableName = requireNonNull(table, "table is null");
this.owner = requireNonNull(owner, "owner is null");
this.location = requireNonNull(location, "location is null");
this.tableType = requireNonNull(tableType, "tableType is null");
}

@Override
@@ -182,11 +185,11 @@ public void commit(@Nullable TableMetadata base, TableMetadata metadata)
.setDatabaseName(database)
.setTableName(tableName)
.setOwner(owner.orElseThrow(() -> new IllegalStateException("Owner not set")))
.setTableType(TableType.EXTERNAL_TABLE.name())
.setTableType(tableType.name())
.setDataColumns(toHiveColumns(metadata.schema().columns()))
.withStorage(storage -> storage.setLocation(metadata.location()))
.withStorage(storage -> storage.setStorageFormat(STORAGE_FORMAT))
.setParameter("EXTERNAL", "TRUE")
.setParameter("EXTERNAL", TableType.EXTERNAL_TABLE == tableType ? "TRUE" : "FALSE")
.setParameter(TABLE_TYPE_PROP, ICEBERG_TABLE_TYPE_VALUE)
.setParameter(METADATA_LOCATION, newMetadataLocation);
String tableComment = metadata.properties().get(TABLE_COMMENT);
Original file line number Diff line number Diff line change
@@ -14,7 +14,9 @@
package io.prestosql.plugin.iceberg;

import io.airlift.configuration.Config;
import io.airlift.configuration.ConfigDescription;
import io.prestosql.plugin.hive.HiveCompressionCodec;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.iceberg.FileFormat;

import javax.validation.constraints.NotNull;
@@ -26,6 +28,7 @@ public class IcebergConfig
{
private IcebergFileFormat fileFormat = ORC;
private HiveCompressionCodec compressionCodec = GZIP;
private TableType tableType = TableType.EXTERNAL_TABLE;

@NotNull
public FileFormat getFileFormat()
@@ -52,4 +55,18 @@ public IcebergConfig setCompressionCodec(HiveCompressionCodec compressionCodec)
this.compressionCodec = compressionCodec;
return this;
}

@NotNull
public TableType getTableType()
{
return tableType;
}

@Config("iceberg.table-type")
@ConfigDescription("Type of the table: `MANAGED_TABLE` or `EXTERNAL_TABLE`")
public IcebergConfig setTableType(TableType tableType)
{
this.tableType = tableType;
return this;
}
}
Original file line number Diff line number Diff line change
@@ -118,6 +118,7 @@
import static io.prestosql.plugin.iceberg.IcebergTableProperties.getFileFormat;
import static io.prestosql.plugin.iceberg.IcebergTableProperties.getPartitioning;
import static io.prestosql.plugin.iceberg.IcebergTableProperties.getTableLocation;
import static io.prestosql.plugin.iceberg.IcebergTableProperties.getTableType;
import static io.prestosql.plugin.iceberg.IcebergUtil.getColumns;
import static io.prestosql.plugin.iceberg.IcebergUtil.getDataPath;
import static io.prestosql.plugin.iceberg.IcebergUtil.getFileFormat;
@@ -428,7 +429,8 @@ public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, Con
targetPath = getTableDefaultLocation(database, hdfsContext, hdfsEnvironment, schemaName, tableName).toString();
}

TableOperations operations = new HiveTableOperations(metastore, hdfsEnvironment, hdfsContext, identity, schemaName, tableName, session.getUser(), targetPath);
org.apache.hadoop.hive.metastore.TableType tableType = getTableType(tableMetadata.getProperties());
TableOperations operations = new HiveTableOperations(metastore, hdfsEnvironment, hdfsContext, identity, schemaName, tableName, session.getUser(), targetPath, tableType);
if (operations.current() != null) {
throw new TableAlreadyExistsException(schemaTableName);
}
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.session.PropertyMetadata;
import io.prestosql.spi.type.ArrayType;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.iceberg.FileFormat;

import javax.inject.Inject;
@@ -35,6 +36,7 @@ public class IcebergTableProperties
public static final String FILE_FORMAT_PROPERTY = "format";
public static final String PARTITIONING_PROPERTY = "partitioning";
public static final String LOCATION_PROPERTY = "location";
public static final String TYPE_PROPERTY = "type";

private final List<PropertyMetadata<?>> tableProperties;

@@ -64,6 +66,12 @@ public IcebergTableProperties(IcebergConfig icebergConfig)
"File system location URI for the table",
null,
false))
.add(enumProperty(
TYPE_PROPERTY,
"Table type for the table",
TableType.class,
icebergConfig.getTableType(),
false))
.build();
}

@@ -88,4 +96,9 @@ public static String getTableLocation(Map<String, Object> tableProperties)
{
return (String) tableProperties.get(LOCATION_PROPERTY);
}

public static TableType getTableType(Map<String, Object> tableProperties)
{
return (TableType) tableProperties.get(TYPE_PROPERTY);
}
}
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@

import com.google.common.collect.ImmutableMap;
import io.prestosql.plugin.hive.HiveCompressionCodec;
import org.apache.hadoop.hive.metastore.TableType;
import org.testng.annotations.Test;

import java.util.Map;
@@ -33,7 +34,8 @@ public void testDefaults()
{
assertRecordedDefaults(recordDefaults(IcebergConfig.class)
.setFileFormat(ORC)
.setCompressionCodec(GZIP));
.setCompressionCodec(GZIP)
.setTableType(TableType.EXTERNAL_TABLE));
}

@Test
@@ -42,11 +44,13 @@ public void testExplicitPropertyMappings()
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
.put("iceberg.file-format", "Parquet")
.put("iceberg.compression-codec", "NONE")
.put("iceberg.table-type", "MANAGED_TABLE")
.build();

IcebergConfig expected = new IcebergConfig()
.setFileFormat(PARQUET)
.setCompressionCodec(HiveCompressionCodec.NONE);
.setCompressionCodec(HiveCompressionCodec.NONE)
.setTableType(TableType.MANAGED_TABLE);

assertFullMapping(properties, expected);
}