diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java index 9f7a88c5ff64..00b61d6bfb7c 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveMetadata.java @@ -177,7 +177,6 @@ import static io.prestosql.plugin.hive.HiveTableProperties.getOrcBloomFilterFpp; import static io.prestosql.plugin.hive.HiveTableProperties.getPartitionedBy; import static io.prestosql.plugin.hive.HiveTableProperties.getSingleCharacterProperty; -import static io.prestosql.plugin.hive.HiveType.HIVE_STRING; import static io.prestosql.plugin.hive.HiveType.toHiveType; import static io.prestosql.plugin.hive.HiveWriterFactory.computeBucketedFileName; import static io.prestosql.plugin.hive.PartitionUpdate.UpdateMode.APPEND; @@ -188,7 +187,6 @@ import static io.prestosql.plugin.hive.metastore.MetastoreUtil.getProtectMode; import static io.prestosql.plugin.hive.metastore.MetastoreUtil.verifyOnline; import static io.prestosql.plugin.hive.metastore.PrincipalPrivileges.fromHivePrivilegeInfos; -import static io.prestosql.plugin.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; import static io.prestosql.plugin.hive.metastore.StorageFormat.fromHiveStorageFormat; import static io.prestosql.plugin.hive.util.CompressionConfigUtil.configureCompression; import static io.prestosql.plugin.hive.util.ConfigurationUtils.toJobConf; @@ -197,8 +195,6 @@ import static io.prestosql.plugin.hive.util.HiveUtil.PRESTO_VIEW_FLAG; import static io.prestosql.plugin.hive.util.HiveUtil.buildHiveViewConnectorDefinition; import static io.prestosql.plugin.hive.util.HiveUtil.columnExtraInfo; -import static io.prestosql.plugin.hive.util.HiveUtil.decodeViewData; -import static io.prestosql.plugin.hive.util.HiveUtil.encodeViewData; import static io.prestosql.plugin.hive.util.HiveUtil.getPartitionKeyColumnHandles; import static io.prestosql.plugin.hive.util.HiveUtil.hiveColumnHandles; import static io.prestosql.plugin.hive.util.HiveUtil.isPrestoView; @@ -1635,31 +1631,11 @@ private static Map getColumnStatistics(Map properties = ImmutableMap.builder() - .put(TABLE_COMMENT, "Presto View") - .put(PRESTO_VIEW_FLAG, "true") - .put(PRESTO_VERSION_NAME, prestoVersion) - .put(PRESTO_QUERY_ID_NAME, session.getQueryId()) - .build(); + String owner = session.getUser(); - Column dummyColumn = new Column("dummy", HIVE_STRING, Optional.empty()); - - Table.Builder tableBuilder = Table.builder() - .setDatabaseName(viewName.getSchemaName()) - .setTableName(viewName.getTableName()) - .setOwner(session.getUser()) - .setTableType(TableType.VIRTUAL_VIEW.name()) - .setDataColumns(ImmutableList.of(dummyColumn)) - .setPartitionColumns(ImmutableList.of()) - .setParameters(properties) - .setViewOriginalText(Optional.of(encodeViewData(definition))) - .setViewExpandedText(Optional.of("/* Presto View */")); + Table table = HiveUtil.buildViewTable(definition, viewName, owner, session.getQueryId(), prestoVersion); - tableBuilder.getStorageBuilder() - .setStorageFormat(VIEW_STORAGE_FORMAT) - .setLocation(""); - Table table = tableBuilder.build(); - PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(session.getUser()); + PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(owner); Optional existing = metastore.getTable(identity, viewName.getSchemaName(), viewName.getTableName()); if (existing.isPresent()) { @@ -1715,29 +1691,11 @@ public List listViews(ConnectorSession session, Optional getView(ConnectorSession session, SchemaTableName viewName) { - return metastore.getTable(new HiveIdentity(session), viewName.getSchemaName(), viewName.getTableName()) - .flatMap(view -> { - if (isPrestoView(view)) { - ConnectorViewDefinition definition = decodeViewData(view.getViewOriginalText() - .orElseThrow(() -> new PrestoException(HIVE_INVALID_METADATA, "No view original text: " + viewName))); - // use owner from table metadata if it exists - if (view.getOwner() != null && !definition.isRunAsInvoker()) { - definition = new ConnectorViewDefinition( - definition.getOriginalSql(), - definition.getCatalog(), - definition.getSchema(), - definition.getColumns(), - definition.getComment(), - Optional.of(view.getOwner()), - false); - } - return Optional.of(definition); - } - if (translateHiveViews && isHiveOrPrestoView(view)) { - return Optional.of(buildHiveViewConnectorDefinition(catalogName, view)); - } - return Optional.empty(); - }); + HiveIdentity identity = new HiveIdentity(session); + + return metastore.getTable(identity, viewName.getSchemaName(), viewName.getTableName()) + .filter(HiveUtil::isPrestoView) + .map(HiveUtil::buildViewDefinition); } private boolean isHiveOrPrestoView(Table table) diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveUtil.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveUtil.java index 5d6f887d5fb6..0dbe39969e7b 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveUtil.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/util/HiveUtil.java @@ -17,6 +17,7 @@ import com.google.common.base.Splitter; import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.airlift.compress.lzo.LzoCodec; import io.airlift.compress.lzo.LzopCodec; import io.airlift.json.JsonCodec; @@ -38,6 +39,7 @@ import io.prestosql.spi.connector.ConnectorViewDefinition; import io.prestosql.spi.connector.ConnectorViewDefinition.ViewColumn; import io.prestosql.spi.connector.RecordCursor; +import io.prestosql.spi.connector.SchemaTableName; import io.prestosql.spi.predicate.NullableValue; import io.prestosql.spi.type.ArrayType; import io.prestosql.spi.type.CharType; @@ -55,6 +57,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.JavaUtils; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.SymlinkTextInputFormat; @@ -95,6 +98,7 @@ import java.math.BigInteger; import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Properties; @@ -123,12 +127,16 @@ import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_INVALID_VIEW_DATA; import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_SERDE_NOT_FOUND; import static io.prestosql.plugin.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT; +import static io.prestosql.plugin.hive.HiveMetadata.PRESTO_QUERY_ID_NAME; +import static io.prestosql.plugin.hive.HiveMetadata.PRESTO_VERSION_NAME; import static io.prestosql.plugin.hive.HiveMetadata.SKIP_FOOTER_COUNT_KEY; import static io.prestosql.plugin.hive.HiveMetadata.SKIP_HEADER_COUNT_KEY; import static io.prestosql.plugin.hive.HiveMetadata.TABLE_COMMENT; import static io.prestosql.plugin.hive.HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION; +import static io.prestosql.plugin.hive.HiveType.HIVE_STRING; import static io.prestosql.plugin.hive.HiveQlTranslation.translateHiveQlToPrestoSql; import static io.prestosql.plugin.hive.HiveType.toHiveTypes; +import static io.prestosql.plugin.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; import static io.prestosql.plugin.hive.util.ConfigurationUtils.copy; import static io.prestosql.plugin.hive.util.ConfigurationUtils.toJobConf; import static io.prestosql.plugin.hive.util.HiveBucketing.containsTimestampBucketedV2; @@ -1039,4 +1047,49 @@ public static List getColumnTypes(Properties schema) { return toHiveTypes(schema.getProperty(IOConstants.COLUMNS_TYPES, "")); } + + public static Table buildViewTable(ConnectorViewDefinition definition, SchemaTableName viewName, String owner, String queryId, String prestoVersion) + { + Map properties = ImmutableMap.builder() + .put(TABLE_COMMENT, "Presto View") + .put(PRESTO_VIEW_FLAG, "true") + .put(PRESTO_VERSION_NAME, prestoVersion) + .put(PRESTO_QUERY_ID_NAME, queryId) + .build(); + + Column dummyColumn = new Column("dummy", HIVE_STRING, Optional.empty()); + + return Table.builder() + .setDatabaseName(viewName.getSchemaName()) + .setTableName(viewName.getTableName()) + .setOwner(owner) + .setTableType(TableType.VIRTUAL_VIEW.name()) + .setDataColumns(ImmutableList.of(dummyColumn)) + .setPartitionColumns(ImmutableList.of()) + .setParameters(properties) + .setViewOriginalText(Optional.of(encodeViewData(definition))) + .setViewExpandedText(Optional.of("/* Presto View */")) + .withStorage(storage -> storage + .setStorageFormat(VIEW_STORAGE_FORMAT) + .setLocation("")) + .build(); + } + + public static ConnectorViewDefinition buildViewDefinition(Table view) + { + ConnectorViewDefinition definition = decodeViewData(view.getViewOriginalText() + .orElseThrow(() -> new PrestoException(HIVE_INVALID_METADATA, "No view original text: " + view.getTableName()))); + // use owner from table metadata if it exists + if (view.getOwner() != null && !definition.isRunAsInvoker()) { + definition = new ConnectorViewDefinition( + definition.getOriginalSql(), + definition.getCatalog(), + definition.getSchema(), + definition.getColumns(), + definition.getComment(), + Optional.of(view.getOwner()), + false); + } + return definition; + } } diff --git a/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/IcebergMetadata.java b/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/IcebergMetadata.java index 2a7432dafa61..fabbb028e36e 100644 --- a/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/IcebergMetadata.java +++ b/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/IcebergMetadata.java @@ -23,10 +23,13 @@ import io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext; import io.prestosql.plugin.hive.HiveWrittenPartitions; import io.prestosql.plugin.hive.TableAlreadyExistsException; +import io.prestosql.plugin.hive.ViewAlreadyExistsException; import io.prestosql.plugin.hive.authentication.HiveIdentity; import io.prestosql.plugin.hive.metastore.Database; import io.prestosql.plugin.hive.metastore.HiveMetastore; +import io.prestosql.plugin.hive.metastore.PrincipalPrivileges; import io.prestosql.plugin.hive.metastore.Table; +import io.prestosql.plugin.hive.util.HiveUtil; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; @@ -39,6 +42,7 @@ import io.prestosql.spi.connector.ConnectorTableHandle; import io.prestosql.spi.connector.ConnectorTableMetadata; import io.prestosql.spi.connector.ConnectorTableProperties; +import io.prestosql.spi.connector.ConnectorViewDefinition; import io.prestosql.spi.connector.Constraint; import io.prestosql.spi.connector.ConstraintApplicationResult; import io.prestosql.spi.connector.SchemaNotFoundException; @@ -46,6 +50,7 @@ import io.prestosql.spi.connector.SchemaTablePrefix; import io.prestosql.spi.connector.SystemTable; import io.prestosql.spi.connector.TableNotFoundException; +import io.prestosql.spi.connector.ViewNotFoundException; import io.prestosql.spi.predicate.TupleDomain; import io.prestosql.spi.statistics.ComputedStatistics; import io.prestosql.spi.type.DecimalType; @@ -81,6 +86,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.prestosql.plugin.hive.HiveSchemaProperties.getLocation; +import static io.prestosql.plugin.hive.metastore.MetastoreUtil.buildInitialPrivilegeSet; import static io.prestosql.plugin.hive.util.HiveWriteUtils.getTableDefaultLocation; import static io.prestosql.plugin.iceberg.DomainConverter.convertTupleDomainTypes; import static io.prestosql.plugin.iceberg.ExpressionConverter.toIcebergExpression; @@ -118,6 +124,7 @@ public class IcebergMetadata private final HdfsEnvironment hdfsEnvironment; private final TypeManager typeManager; private final JsonCodec commitTaskCodec; + private final String prestoVersion; private Transaction transaction; @@ -125,12 +132,14 @@ public IcebergMetadata( HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, TypeManager typeManager, - JsonCodec commitTaskCodec) + JsonCodec commitTaskCodec, + String prestoVersion) { this.metastore = requireNonNull(metastore, "metastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.commitTaskCodec = requireNonNull(commitTaskCodec, "commitTaskCodec is null"); + this.prestoVersion = requireNonNull(prestoVersion, "prestoVersion is null"); } @Override @@ -564,4 +573,56 @@ public Optional> applyFilter(C table.getPredicate().intersect(newDomain)), constraint.getSummary())); } + + @Override + public void createView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition definition, boolean replace) + { + HiveIdentity identity = new HiveIdentity(session); + String owner = session.getUser(); + + Table table = HiveUtil.buildViewTable(definition, viewName, owner, session.getQueryId(), prestoVersion); + + PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(owner); + + Optional
existing = metastore.getTable(identity, viewName.getSchemaName(), viewName.getTableName()); + if (existing.isPresent()) { + if (!replace || !HiveUtil.isPrestoView(existing.get())) { + throw new ViewAlreadyExistsException(viewName); + } + + metastore.replaceTable(identity, viewName.getSchemaName(), viewName.getTableName(), table, principalPrivileges); + return; + } + + try { + metastore.createTable(identity, table, principalPrivileges); + } + catch (TableAlreadyExistsException e) { + throw new ViewAlreadyExistsException(e.getTableName()); + } + } + + @Override + public Optional getView(ConnectorSession session, SchemaTableName viewName) + { + return metastore.getTable(new HiveIdentity(session), viewName.getSchemaName(), viewName.getTableName()) + .filter(HiveUtil::isPrestoView) + .map(HiveUtil::buildViewDefinition); + } + + @Override + public void dropView(ConnectorSession session, SchemaTableName viewName) + { + if (!getView(session, viewName).isPresent()) { + throw new ViewNotFoundException(viewName); + } + + try { + HiveIdentity identity = new HiveIdentity(session); + metastore.dropTable(identity, viewName.getSchemaName(), viewName.getTableName(), false); + } + catch (TableNotFoundException e) { + throw new ViewNotFoundException(e.getTableName()); + } + } } diff --git a/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/IcebergMetadataFactory.java b/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/IcebergMetadataFactory.java index 826ab5eb52d6..34ebe1e6ecfd 100644 --- a/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/IcebergMetadataFactory.java +++ b/presto-iceberg/src/main/java/io/prestosql/plugin/iceberg/IcebergMetadataFactory.java @@ -15,6 +15,7 @@ import io.airlift.json.JsonCodec; import io.prestosql.plugin.hive.HdfsEnvironment; +import io.prestosql.plugin.hive.NodeVersion; import io.prestosql.plugin.hive.metastore.HiveMetastore; import io.prestosql.spi.type.TypeManager; @@ -30,6 +31,7 @@ public class IcebergMetadataFactory private final TypeManager typeManager; private final JsonCodec commitTaskCodec; private final long metastoreTransactionCacheSize; + private final String prestoVersion; @Inject public IcebergMetadataFactory( @@ -37,13 +39,15 @@ public IcebergMetadataFactory( HiveMetastore metastore, HdfsEnvironment hdfsEnvironment, TypeManager typeManager, - JsonCodec commitTaskDataJsonCodec) + JsonCodec commitTaskDataJsonCodec, + NodeVersion nodeVersion) { this(metastore, hdfsEnvironment, typeManager, commitTaskDataJsonCodec, - config.getMetastoreTransactionCacheSize()); + config.getMetastoreTransactionCacheSize(), + nodeVersion.toString()); } public IcebergMetadataFactory( @@ -51,13 +55,15 @@ public IcebergMetadataFactory( HdfsEnvironment hdfsEnvironment, TypeManager typeManager, JsonCodec commitTaskCodec, - long metastoreTransactionCacheSize) + long metastoreTransactionCacheSize, + String prestoVersion) { this.metastore = requireNonNull(metastore, "metastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.commitTaskCodec = requireNonNull(commitTaskCodec, "commitTaskCodec is null"); this.metastoreTransactionCacheSize = metastoreTransactionCacheSize; + this.prestoVersion = requireNonNull(prestoVersion, "prestoVersion is null"); } public IcebergMetadata create() @@ -66,6 +72,7 @@ public IcebergMetadata create() memoizeMetastore(metastore, metastoreTransactionCacheSize), hdfsEnvironment, typeManager, - commitTaskCodec); + commitTaskCodec, + prestoVersion); } }