diff --git a/plugin/trino-delta-lake/pom.xml b/plugin/trino-delta-lake/pom.xml index 1e189a5cabc3..84646b248e97 100644 --- a/plugin/trino-delta-lake/pom.xml +++ b/plugin/trino-delta-lake/pom.xml @@ -423,10 +423,12 @@ **/TestDeltaLakeAdlsConnectorSmokeTest.java **/TestDeltaLakeGlueMetastore.java **/TestDeltaLakeCleanUpGlueMetastore.java + **/TestDeltaLakeSharedGlueMetastoreViews.java **/TestDeltaLakeSharedGlueMetastoreWithTableRedirections.java **/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java **/TestDeltaLakeRenameToWithGlueMetastore.java **/TestDeltaLakeRegisterTableProcedureWithGlue.java + **/TestDeltaLakeViewsGlueMetastore.java **/TestDeltaLakeGcsConnectorSmokeTest.java @@ -474,10 +476,12 @@ **/TestDeltaLakeAdlsConnectorSmokeTest.java **/TestDeltaLakeGlueMetastore.java **/TestDeltaLakeCleanUpGlueMetastore.java + **/TestDeltaLakeSharedGlueMetastoreViews.java **/TestDeltaLakeSharedGlueMetastoreWithTableRedirections.java **/TestDeltaLakeTableWithCustomLocationUsingGlueMetastore.java **/TestDeltaLakeRenameToWithGlueMetastore.java **/TestDeltaLakeRegisterTableProcedureWithGlue.java + **/TestDeltaLakeViewsGlueMetastore.java diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java index 5748370105f1..c0e5e68d51c1 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadata.java @@ -53,6 +53,7 @@ import io.trino.plugin.hive.HiveType; import io.trino.plugin.hive.SchemaAlreadyExistsException; import io.trino.plugin.hive.TableAlreadyExistsException; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.Column; import io.trino.plugin.hive.metastore.Database; import io.trino.plugin.hive.metastore.HivePrincipal; @@ -82,6 +83,7 @@ import io.trino.spi.connector.ConnectorTableLayout; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.ConnectorTableProperties; +import io.trino.spi.connector.ConnectorViewDefinition; import io.trino.spi.connector.Constraint; import io.trino.spi.connector.ConstraintApplicationResult; import io.trino.spi.connector.ProjectionApplicationResult; @@ -284,6 +286,7 @@ public class DeltaLakeMetadata private final HdfsEnvironment hdfsEnvironment; private final TypeManager typeManager; private final AccessControlMetadata accessControlMetadata; + private final TrinoViewHiveMetastore trinoViewHiveMetastore; private final CheckpointWriterManager checkpointWriterManager; private final long defaultCheckpointInterval; private final int domainCompactionThreshold; @@ -307,6 +310,7 @@ public DeltaLakeMetadata( HdfsEnvironment hdfsEnvironment, TypeManager typeManager, AccessControlMetadata accessControlMetadata, + TrinoViewHiveMetastore trinoViewHiveMetastore, int domainCompactionThreshold, boolean unsafeWritesEnabled, JsonCodec dataFileInfoCodec, @@ -327,6 +331,7 @@ public DeltaLakeMetadata( this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.accessControlMetadata = requireNonNull(accessControlMetadata, "accessControlMetadata is null"); + this.trinoViewHiveMetastore = requireNonNull(trinoViewHiveMetastore, "trinoViewHiveMetastore is null"); this.domainCompactionThreshold = domainCompactionThreshold; this.unsafeWritesEnabled = unsafeWritesEnabled; this.dataFileInfoCodec = requireNonNull(dataFileInfoCodec, "dataFileInfoCodec is null"); @@ -2075,6 +2080,36 @@ public Map getSchemaProperties(ConnectorSession session, Catalog return db.map(DeltaLakeSchemaProperties::fromDatabase).orElseThrow(() -> new SchemaNotFoundException(schema)); } + @Override + public void createView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition definition, boolean replace) + { + trinoViewHiveMetastore.createView(session, viewName, definition, replace); + } + + @Override + public void dropView(ConnectorSession session, SchemaTableName viewName) + { + trinoViewHiveMetastore.dropView(viewName); + } + + @Override + public List listViews(ConnectorSession session, Optional schemaName) + { + return trinoViewHiveMetastore.listViews(schemaName); + } + + @Override + public Map getViews(ConnectorSession session, Optional schemaName) + { + return trinoViewHiveMetastore.getViews(schemaName); + } + + @Override + public Optional getView(ConnectorSession session, SchemaTableName viewName) + { + return trinoViewHiveMetastore.getView(viewName); + } + @Override public void createRole(ConnectorSession session, String role, Optional grantor) { diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java index 419e1c025563..49e0e77735f8 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java @@ -21,8 +21,11 @@ import io.trino.plugin.deltalake.transactionlog.TransactionLogAccess; import io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointWriterManager; import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriterFactory; +import io.trino.plugin.hive.NodeVersion; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; +import io.trino.plugin.hive.security.AccessControlMetadata; import io.trino.spi.NodeManager; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.type.TypeManager; @@ -58,6 +61,7 @@ public class DeltaLakeMetadataFactory private final boolean useUniqueTableLocation; private final boolean allowManagedTableRename; + private final String trinoVersion; @Inject public DeltaLakeMetadataFactory( @@ -76,7 +80,8 @@ public DeltaLakeMetadataFactory( CheckpointWriterManager checkpointWriterManager, DeltaLakeRedirectionsProvider deltaLakeRedirectionsProvider, CachingExtendedStatisticsAccess statisticsAccess, - @AllowDeltaLakeManagedTableRename boolean allowManagedTableRename) + @AllowDeltaLakeManagedTableRename boolean allowManagedTableRename, + NodeVersion nodeVersion) { this.hiveMetastoreFactory = requireNonNull(hiveMetastoreFactory, "hiveMetastore is null"); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); @@ -99,6 +104,7 @@ public DeltaLakeMetadataFactory( this.deleteSchemaLocationsFallback = deltaLakeConfig.isDeleteSchemaLocationsFallback(); this.useUniqueTableLocation = deltaLakeConfig.isUniqueTableLocation(); this.allowManagedTableRename = allowManagedTableRename; + this.trinoVersion = requireNonNull(nodeVersion, "nodeVersion is null").toString(); } public DeltaLakeMetadata create(ConnectorIdentity identity) @@ -107,18 +113,25 @@ public DeltaLakeMetadata create(ConnectorIdentity identity) CachingHiveMetastore cachingHiveMetastore = memoizeMetastore( hiveMetastoreFactory.createMetastore(Optional.of(identity)), perTransactionMetastoreCacheMaximumSize); + AccessControlMetadata accessControlMetadata = accessControlMetadataFactory.create(cachingHiveMetastore); HiveMetastoreBackedDeltaLakeMetastore deltaLakeMetastore = new HiveMetastoreBackedDeltaLakeMetastore( cachingHiveMetastore, transactionLogAccess, typeManager, statisticsAccess, fileSystemFactory); + TrinoViewHiveMetastore trinoViewHiveMetastore = new TrinoViewHiveMetastore( + cachingHiveMetastore, + accessControlMetadata.isUsingSystemSecurity(), + trinoVersion, + "Trino Delta Lake connector"); return new DeltaLakeMetadata( deltaLakeMetastore, fileSystemFactory, hdfsEnvironment, typeManager, - accessControlMetadataFactory.create(cachingHiveMetastore), + accessControlMetadata, + trinoViewHiveMetastore, domainCompactionThreshold, unsafeWritesEnabled, dataFileInfoCodec, diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java index 2a43c3ed842d..94290ae288cd 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeConnectorSmokeTest.java @@ -190,6 +190,9 @@ protected QueryRunner createQueryRunner() protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) { switch (connectorBehavior) { + case SUPPORTS_CREATE_VIEW: + return true; + case SUPPORTS_RENAME_SCHEMA: return false; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeMinioConnectorTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeMinioConnectorTest.java index fa39e3c6b0b7..d56b2611b23b 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeMinioConnectorTest.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeMinioConnectorTest.java @@ -118,6 +118,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) case SUPPORTS_DELETE: case SUPPORTS_UPDATE: case SUPPORTS_MERGE: + case SUPPORTS_CREATE_VIEW: return true; default: diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreViewsTest.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreViewsTest.java new file mode 100644 index 000000000000..9c88a40c67ea --- /dev/null +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/BaseDeltaLakeSharedMetastoreViewsTest.java @@ -0,0 +1,166 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.deltalake; + +import com.google.common.collect.ImmutableMap; +import io.trino.Session; +import io.trino.plugin.deltalake.metastore.TestingDeltaLakeMetastoreModule; +import io.trino.plugin.hive.TestingHivePlugin; +import io.trino.plugin.hive.metastore.HiveMetastore; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; + +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static com.google.inject.util.Modules.EMPTY_MODULE; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; + +/** + * Tests querying views on a schema which has a mix of Hive and Delta Lake tables. + */ +public abstract class BaseDeltaLakeSharedMetastoreViewsTest + extends AbstractTestQueryFramework +{ + protected static final String DELTA_CATALOG_NAME = "delta_lake"; + protected static final String HIVE_CATALOG_NAME = "hive"; + protected static final String SCHEMA = "test_shared_schema_views_" + randomNameSuffix(); + + private String dataDirectory; + private HiveMetastore metastore; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + Session session = testSessionBuilder() + .setCatalog(DELTA_CATALOG_NAME) + .setSchema(SCHEMA) + .build(); + DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session).build(); + + this.dataDirectory = queryRunner.getCoordinator().getBaseDataDir().resolve("delta_lake_data").toString(); + this.metastore = createTestMetastore(dataDirectory); + + queryRunner.installPlugin(new TestingDeltaLakePlugin(Optional.of(new TestingDeltaLakeMetastoreModule(metastore)), EMPTY_MODULE)); + queryRunner.createCatalog(DELTA_CATALOG_NAME, "delta-lake"); + + queryRunner.installPlugin(new TestingHivePlugin(metastore)); + + ImmutableMap hiveProperties = ImmutableMap.builder() + .put("hive.allow-drop-table", "true") + .buildOrThrow(); + + queryRunner.createCatalog(HIVE_CATALOG_NAME, "hive", hiveProperties); + queryRunner.execute("CREATE SCHEMA " + SCHEMA); + + return queryRunner; + } + + protected abstract HiveMetastore createTestMetastore(String dataDirectory); + + @Test + public void testViewWithLiteralColumnCreatedInDeltaLakeIsReadableInHive() + { + String deltaViewName = "delta_view_" + randomNameSuffix(); + String deltaView = format("%s.%s.%s", DELTA_CATALOG_NAME, SCHEMA, deltaViewName); + String deltaViewOnHiveCatalog = format("%s.%s.%s", HIVE_CATALOG_NAME, SCHEMA, deltaViewName); + try { + assertUpdate(format("CREATE VIEW %s AS SELECT 1 bee", deltaView)); + assertQuery(format("SELECT * FROM %s", deltaView), "VALUES 1"); + assertQuery(format("SELECT * FROM %s", deltaViewOnHiveCatalog), "VALUES 1"); + assertQuery(format("SELECT table_type FROM %s.information_schema.tables WHERE table_name = '%s' AND table_schema='%s'", HIVE_CATALOG_NAME, deltaViewName, SCHEMA), "VALUES 'VIEW'"); + } + finally { + assertUpdate(format("DROP VIEW IF EXISTS %s", deltaView)); + } + } + + @Test + public void testViewOnDeltaLakeTableCreatedInDeltaLakeIsReadableInHive() + { + String deltaTableName = "delta_table_" + randomNameSuffix(); + String deltaTable = format("%s.%s.%s", DELTA_CATALOG_NAME, SCHEMA, deltaTableName); + String deltaViewName = "delta_view_" + randomNameSuffix(); + String deltaView = format("%s.%s.%s", DELTA_CATALOG_NAME, SCHEMA, deltaViewName); + String deltaViewOnHiveCatalog = format("%s.%s.%s", HIVE_CATALOG_NAME, SCHEMA, deltaViewName); + try { + assertUpdate(format("CREATE TABLE %s AS SELECT 1 bee", deltaTable), 1); + assertUpdate(format("CREATE VIEW %s AS SELECT * from %s", deltaView, deltaTable)); + assertQuery(format("SELECT * FROM %s", deltaView), "VALUES 1"); + assertQuery(format("SELECT * FROM %s", deltaViewOnHiveCatalog), "VALUES 1"); + assertQuery(format("SELECT table_type FROM %s.information_schema.tables WHERE table_name = '%s' AND table_schema='%s'", HIVE_CATALOG_NAME, deltaViewName, SCHEMA), "VALUES 'VIEW'"); + } + finally { + assertUpdate(format("DROP TABLE IF EXISTS %s", deltaTable)); + assertUpdate(format("DROP VIEW IF EXISTS %s", deltaView)); + } + } + + @Test + public void testViewWithLiteralColumnCreatedInHiveIsReadableInDeltaLake() + { + String trinoViewOnHiveName = "trino_view_on_hive_" + randomNameSuffix(); + String trinoViewOnHive = format("%s.%s.%s", HIVE_CATALOG_NAME, SCHEMA, trinoViewOnHiveName); + String trinoViewOnHiveOnDeltaCatalog = format("%s.%s.%s", DELTA_CATALOG_NAME, SCHEMA, trinoViewOnHiveName); + try { + assertUpdate(format("CREATE VIEW %s AS SELECT 1 bee", trinoViewOnHive)); + assertQuery(format("SELECT * FROM %s", trinoViewOnHive), "VALUES 1"); + assertQuery(format("SELECT * FROM %s", trinoViewOnHiveOnDeltaCatalog), "VALUES 1"); + assertQuery(format("SELECT table_type FROM %s.information_schema.tables WHERE table_name = '%s' AND table_schema='%s'", HIVE_CATALOG_NAME, trinoViewOnHiveName, SCHEMA), "VALUES 'VIEW'"); + } + finally { + assertUpdate(format("DROP VIEW IF EXISTS %s", trinoViewOnHive)); + } + } + + @Test + public void testViewOnHiveTableCreatedInHiveIsReadableInDeltaLake() + { + String hiveTableName = "hive_table_" + randomNameSuffix(); + String hiveTable = format("%s.%s.%s", HIVE_CATALOG_NAME, SCHEMA, hiveTableName); + String trinoViewOnHiveName = "trino_view_on_hive_" + randomNameSuffix(); + String trinoViewOnHive = format("%s.%s.%s", HIVE_CATALOG_NAME, SCHEMA, trinoViewOnHiveName); + String trinoViewOnHiveOnDeltaCatalog = format("%s.%s.%s", DELTA_CATALOG_NAME, SCHEMA, trinoViewOnHiveName); + try { + assertUpdate(format("CREATE TABLE %s AS SELECT 1 bee", hiveTable), 1); + assertUpdate(format("CREATE VIEW %s AS SELECT 1 bee", trinoViewOnHive)); + assertQuery(format("SELECT * FROM %s", trinoViewOnHive), "VALUES 1"); + assertQuery(format("SELECT * FROM %s", trinoViewOnHiveOnDeltaCatalog), "VALUES 1"); + assertQuery(format("SELECT table_type FROM %s.information_schema.tables WHERE table_name = '%s' AND table_schema='%s'", DELTA_CATALOG_NAME, trinoViewOnHiveName, SCHEMA), "VALUES 'VIEW'"); + } + finally { + assertUpdate(format("DROP TABLE IF EXISTS %s", hiveTable)); + assertUpdate(format("DROP VIEW IF EXISTS %s", trinoViewOnHive)); + } + } + + @AfterClass(alwaysRun = true) + public void cleanup() + throws IOException + { + if (metastore != null) { + metastore.dropDatabase(SCHEMA, false); + deleteRecursively(Path.of(dataDirectory), ALLOW_INSECURE); + } + } +} diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedFileMetastoreViews.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedFileMetastoreViews.java new file mode 100644 index 000000000000..f41a91d836d4 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedFileMetastoreViews.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.deltalake; + +import io.trino.plugin.hive.metastore.HiveMetastore; + +import java.io.File; + +import static io.trino.plugin.hive.metastore.file.FileHiveMetastore.createTestingFileHiveMetastore; + +public class TestDeltaLakeSharedFileMetastoreViews + extends BaseDeltaLakeSharedMetastoreViewsTest +{ + @Override + protected HiveMetastore createTestMetastore(String dataDirectory) + { + return createTestingFileHiveMetastore(new File(dataDirectory)); + } +} diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedGlueMetastoreViews.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedGlueMetastoreViews.java new file mode 100644 index 000000000000..2e2008d8ac77 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSharedGlueMetastoreViews.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.deltalake; + +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import io.trino.plugin.hive.metastore.HiveMetastore; +import io.trino.plugin.hive.metastore.glue.DefaultGlueColumnStatisticsProviderFactory; +import io.trino.plugin.hive.metastore.glue.GlueHiveMetastore; +import io.trino.plugin.hive.metastore.glue.GlueHiveMetastoreConfig; + +import java.util.Optional; + +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; + +/** + * Requires AWS credentials, which can be provided any way supported by the DefaultProviderChain + * See https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default + */ +public class TestDeltaLakeSharedGlueMetastoreViews + extends BaseDeltaLakeSharedMetastoreViewsTest +{ + @Override + protected HiveMetastore createTestMetastore(String dataDirectory) + { + return new GlueHiveMetastore( + HDFS_ENVIRONMENT, + new GlueHiveMetastoreConfig() + .setDefaultWarehouseDir(dataDirectory), + DefaultAWSCredentialsProviderChain.getInstance(), + directExecutor(), + new DefaultGlueColumnStatisticsProviderFactory(directExecutor(), directExecutor()), + Optional.empty(), + table -> true); + } +} diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java index 129943d81f61..c45c8584e143 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestDeltaLakeMetastoreAccessOperations.java @@ -153,18 +153,24 @@ public void testSelectWithFilter() public void testSelectFromView() { assertUpdate("CREATE TABLE test_select_view_table (id VARCHAR, age INT)"); - assertQueryFails( - "CREATE VIEW test_select_view_view AS SELECT id, age FROM test_select_view_table", - "This connector does not support creating views"); + assertUpdate("CREATE VIEW test_select_view_view AS SELECT id, age FROM test_select_view_table"); + + assertMetastoreInvocations("SELECT * FROM test_select_view_view", + ImmutableMultiset.builder() + .addCopies(GET_TABLE, 2) + .build()); } @Test public void testSelectFromViewWithFilter() { assertUpdate("CREATE TABLE test_select_view_where_table AS SELECT 2 as age", 1); - assertQueryFails( - "CREATE VIEW test_select_view_where_view AS SELECT age FROM test_select_view_where_table", - "This connector does not support creating views"); + assertUpdate("CREATE VIEW test_select_view_where_view AS SELECT age FROM test_select_view_where_table"); + + assertMetastoreInvocations("SELECT * FROM test_select_view_where_view WHERE age = 2", + ImmutableMultiset.builder() + .addCopies(GET_TABLE, 2) + .build()); } @Test diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java index f5d4fab2ae48..ffdf9566ea8c 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java @@ -61,7 +61,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.function.Consumer; import static com.google.common.base.Verify.verify; @@ -80,8 +79,8 @@ import static io.trino.plugin.hive.metastore.StorageFormat.fromHiveStorageFormat; import static io.trino.spi.security.PrincipalType.ROLE; import static io.trino.testing.TestingConnectorSession.SESSION; +import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; -import static java.util.Locale.ENGLISH; import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE; import static org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW; import static org.assertj.core.api.Assertions.assertThat; @@ -144,7 +143,7 @@ public void setUp() .collect(toImmutableList())) .build(); - databaseName = "test_delta_glue" + randomName(); + databaseName = "test_delta_glue" + randomNameSuffix(); metastoreClient.createDatabase(Database.builder() .setDatabaseName(databaseName) .setOwnerName(Optional.of("public")) @@ -175,10 +174,10 @@ public void tearDown() public void testHideNonDeltaLakeTable() throws Exception { - SchemaTableName deltaLakeTable = new SchemaTableName(databaseName, "delta_lake_table_" + randomName()); - SchemaTableName nonDeltaLakeTable1 = new SchemaTableName(databaseName, "hive_table_" + randomName()); - SchemaTableName nonDeltaLakeTable2 = new SchemaTableName(databaseName, "hive_table_" + randomName()); - SchemaTableName nonDeltaLakeView1 = new SchemaTableName(databaseName, "hive_view_" + randomName()); + SchemaTableName deltaLakeTable = new SchemaTableName(databaseName, "delta_lake_table_" + randomNameSuffix()); + SchemaTableName nonDeltaLakeTable1 = new SchemaTableName(databaseName, "hive_table_" + randomNameSuffix()); + SchemaTableName nonDeltaLakeTable2 = new SchemaTableName(databaseName, "hive_table_" + randomNameSuffix()); + SchemaTableName nonDeltaLakeView1 = new SchemaTableName(databaseName, "hive_view_" + randomNameSuffix()); String deltaLakeTableLocation = tableLocation(deltaLakeTable); createTable(deltaLakeTable, deltaLakeTableLocation, tableBuilder -> { @@ -313,9 +312,4 @@ private void createView(SchemaTableName viewName, String tableLocation, Consumer PrincipalPrivileges principalPrivileges = new PrincipalPrivileges(ImmutableMultimap.of(), ImmutableMultimap.of()); metastoreClient.createTable(table.build(), principalPrivileges); } - - private static String randomName() - { - return UUID.randomUUID().toString().toLowerCase(ENGLISH).replace("-", ""); - } } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java new file mode 100644 index 000000000000..1f3c31b84745 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeViewsGlueMetastore.java @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.deltalake.metastore.glue; + +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import io.trino.Session; +import io.trino.plugin.deltalake.TestingDeltaLakePlugin; +import io.trino.plugin.deltalake.metastore.TestingDeltaLakeMetastoreModule; +import io.trino.plugin.hive.metastore.HiveMetastore; +import io.trino.plugin.hive.metastore.glue.DefaultGlueColumnStatisticsProviderFactory; +import io.trino.plugin.hive.metastore.glue.GlueHiveMetastore; +import io.trino.plugin.hive.metastore.glue.GlueHiveMetastoreConfig; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import io.trino.testing.sql.TestTable; +import io.trino.testing.sql.TestView; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; + +import static com.google.common.io.MoreFiles.deleteRecursively; +import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static com.google.inject.util.Modules.EMPTY_MODULE; +import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; + +public class TestDeltaLakeViewsGlueMetastore + extends AbstractTestQueryFramework +{ + private static final String SCHEMA = "test_delta_lake_glue_views_" + randomNameSuffix(); + private static final String CATALOG_NAME = "test_delta_lake_glue_views"; + private String dataDirectory; + private HiveMetastore metastore; + + private HiveMetastore createTestMetastore(String dataDirectory) + { + return new GlueHiveMetastore( + HDFS_ENVIRONMENT, + new GlueHiveMetastoreConfig() + .setDefaultWarehouseDir(dataDirectory), + DefaultAWSCredentialsProviderChain.getInstance(), + directExecutor(), + new DefaultGlueColumnStatisticsProviderFactory(directExecutor(), directExecutor()), + Optional.empty(), + table -> true); + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + Session deltaLakeSession = testSessionBuilder() + .setCatalog(CATALOG_NAME) + .setSchema(SCHEMA) + .build(); + + DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(deltaLakeSession).build(); + + dataDirectory = queryRunner.getCoordinator().getBaseDataDir().resolve("data_delta_lake_views").toString(); + metastore = createTestMetastore(dataDirectory); + + queryRunner.installPlugin(new TestingDeltaLakePlugin(Optional.of(new TestingDeltaLakeMetastoreModule(metastore)), EMPTY_MODULE)); + queryRunner.createCatalog(CATALOG_NAME, "delta-lake"); + + queryRunner.execute("CREATE SCHEMA " + SCHEMA); + return queryRunner; + } + + @Test + public void testCreateView() + { + String tableName = "test_glue_table_" + randomNameSuffix(); + String viewName = "test_glue_view_" + randomNameSuffix(); + try (TestTable table = new TestTable(getQueryRunner()::execute, tableName, "AS SELECT 'test' x"); + TestView view = new TestView(getQueryRunner()::execute, viewName, "SELECT * FROM " + table.getName())) { + assertQuery(format("SELECT * FROM %s", view.getName()), "VALUES 'test'"); + assertQuery(format("SELECT table_type FROM information_schema.tables WHERE table_name = '%s' AND table_schema='%s'", view.getName(), SCHEMA), "VALUES 'VIEW'"); + } + } + + @AfterClass(alwaysRun = true) + public void cleanup() + throws IOException + { + if (metastore != null) { + metastore.dropDatabase(SCHEMA, false); + deleteRecursively(Path.of(dataDirectory), ALLOW_INSECURE); + } + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewHiveMetastore.java new file mode 100644 index 000000000000..08ac71168857 --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewHiveMetastore.java @@ -0,0 +1,176 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.plugin.hive.metastore.Column; +import io.trino.plugin.hive.metastore.HiveMetastore; +import io.trino.plugin.hive.metastore.PrincipalPrivileges; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorViewDefinition; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.TableNotFoundException; +import io.trino.spi.connector.ViewNotFoundException; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.plugin.hive.HiveMetadata.PRESTO_VIEW_COMMENT; +import static io.trino.plugin.hive.HiveMetadata.PRESTO_VIEW_EXPANDED_TEXT_MARKER; +import static io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT; +import static io.trino.plugin.hive.HiveType.HIVE_STRING; +import static io.trino.plugin.hive.TrinoViewUtil.createViewProperties; +import static io.trino.plugin.hive.ViewReaderUtil.encodeViewData; +import static io.trino.plugin.hive.ViewReaderUtil.isPrestoView; +import static io.trino.plugin.hive.metastore.MetastoreUtil.buildInitialPrivilegeSet; +import static io.trino.plugin.hive.metastore.PrincipalPrivileges.NO_PRIVILEGES; +import static io.trino.plugin.hive.metastore.StorageFormat.VIEW_STORAGE_FORMAT; +import static io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema; +import static io.trino.spi.StandardErrorCode.TABLE_NOT_FOUND; +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW; + +public final class TrinoViewHiveMetastore +{ + private final boolean isUsingSystemSecurity; + private final HiveMetastore metastore; + private final String trinoVersion; + private final String connectorName; + + public TrinoViewHiveMetastore(HiveMetastore metastore, boolean isUsingSystemSecurity, String trinoVersion, String connectorName) + { + this.metastore = requireNonNull(metastore, "metastore is null"); + this.isUsingSystemSecurity = isUsingSystemSecurity; + this.trinoVersion = requireNonNull(trinoVersion, "trinoVersion is null"); + this.connectorName = requireNonNull(connectorName, "connectorName is null"); + } + + public void createView(ConnectorSession session, SchemaTableName schemaViewName, ConnectorViewDefinition definition, boolean replace) + { + if (isUsingSystemSecurity) { + definition = definition.withoutOwner(); + } + + io.trino.plugin.hive.metastore.Table.Builder tableBuilder = io.trino.plugin.hive.metastore.Table.builder() + .setDatabaseName(schemaViewName.getSchemaName()) + .setTableName(schemaViewName.getTableName()) + .setOwner(isUsingSystemSecurity ? Optional.empty() : Optional.of(session.getUser())) + .setTableType(VIRTUAL_VIEW.name()) + .setDataColumns(ImmutableList.of(new Column("dummy", HIVE_STRING, Optional.empty()))) + .setPartitionColumns(ImmutableList.of()) + .setParameters(createViewProperties(session, trinoVersion, connectorName)) + .setViewOriginalText(Optional.of(encodeViewData(definition))) + .setViewExpandedText(Optional.of(PRESTO_VIEW_EXPANDED_TEXT_MARKER)); + + tableBuilder.getStorageBuilder() + .setStorageFormat(VIEW_STORAGE_FORMAT) + .setLocation(""); + io.trino.plugin.hive.metastore.Table table = tableBuilder.build(); + PrincipalPrivileges principalPrivileges = isUsingSystemSecurity ? NO_PRIVILEGES : buildInitialPrivilegeSet(session.getUser()); + + Optional existing = metastore.getTable(schemaViewName.getSchemaName(), schemaViewName.getTableName()); + if (existing.isPresent()) { + if (!replace || !isPrestoView(existing.get())) { + throw new ViewAlreadyExistsException(schemaViewName); + } + + metastore.replaceTable(schemaViewName.getSchemaName(), schemaViewName.getTableName(), table, principalPrivileges); + return; + } + + try { + metastore.createTable(table, principalPrivileges); + } + catch (TableAlreadyExistsException e) { + throw new ViewAlreadyExistsException(e.getTableName()); + } + } + + public void dropView(SchemaTableName schemaViewName) + { + if (getView(schemaViewName).isEmpty()) { + throw new ViewNotFoundException(schemaViewName); + } + + try { + metastore.dropTable(schemaViewName.getSchemaName(), schemaViewName.getTableName(), true); + } + catch (TableNotFoundException e) { + throw new ViewNotFoundException(e.getTableName()); + } + } + + public List listViews(Optional database) + { + return listDatabases(database).stream() + .flatMap(this::listViews) + .collect(toImmutableList()); + } + + private List listDatabases(Optional database) + { + if (database.isPresent()) { + if (isHiveSystemSchema(database.get())) { + return ImmutableList.of(); + } + return ImmutableList.of(database.get()); + } + return metastore.getAllDatabases(); + } + + public Map getViews(Optional schemaName) + { + ImmutableMap.Builder views = ImmutableMap.builder(); + for (SchemaTableName name : listViews(schemaName)) { + try { + getView(name).ifPresent(view -> views.put(name, view)); + } + catch (TrinoException e) { + if (e.getErrorCode().equals(TABLE_NOT_FOUND.toErrorCode())) { + // Ignore view that was dropped during query execution (race condition) + } + else { + throw e; + } + } + } + return views.buildOrThrow(); + } + + private Stream listViews(String schema) + { + // Filter on PRESTO_VIEW_COMMENT to distinguish from materialized views + return metastore.getTablesWithParameter(schema, TABLE_COMMENT, PRESTO_VIEW_COMMENT).stream() + .map(table -> new SchemaTableName(schema, table)); + } + + public Optional getView(SchemaTableName viewName) + { + if (isHiveSystemSchema(viewName.getSchemaName())) { + return Optional.empty(); + } + return metastore.getTable(viewName.getSchemaName(), viewName.getTableName()) + .flatMap(view -> TrinoViewUtil.getView( + viewName, + view.getViewOriginalText(), + view.getTableType(), + view.getParameters(), + view.getOwner())); + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewUtil.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewUtil.java new file mode 100644 index 000000000000..189b2caa354d --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TrinoViewUtil.java @@ -0,0 +1,86 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive; + +import com.google.common.collect.ImmutableMap; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorViewDefinition; +import io.trino.spi.connector.SchemaTableName; + +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.trino.plugin.hive.HiveMetadata.PRESTO_QUERY_ID_NAME; +import static io.trino.plugin.hive.HiveMetadata.PRESTO_VERSION_NAME; +import static io.trino.plugin.hive.HiveMetadata.PRESTO_VIEW_COMMENT; +import static io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT; +import static io.trino.plugin.hive.HiveMetadata.TRINO_CREATED_BY; +import static io.trino.plugin.hive.ViewReaderUtil.PRESTO_VIEW_FLAG; +import static io.trino.plugin.hive.ViewReaderUtil.isHiveOrPrestoView; +import static io.trino.plugin.hive.ViewReaderUtil.isPrestoView; + +public final class TrinoViewUtil +{ + private TrinoViewUtil() {} + + public static Optional getView( + SchemaTableName viewName, + Optional viewOriginalText, + String tableType, + Map tableParameters, + Optional tableOwner) + { + if (!isView(tableType, tableParameters)) { + // Filter out Tables and Materialized Views + return Optional.empty(); + } + + if (!isPrestoView(tableParameters)) { + // Hive views are not compatible + throw new HiveViewNotSupportedException(viewName); + } + + checkArgument(viewOriginalText.isPresent(), "viewOriginalText must be present"); + ConnectorViewDefinition definition = ViewReaderUtil.PrestoViewReader.decodeViewData(viewOriginalText.get()); + // use owner from table metadata if it exists + if (tableOwner.isPresent() && !definition.isRunAsInvoker()) { + definition = new ConnectorViewDefinition( + definition.getOriginalSql(), + definition.getCatalog(), + definition.getSchema(), + definition.getColumns(), + definition.getComment(), + tableOwner, + false); + } + return Optional.of(definition); + } + + private static boolean isView(String tableType, Map tableParameters) + { + return isHiveOrPrestoView(tableType) && PRESTO_VIEW_COMMENT.equals(tableParameters.get(TABLE_COMMENT)); + } + + public static Map createViewProperties(ConnectorSession session, String trinoVersion, String connectorName) + { + return ImmutableMap.builder() + .put(PRESTO_VIEW_FLAG, "true") + .put(TRINO_CREATED_BY, connectorName) + .put(PRESTO_VERSION_NAME, trinoVersion) + .put(PRESTO_QUERY_ID_NAME, session.getQueryId()) + .put(TABLE_COMMENT, PRESTO_VIEW_COMMENT) + .buildOrThrow(); + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java index e446641f9e86..580cf6cc423f 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java @@ -126,6 +126,7 @@ import java.util.function.Function; import java.util.function.Predicate; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.base.Verify.verify; @@ -168,6 +169,7 @@ public class GlueHiveMetastore private static final int BATCH_UPDATE_PARTITION_MAX_PAGE_SIZE = 100; private static final int AWS_GLUE_GET_PARTITIONS_MAX_RESULTS = 1000; private static final Comparator> PARTITION_VALUE_COMPARATOR = lexicographical(String.CASE_INSENSITIVE_ORDER); + private static final Predicate VIEWS_FILTER = table -> VIRTUAL_VIEW.name().equals(table.getTableType()); private final HdfsEnvironment hdfsEnvironment; private final HdfsContext hdfsContext; @@ -441,12 +443,16 @@ public List getAllTables(String databaseName) @Override public synchronized List getTablesWithParameter(String databaseName, String parameterKey, String parameterValue) { - // TODO - throw new UnsupportedOperationException("getTablesWithParameter for GlueHiveMetastore is not implemented"); + return getAllViews(databaseName, table -> parameterValue.equals(firstNonNull(table.getParameters(), ImmutableMap.of()).get(parameterKey))); } @Override public List getAllViews(String databaseName) + { + return getAllViews(databaseName, table -> true); + } + + private List getAllViews(String databaseName, Predicate additionalFilter) { try { List views = getPaginatedResults( @@ -458,7 +464,7 @@ public List getAllViews(String databaseName) stats.getGetTables()) .map(GetTablesResult::getTableList) .flatMap(List::stream) - .filter(table -> VIRTUAL_VIEW.name().equals(table.getTableType())) + .filter(VIEWS_FILTER.and(additionalFilter)) .map(com.amazonaws.services.glue.model.Table::getName) .collect(toImmutableList()); return views; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java index fd4bfbde91c9..78387a4518bf 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractTrinoCatalog.java @@ -17,8 +17,6 @@ import io.trino.filesystem.TrinoFileSystem; import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.HiveMetadata; -import io.trino.plugin.hive.HiveViewNotSupportedException; -import io.trino.plugin.hive.ViewReaderUtil; import io.trino.plugin.iceberg.ColumnIdentity; import io.trino.plugin.iceberg.IcebergMaterializedViewDefinition; import io.trino.plugin.iceberg.IcebergUtil; @@ -48,15 +46,12 @@ import java.util.Map; import java.util.Optional; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.plugin.hive.HiveMetadata.STORAGE_TABLE; import static io.trino.plugin.hive.HiveMetadata.TABLE_COMMENT; import static io.trino.plugin.hive.ViewReaderUtil.ICEBERG_MATERIALIZED_VIEW_COMMENT; import static io.trino.plugin.hive.ViewReaderUtil.PRESTO_VIEW_FLAG; -import static io.trino.plugin.hive.ViewReaderUtil.isHiveOrPrestoView; -import static io.trino.plugin.hive.ViewReaderUtil.isPrestoView; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_FILESYSTEM_ERROR; import static io.trino.plugin.iceberg.IcebergMaterializedViewAdditionalProperties.STORAGE_SCHEMA; import static io.trino.plugin.iceberg.IcebergMaterializedViewAdditionalProperties.getStorageSchema; @@ -74,31 +69,24 @@ public abstract class AbstractTrinoCatalog implements TrinoCatalog { - // Be compatible with views defined by the Hive connector, which can be useful under certain conditions. + public static final String TRINO_CREATED_BY_VALUE = "Trino Iceberg connector"; protected static final String TRINO_CREATED_BY = HiveMetadata.TRINO_CREATED_BY; - protected static final String TRINO_CREATED_BY_VALUE = "Trino Iceberg connector"; - protected static final String PRESTO_VIEW_COMMENT = HiveMetadata.PRESTO_VIEW_COMMENT; - protected static final String PRESTO_VERSION_NAME = HiveMetadata.PRESTO_VERSION_NAME; protected static final String PRESTO_QUERY_ID_NAME = HiveMetadata.PRESTO_QUERY_ID_NAME; - protected static final String PRESTO_VIEW_EXPANDED_TEXT_MARKER = HiveMetadata.PRESTO_VIEW_EXPANDED_TEXT_MARKER; private final CatalogName catalogName; private final TypeManager typeManager; protected final IcebergTableOperationsProvider tableOperationsProvider; - private final String trinoVersion; private final boolean useUniqueTableLocation; protected AbstractTrinoCatalog( CatalogName catalogName, TypeManager typeManager, IcebergTableOperationsProvider tableOperationsProvider, - String trinoVersion, boolean useUniqueTableLocation) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.tableOperationsProvider = requireNonNull(tableOperationsProvider, "tableOperationsProvider is null"); - this.trinoVersion = requireNonNull(trinoVersion, "trinoVersion is null"); this.useUniqueTableLocation = useUniqueTableLocation; } @@ -199,56 +187,6 @@ protected void deleteTableDirectory(TrinoFileSystem fileSystem, SchemaTableName } } - protected Optional getView( - SchemaTableName viewName, - Optional viewOriginalText, - String tableType, - Map tableParameters, - Optional tableOwner) - { - if (!isView(tableType, tableParameters)) { - // Filter out Tables and Materialized Views - return Optional.empty(); - } - - if (!isPrestoView(tableParameters)) { - // Hive views are not compatible - throw new HiveViewNotSupportedException(viewName); - } - - checkArgument(viewOriginalText.isPresent(), "viewOriginalText must be present"); - ConnectorViewDefinition definition = ViewReaderUtil.PrestoViewReader.decodeViewData(viewOriginalText.get()); - // use owner from table metadata if it exists - if (tableOwner.isPresent() && !definition.isRunAsInvoker()) { - definition = new ConnectorViewDefinition( - definition.getOriginalSql(), - definition.getCatalog(), - definition.getSchema(), - definition.getColumns(), - definition.getComment(), - tableOwner, - false); - } - return Optional.of(definition); - } - - private static boolean isView(String tableType, Map tableParameters) - - { - return isHiveOrPrestoView(tableType) && PRESTO_VIEW_COMMENT.equals(tableParameters.get(TABLE_COMMENT)); - } - - protected Map createViewProperties(ConnectorSession session) - { - return ImmutableMap.builder() - .put(PRESTO_VIEW_FLAG, "true") // Ensures compatibility with views created by the Hive connector - .put(TRINO_CREATED_BY, TRINO_CREATED_BY_VALUE) - .put(PRESTO_VERSION_NAME, trinoVersion) - .put(PRESTO_QUERY_ID_NAME, session.getQueryId()) - .put(TABLE_COMMENT, PRESTO_VIEW_COMMENT) - .buildOrThrow(); - } - protected SchemaTableName createMaterializedViewStorageTable(ConnectorSession session, SchemaTableName viewName, ConnectorMaterializedViewDefinition definition) { // Generate a storage table name and create a storage table. The properties in the definition are table properties for the diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java index 94a5b4c4142c..b023a42b3b44 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java @@ -38,6 +38,7 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.SchemaAlreadyExistsException; +import io.trino.plugin.hive.TrinoViewUtil; import io.trino.plugin.hive.ViewAlreadyExistsException; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; import io.trino.plugin.iceberg.catalog.AbstractTrinoCatalog; @@ -83,6 +84,7 @@ import static io.trino.plugin.hive.HiveErrorCode.HIVE_DATABASE_LOCATION_ERROR; import static io.trino.plugin.hive.HiveErrorCode.HIVE_METASTORE_ERROR; import static io.trino.plugin.hive.HiveMetadata.STORAGE_TABLE; +import static io.trino.plugin.hive.TrinoViewUtil.createViewProperties; import static io.trino.plugin.hive.ViewReaderUtil.encodeViewData; import static io.trino.plugin.hive.ViewReaderUtil.isPrestoView; import static io.trino.plugin.hive.ViewReaderUtil.isTrinoMaterializedView; @@ -119,6 +121,7 @@ public class TrinoGlueCatalog { private static final Logger LOG = Logger.get(TrinoGlueCatalog.class); + private final String trinoVersion; private final TrinoFileSystemFactory fileSystemFactory; private final Optional defaultSchemaLocation; private final AWSGlueAsync glueClient; @@ -139,7 +142,8 @@ public TrinoGlueCatalog( Optional defaultSchemaLocation, boolean useUniqueTableLocation) { - super(catalogName, typeManager, tableOperationsProvider, trinoVersion, useUniqueTableLocation); + super(catalogName, typeManager, tableOperationsProvider, useUniqueTableLocation); + this.trinoVersion = requireNonNull(trinoVersion, "trinoVersion is null"); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.glueClient = requireNonNull(glueClient, "glueClient is null"); this.stats = requireNonNull(stats, "stats is null"); @@ -460,7 +464,7 @@ else if (isPrestoView(parameters) && !viewCache.containsKey(schemaTableName)) { } try { - getView(schemaTableName, + TrinoViewUtil.getView(schemaTableName, Optional.ofNullable(table.getViewOriginalText()), table.getTableType(), parameters, @@ -537,7 +541,11 @@ public void setTablePrincipal(ConnectorSession session, SchemaTableName schemaTa public void createView(ConnectorSession session, SchemaTableName schemaViewName, ConnectorViewDefinition definition, boolean replace) { // If a view is created between listing the existing view and calling createTable, retry - TableInput viewTableInput = getViewTableInput(schemaViewName.getTableName(), encodeViewData(definition), session.getUser(), createViewProperties(session)); + TableInput viewTableInput = getViewTableInput( + schemaViewName.getTableName(), + encodeViewData(definition), + session.getUser(), + createViewProperties(session, trinoVersion, TRINO_CREATED_BY_VALUE)); Failsafe.with(new RetryPolicy<>() .withMaxRetries(3) .withDelay(Duration.ofMillis(100)) @@ -584,7 +592,7 @@ public void renameView(ConnectorSession session, SchemaTableName source, SchemaT target.getTableName(), existingView.getViewOriginalText(), existingView.getOwner(), - createViewProperties(session)); + createViewProperties(session, trinoVersion, TRINO_CREATED_BY_VALUE)); CreateTableRequest createTableRequest = new CreateTableRequest() .withDatabaseName(target.getSchemaName()) .withTableInput(viewTableInput); @@ -678,7 +686,7 @@ public Optional getView(ConnectorSession session, Schem return Optional.empty(); } com.amazonaws.services.glue.model.Table viewDefinition = table.get(); - return getView( + return TrinoViewUtil.getView( viewName, Optional.ofNullable(viewDefinition.getViewOriginalText()), viewDefinition.getTableType(), @@ -724,7 +732,11 @@ public void updateViewColumnComment(ConnectorSession session, SchemaTableName vi private void updateView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition newDefinition) { - TableInput viewTableInput = getViewTableInput(viewName.getTableName(), encodeViewData(newDefinition), session.getUser(), createViewProperties(session)); + TableInput viewTableInput = getViewTableInput( + viewName.getTableName(), + encodeViewData(newDefinition), + session.getUser(), + createViewProperties(session, trinoVersion, TRINO_CREATED_BY_VALUE)); try { stats.getUpdateTable().call(() -> diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java index eaf6e969b543..45de0751e5f9 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java @@ -19,8 +19,8 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.HiveSchemaProperties; -import io.trino.plugin.hive.TableAlreadyExistsException; -import io.trino.plugin.hive.ViewAlreadyExistsException; +import io.trino.plugin.hive.TrinoViewHiveMetastore; +import io.trino.plugin.hive.TrinoViewUtil; import io.trino.plugin.hive.metastore.Column; import io.trino.plugin.hive.metastore.Database; import io.trino.plugin.hive.metastore.HivePrincipal; @@ -58,7 +58,6 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -70,7 +69,6 @@ import static io.trino.plugin.hive.ViewReaderUtil.ICEBERG_MATERIALIZED_VIEW_COMMENT; import static io.trino.plugin.hive.ViewReaderUtil.encodeViewData; import static io.trino.plugin.hive.ViewReaderUtil.isHiveOrPrestoView; -import static io.trino.plugin.hive.ViewReaderUtil.isPrestoView; import static io.trino.plugin.hive.ViewReaderUtil.isTrinoMaterializedView; import static io.trino.plugin.hive.metastore.MetastoreUtil.buildInitialPrivilegeSet; import static io.trino.plugin.hive.metastore.PrincipalPrivileges.NO_PRIVILEGES; @@ -109,6 +107,7 @@ public class TrinoHiveCatalog public static final String DEPENDS_ON_TABLES = "dependsOnTables"; private final CachingHiveMetastore metastore; + private final TrinoViewHiveMetastore trinoViewHiveMetastore; private final TrinoFileSystemFactory fileSystemFactory; private final boolean isUsingSystemSecurity; private final boolean deleteSchemaLocationsFallback; @@ -118,16 +117,17 @@ public class TrinoHiveCatalog public TrinoHiveCatalog( CatalogName catalogName, CachingHiveMetastore metastore, + TrinoViewHiveMetastore trinoViewHiveMetastore, TrinoFileSystemFactory fileSystemFactory, TypeManager typeManager, IcebergTableOperationsProvider tableOperationsProvider, - String trinoVersion, boolean useUniqueTableLocation, boolean isUsingSystemSecurity, boolean deleteSchemaLocationsFallback) { - super(catalogName, typeManager, tableOperationsProvider, trinoVersion, useUniqueTableLocation); + super(catalogName, typeManager, tableOperationsProvider, useUniqueTableLocation); this.metastore = requireNonNull(metastore, "metastore is null"); + this.trinoViewHiveMetastore = requireNonNull(trinoViewHiveMetastore, "trinoViewHiveMetastore is null"); this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.isUsingSystemSecurity = isUsingSystemSecurity; this.deleteSchemaLocationsFallback = deleteSchemaLocationsFallback; @@ -348,7 +348,7 @@ public void updateViewComment(ConnectorSession session, SchemaTableName viewName io.trino.plugin.hive.metastore.Table view = metastore.getTable(viewName.getSchemaName(), viewName.getTableName()) .orElseThrow(() -> new ViewNotFoundException(viewName)); - ConnectorViewDefinition definition = getView(viewName, view.getViewOriginalText(), view.getTableType(), view.getParameters(), view.getOwner()) + ConnectorViewDefinition definition = TrinoViewUtil.getView(viewName, view.getViewOriginalText(), view.getTableType(), view.getParameters(), view.getOwner()) .orElseThrow(() -> new ViewNotFoundException(viewName)); ConnectorViewDefinition newDefinition = new ConnectorViewDefinition( definition.getOriginalSql(), @@ -368,7 +368,7 @@ public void updateViewColumnComment(ConnectorSession session, SchemaTableName vi io.trino.plugin.hive.metastore.Table view = metastore.getTable(viewName.getSchemaName(), viewName.getTableName()) .orElseThrow(() -> new ViewNotFoundException(viewName)); - ConnectorViewDefinition definition = getView(viewName, view.getViewOriginalText(), view.getTableType(), view.getParameters(), view.getOwner()) + ConnectorViewDefinition definition = TrinoViewUtil.getView(viewName, view.getViewOriginalText(), view.getTableType(), view.getParameters(), view.getOwner()) .orElseThrow(() -> new ViewNotFoundException(viewName)); ConnectorViewDefinition newDefinition = new ConnectorViewDefinition( definition.getOriginalSql(), @@ -421,43 +421,7 @@ public void setTablePrincipal(ConnectorSession session, SchemaTableName schemaTa @Override public void createView(ConnectorSession session, SchemaTableName schemaViewName, ConnectorViewDefinition definition, boolean replace) { - if (isUsingSystemSecurity) { - definition = definition.withoutOwner(); - } - - io.trino.plugin.hive.metastore.Table.Builder tableBuilder = io.trino.plugin.hive.metastore.Table.builder() - .setDatabaseName(schemaViewName.getSchemaName()) - .setTableName(schemaViewName.getTableName()) - .setOwner(isUsingSystemSecurity ? Optional.empty() : Optional.of(session.getUser())) - .setTableType(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW.name()) - .setDataColumns(ImmutableList.of(new Column("dummy", HIVE_STRING, Optional.empty()))) - .setPartitionColumns(ImmutableList.of()) - .setParameters(createViewProperties(session)) - .setViewOriginalText(Optional.of(encodeViewData(definition))) - .setViewExpandedText(Optional.of(PRESTO_VIEW_EXPANDED_TEXT_MARKER)); - - tableBuilder.getStorageBuilder() - .setStorageFormat(VIEW_STORAGE_FORMAT) - .setLocation(""); - io.trino.plugin.hive.metastore.Table table = tableBuilder.build(); - PrincipalPrivileges principalPrivileges = isUsingSystemSecurity ? NO_PRIVILEGES : buildInitialPrivilegeSet(session.getUser()); - - Optional existing = metastore.getTable(schemaViewName.getSchemaName(), schemaViewName.getTableName()); - if (existing.isPresent()) { - if (!replace || !isPrestoView(existing.get())) { - throw new ViewAlreadyExistsException(schemaViewName); - } - - metastore.replaceTable(schemaViewName.getSchemaName(), schemaViewName.getTableName(), table, principalPrivileges); - return; - } - - try { - metastore.createTable(table, principalPrivileges); - } - catch (TableAlreadyExistsException e) { - throw new ViewAlreadyExistsException(e.getTableName()); - } + trinoViewHiveMetastore.createView(session, schemaViewName, definition, replace); } @Override @@ -477,46 +441,19 @@ public void setViewPrincipal(ConnectorSession session, SchemaTableName schemaVie @Override public void dropView(ConnectorSession session, SchemaTableName schemaViewName) { - if (getView(session, schemaViewName).isEmpty()) { - throw new ViewNotFoundException(schemaViewName); - } - - try { - metastore.dropTable(schemaViewName.getSchemaName(), schemaViewName.getTableName(), true); - } - catch (TableNotFoundException e) { - throw new ViewNotFoundException(e.getTableName()); - } + trinoViewHiveMetastore.dropView(schemaViewName); } @Override public List listViews(ConnectorSession session, Optional namespace) { - return listNamespaces(session, namespace).stream() - .flatMap(this::listViews) - .collect(toImmutableList()); - } - - private Stream listViews(String schema) - { - // Filter on PRESTO_VIEW_COMMENT to distinguish from materialized views - return metastore.getTablesWithParameter(schema, TABLE_COMMENT, PRESTO_VIEW_COMMENT).stream() - .map(table -> new SchemaTableName(schema, table)); + return trinoViewHiveMetastore.listViews(namespace); } @Override public Optional getView(ConnectorSession session, SchemaTableName viewName) { - if (isHiveSystemSchema(viewName.getSchemaName())) { - return Optional.empty(); - } - return metastore.getTable(viewName.getSchemaName(), viewName.getTableName()) - .flatMap(view -> getView( - viewName, - view.getViewOriginalText(), - view.getTableType(), - view.getParameters(), - view.getOwner())); + return trinoViewHiveMetastore.getView(viewName); } @Override diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java index 162ebe1bcf33..c86f633f6704 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java @@ -16,7 +16,9 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.NodeVersion; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastoreFactory; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergSecurityConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; @@ -31,6 +33,7 @@ import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.memoizeMetastore; import static io.trino.plugin.iceberg.IcebergSecurityConfig.IcebergSecurity.SYSTEM; +import static io.trino.plugin.iceberg.catalog.AbstractTrinoCatalog.TRINO_CREATED_BY_VALUE; import static java.util.Objects.requireNonNull; public class TrinoHiveCatalogFactory @@ -71,13 +74,14 @@ public TrinoHiveCatalogFactory( @Override public TrinoCatalog create(ConnectorIdentity identity) { + CachingHiveMetastore metastore = memoizeMetastore(metastoreFactory.createMetastore(Optional.of(identity)), 1000); return new TrinoHiveCatalog( catalogName, - memoizeMetastore(metastoreFactory.createMetastore(Optional.of(identity)), 1000), + metastore, + new TrinoViewHiveMetastore(metastore, isUsingSystemSecurity, trinoVersion, TRINO_CREATED_BY_VALUE), fileSystemFactory, typeManager, tableOperationsProvider, - trinoVersion, isUniqueTableLocation, isUsingSystemSecurity, deleteSchemaLocationsFallback); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java index 74a35a398f56..db6f73eb386a 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMergeAppend.java @@ -17,7 +17,9 @@ import io.trino.filesystem.hdfs.HdfsFileSystemFactory; import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.NodeVersion; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastore; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.file.FileHiveMetastore; import io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; @@ -61,13 +63,14 @@ protected QueryRunner createQueryRunner() throws Exception .setMetastoreUser("test")); TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT); tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory); + CachingHiveMetastore cachingHiveMetastore = memoizeMetastore(metastore, 1000); trinoCatalog = new TrinoHiveCatalog( new CatalogName("catalog"), - memoizeMetastore(metastore, 1000), + cachingHiveMetastore, + new TrinoViewHiveMetastore(cachingHiveMetastore, false, "trino-version", "test"), fileSystemFactory, new TestingTypeManager(), tableOperationsProvider, - "trino-version", false, false, false); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java index a7256aed5846..063fc4404db2 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergOrcMetricsCollection.java @@ -18,8 +18,10 @@ import io.trino.filesystem.hdfs.HdfsFileSystemFactory; import io.trino.plugin.base.CatalogName; import io.trino.plugin.hive.NodeVersion; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastoreConfig; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.file.FileHiveMetastore; import io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; @@ -91,13 +93,14 @@ protected QueryRunner createQueryRunner() .setMetastoreUser("test")); TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT); tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory); + CachingHiveMetastore cachingHiveMetastore = memoizeMetastore(metastore, 1000); trinoCatalog = new TrinoHiveCatalog( new CatalogName("catalog"), - memoizeMetastore(metastore, 1000), + cachingHiveMetastore, + new TrinoViewHiveMetastore(cachingHiveMetastore, false, "trino-version", "test"), fileSystemFactory, new TestingTypeManager(), tableOperationsProvider, - "trino-version", false, false, false); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java index ca8b9d1134b9..bb8c9651cd30 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java @@ -21,7 +21,9 @@ import io.trino.filesystem.hdfs.HdfsFileSystemFactory; import io.trino.hdfs.HdfsContext; import io.trino.plugin.base.CatalogName; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastore; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.file.FileMetastoreTableOperationsProvider; import io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalog; @@ -84,13 +86,14 @@ protected QueryRunner createQueryRunner() this.metastoreDir = new File(tempDir, "iceberg_data"); HiveMetastore metastore = createTestingFileHiveMetastore(metastoreDir); TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT); + CachingHiveMetastore cachingHiveMetastore = memoizeMetastore(metastore, 1000); this.catalog = new TrinoHiveCatalog( new CatalogName("hive"), - memoizeMetastore(metastore, 1000), + cachingHiveMetastore, + new TrinoViewHiveMetastore(cachingHiveMetastore, false, "trino-version", "test"), fileSystemFactory, new TestingTypeManager(), new FileMetastoreTableOperationsProvider(fileSystemFactory), - "test", false, false, false); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java index 75bf3b869932..64f74e0053d0 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java @@ -26,6 +26,7 @@ import io.trino.hdfs.HdfsEnvironment; import io.trino.hdfs.authentication.NoHdfsAuthentication; import io.trino.plugin.base.CatalogName; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastore; import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider; @@ -70,6 +71,7 @@ import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; +import static io.trino.plugin.hive.metastore.cache.CachingHiveMetastore.memoizeMetastore; import static io.trino.plugin.hive.metastore.file.FileHiveMetastore.createTestingFileHiveMetastore; import static io.trino.plugin.iceberg.IcebergUtil.loadIcebergTable; import static io.trino.testing.TestingConnectorSession.SESSION; @@ -539,13 +541,14 @@ private BaseTable loadTable(String tableName) { TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT); IcebergTableOperationsProvider tableOperationsProvider = new FileMetastoreTableOperationsProvider(fileSystemFactory); + CachingHiveMetastore cachingHiveMetastore = memoizeMetastore(metastore, 1000); TrinoCatalog catalog = new TrinoHiveCatalog( new CatalogName("hive"), - CachingHiveMetastore.memoizeMetastore(metastore, 1000), + cachingHiveMetastore, + new TrinoViewHiveMetastore(cachingHiveMetastore, false, "trino-version", "test"), fileSystemFactory, new TestingTypeManager(), tableOperationsProvider, - "test", false, false, false); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java index 91c90bb6dc00..42e5ca9b4dcd 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java @@ -16,7 +16,9 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.hdfs.HdfsFileSystemFactory; import io.trino.plugin.base.CatalogName; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.metastore.HiveMetastore; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.iceberg.catalog.BaseTrinoCatalogTest; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.hms.TrinoHiveCatalog; @@ -62,13 +64,14 @@ public void tearDown() protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) { TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT); + CachingHiveMetastore cachingHiveMetastore = memoizeMetastore(metastore, 1000); return new TrinoHiveCatalog( new CatalogName("catalog"), - memoizeMetastore(metastore, 1000), + cachingHiveMetastore, + new TrinoViewHiveMetastore(cachingHiveMetastore, false, "trino-version", "test"), fileSystemFactory, new TestingTypeManager(), new FileMetastoreTableOperationsProvider(fileSystemFactory), - "trino-version", useUniqueTableLocations, false, false); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java index 3956893259bd..5034a93d7818 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java @@ -23,7 +23,9 @@ import io.trino.hdfs.HdfsEnvironment; import io.trino.hdfs.authentication.NoHdfsAuthentication; import io.trino.plugin.base.CatalogName; +import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastore; import io.trino.plugin.hive.metastore.thrift.ThriftMetastore; import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreConfig; @@ -98,9 +100,11 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) .setMetastoreTimeout(new Duration(1, MINUTES))) .metastoreClient(dataLake.getHiveHadoop().getHiveMetastoreEndpoint()) .build(); + CachingHiveMetastore metastore = memoizeMetastore(new BridgingHiveMetastore(thriftMetastore), 1000); return new TrinoHiveCatalog( new CatalogName("catalog"), - memoizeMetastore(new BridgingHiveMetastore(thriftMetastore), 1000), + metastore, + new TrinoViewHiveMetastore(metastore, false, "trino-version", "Test"), fileSystemFactory, new TestingTypeManager(), new HiveMetastoreTableOperationsProvider(fileSystemFactory, new ThriftMetastoreFactory() @@ -118,7 +122,6 @@ public ThriftMetastore createMetastore(Optional identity) return thriftMetastore; } }), - "trino-version", useUniqueTableLocations, false, false); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeCompatibility.java index 673f3a2051f8..85614f96b486 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestHiveAndDeltaLakeCompatibility.java @@ -17,6 +17,7 @@ import io.trino.testng.services.Flaky; import org.testng.annotations.Test; +import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertThat; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; @@ -44,7 +45,8 @@ public void testInformationSchemaColumnsOnPresenceOfHiveView() onTrino().executeQuery("CREATE VIEW " + hiveViewQualifiedName + " AS SELECT 1 AS col_one"); try { - assertThat(onTrino().executeQuery(format("SELECT table_name FROM delta.information_schema.columns WHERE table_schema = '%s'", schemaName))).hasNoRows(); + assertThat(onTrino().executeQuery(format("SELECT table_name FROM delta.information_schema.columns WHERE table_schema = '%s'", schemaName))) + .containsOnly(row(hiveViewName)); } finally { onTrino().executeQuery("DROP VIEW IF EXISTS " + hiveViewQualifiedName);