From 4c03bd0002ca2f3ca2b8ce832e2d29b4a70b84c9 Mon Sep 17 00:00:00 2001 From: Bin Fan Date: Tue, 11 Jul 2023 21:44:18 -0700 Subject: [PATCH] Remove deprecated Alluxio metastore --- .github/workflows/ci.yml | 4 - .../plugin/deltalake/TestDeltaLakePlugin.java | 23 - .../bin/run_hive_alluxio_tests.sh | 70 --- .../conf/alluxio-docker.yml | 37 -- plugin/trino-hive-hadoop2/pom.xml | 17 - .../plugin/hive/TestHiveAlluxioMetastore.java | 401 -------------- .../io/trino/plugin/hive/TestHivePlugin.java | 14 - plugin/trino-hive/pom.xml | 5 - .../hive/metastore/HiveMetastoreModule.java | 24 - .../alluxio/AlluxioHiveMetastore.java | 504 ------------------ .../alluxio/AlluxioHiveMetastoreConfig.java | 38 -- .../alluxio/AlluxioHiveMetastoreFactory.java | 48 -- .../alluxio/AlluxioMetastoreModule.java | 69 --- .../hive/metastore/alluxio/ProtoUtils.java | 300 ----------- .../trino/plugin/hive/AbstractTestHive.java | 4 +- .../TestAlluxioHiveMetastoreConfig.java | 44 -- .../metastore/alluxio/TestProtoUtils.java | 197 ------- .../TestingAlluxioMetastoreObjects.java | 153 ------ .../io/trino/plugin/hudi/TestHudiPlugin.java | 19 - .../trino-session-property-managers/pom.xml | 12 - testing/trino-faulttolerant-tests/pom.xml | 12 - testing/trino-tests/pom.xml | 10 - 22 files changed, 2 insertions(+), 2003 deletions(-) delete mode 100755 plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh delete mode 100644 plugin/trino-hive-hadoop2/conf/alluxio-docker.yml delete mode 100644 plugin/trino-hive-hadoop2/src/test/java/io/trino/plugin/hive/TestHiveAlluxioMetastore.java delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastore.java delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastoreConfig.java delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastoreFactory.java delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioMetastoreModule.java delete mode 100644 plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/ProtoUtils.java delete mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestAlluxioHiveMetastoreConfig.java delete mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestProtoUtils.java delete mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestingAlluxioMetastoreObjects.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index feb74e06717d..c481076b0c0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -381,10 +381,6 @@ jobs: source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && plugin/trino-hive-hadoop2/bin/run_hive_adl_tests.sh fi - - name: Run Hive Alluxio Tests - run: | - source plugin/trino-hive-hadoop2/conf/hive-tests-${{ matrix.config }}.sh && - plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh - name: Upload test results uses: actions/upload-artifact@v3 # Upload all test reports only on failure, because the artifacts are large diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java index abcfefd6ce9b..f1d4e4ed63df 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePlugin.java @@ -106,29 +106,6 @@ public void testGlueMetastore() .hasMessageContaining("Error: Configuration property 'hive.metastore.uri' was not used"); } - /** - * Verify the Alluxio metastore is not supported for Delta. Delta connector extends Hive connector and Hive connector supports Alluxio metastore. - * We explicitly disallow Alluxio metastore use with Delta. - */ - @Test - public void testAlluxioMetastore() - { - ConnectorFactory factory = getConnectorFactory(); - assertThatThrownBy(() -> factory.create( - "test", - ImmutableMap.of("hive.metastore", "alluxio"), - new TestingConnectorContext())) - .hasMessageMatching("(?s)Unable to create injector, see the following errors:.*" + - "Explicit bindings are required and HiveMetastoreFactory .* is not explicitly bound.*"); - - assertThatThrownBy(() -> factory.create( - "test", - ImmutableMap.of("hive.metastore", "alluxio-deprecated"), - new TestingConnectorContext())) - .hasMessageMatching("(?s)Unable to create injector, see the following errors:.*" + - "Explicit bindings are required and HiveMetastoreFactory .* is not explicitly bound.*"); - } - @Test public void testNoCaching() { diff --git a/plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh b/plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh deleted file mode 100755 index eff9a3aafb56..000000000000 --- a/plugin/trino-hive-hadoop2/bin/run_hive_alluxio_tests.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -x - -. "${BASH_SOURCE%/*}/common.sh" - -abort_if_not_gib_impacted - -export ALLUXIO_BASE_IMAGE="alluxio/alluxio" -export ALLUXIO_IMAGE_TAG="2.1.2" - -ALLUXIO_DOCKER_COMPOSE_LOCATION="${INTEGRATION_TESTS_ROOT}/conf/alluxio-docker.yml" - -function check_alluxio() { - run_in_alluxio alluxio fsadmin report -} - -function run_in_alluxio() { - docker exec -e ALLUXIO_JAVA_OPTS=" -Dalluxio.master.hostname=localhost" \ - "$(alluxio_master_container)" $@ -} - -# Arguments: -# $1: container name -function get_alluxio_container() { - docker-compose -f "${ALLUXIO_DOCKER_COMPOSE_LOCATION}" ps -q "$1" | grep . -} - -function alluxio_master_container() { - get_alluxio_container alluxio-master -} - -function main () { - cleanup_docker_containers "${DOCKER_COMPOSE_LOCATION}" "${ALLUXIO_DOCKER_COMPOSE_LOCATION}" - start_docker_containers "${DOCKER_COMPOSE_LOCATION}" "${ALLUXIO_DOCKER_COMPOSE_LOCATION}" - retry check_hadoop - retry check_alluxio & # data can be generated while we wait for alluxio to start - - # generate test data - exec_in_hadoop_master_container sudo -Eu hdfs hdfs dfs -mkdir /alluxio - exec_in_hadoop_master_container sudo -Eu hdfs hdfs dfs -chmod 777 /alluxio - exec_in_hadoop_master_container sudo -Eu hive beeline -u jdbc:hive2://localhost:10000/default -n hive -f /docker/sql/create-test.sql - - # Alluxio currently doesn't support views - exec_in_hadoop_master_container sudo -Eu hive beeline -u jdbc:hive2://localhost:10000/default -n hive -e 'DROP VIEW trino_test_view;' - - stop_unnecessary_hadoop_services - - wait # make sure alluxio has started - - run_in_alluxio alluxio table attachdb hive thrift://hadoop-master:9083 default - run_in_alluxio alluxio table ls default - - # run product tests - pushd ${PROJECT_ROOT} - set +e - ./mvnw ${MAVEN_TEST:--B} -pl :trino-hive-hadoop2 test -P test-hive-hadoop2-alluxio \ - -Dtest.alluxio.host=localhost \ - -Dtest.alluxio.port=19998 \ - -DHADOOP_USER_NAME=hive - EXIT_CODE=$? - set -e - popd - - cleanup_docker_containers "${DOCKER_COMPOSE_LOCATION}" "${ALLUXIO_DOCKER_COMPOSE_LOCATION}" - - exit ${EXIT_CODE} -} - -main diff --git a/plugin/trino-hive-hadoop2/conf/alluxio-docker.yml b/plugin/trino-hive-hadoop2/conf/alluxio-docker.yml deleted file mode 100644 index dc3c35a0c0c3..000000000000 --- a/plugin/trino-hive-hadoop2/conf/alluxio-docker.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: '2' -services: - alluxio-master: - hostname: alluxio-master - image: '${ALLUXIO_BASE_IMAGE}:${ALLUXIO_IMAGE_TAG}' - command: master - environment: - ALLUXIO_MASTER_JAVA_OPTS: > - -Dalluxio.master.hostname=localhost - -Dalluxio.master.mount.table.root.ufs=hdfs://hadoop-master:9000/alluxio - -Dalluxio.security.authorization.permission.enabled=false - -Dalluxio.master.security.impersonation.presto.users=* - ports: - - '19200:19200' # Master Embedded Journal - - '19999:19999' # Master UI - - '19998:19998' # Master RPC - - '20003:20003' # Job Master Embedded Journal - - '20001:20001' # Job Master RPC - - '20002:20002' # Job Master HTTP - volumes: - - ./files:/docker/files:ro - alluxio-worker: - hostname: alluxio-worker - image: '${ALLUXIO_BASE_IMAGE}:${ALLUXIO_IMAGE_TAG}' - command: worker - shm_size: '500mb' - environment: - ALLUXIO_WORKER_JAVA_OPTS: > - -Dalluxio.worker.hostname=localhost - -Dalluxio.master.hostname=alluxio-master - -Dalluxio.worker.memory.size=500m - ports: - - '29999:29999' # Worker RPC - - '30000:30000' # Worker HTTP - - '30001:30001' # Job Worker RPC - - '30002:30002' # Job Worker Data - - '30003:30003' # Job Worker HTTP diff --git a/plugin/trino-hive-hadoop2/pom.xml b/plugin/trino-hive-hadoop2/pom.xml index 5353dc228384..cbdd444e9d63 100644 --- a/plugin/trino-hive-hadoop2/pom.xml +++ b/plugin/trino-hive-hadoop2/pom.xml @@ -213,7 +213,6 @@ **/TestHive.java - **/TestHiveAlluxioMetastore.java **/TestHiveThriftMetastoreWithS3.java **/TestHiveFileSystemS3.java **/TestHiveFileSystemS3SelectPushdown.java @@ -347,21 +346,5 @@ - - test-hive-hadoop2-alluxio - - - - org.apache.maven.plugins - maven-surefire-plugin - - - **/TestHiveAlluxioMetastore.java - - - - - - diff --git a/plugin/trino-hive-hadoop2/src/test/java/io/trino/plugin/hive/TestHiveAlluxioMetastore.java b/plugin/trino-hive-hadoop2/src/test/java/io/trino/plugin/hive/TestHiveAlluxioMetastore.java deleted file mode 100644 index 02cd4791b5cd..000000000000 --- a/plugin/trino-hive-hadoop2/src/test/java/io/trino/plugin/hive/TestHiveAlluxioMetastore.java +++ /dev/null @@ -1,401 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive; - -import alluxio.client.table.TableMasterClient; -import alluxio.conf.PropertyKey; -import io.trino.plugin.hive.metastore.HiveMetastoreConfig; -import io.trino.plugin.hive.metastore.alluxio.AlluxioHiveMetastore; -import io.trino.plugin.hive.metastore.alluxio.AlluxioHiveMetastoreConfig; -import io.trino.plugin.hive.metastore.alluxio.AlluxioMetastoreModule; -import org.testng.SkipException; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Parameters; -import org.testng.annotations.Test; - -import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; - -public class TestHiveAlluxioMetastore - extends AbstractTestHive -{ - @Parameters({"test.alluxio.host", "test.alluxio.port"}) - @BeforeClass - public void setup(String host, String port) - { - System.setProperty(PropertyKey.Name.SECURITY_LOGIN_USERNAME, "presto"); - System.setProperty(PropertyKey.Name.MASTER_HOSTNAME, host); - HiveConfig hiveConfig = new HiveConfig() - .setParquetTimeZone("UTC") - .setRcfileTimeZone("UTC"); - - AlluxioHiveMetastoreConfig alluxioConfig = new AlluxioHiveMetastoreConfig(); - alluxioConfig.setMasterAddress(host + ":" + port); - TableMasterClient client = AlluxioMetastoreModule.createCatalogMasterClient(alluxioConfig); - setup("default", hiveConfig, new AlluxioHiveMetastore(client, new HiveMetastoreConfig()), HDFS_ENVIRONMENT); - } - - @Override - public void testBucketSortedTables() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testBucketedTableEvolution() - { - // Alluxio metastore does not support create/insert/update operations - } - - @Override - public void testBucketedSortedTableEvolution() - { - // Alluxio metastore does not support create/insert/update operations - } - - @Override - public void testBucketedTableValidation() - throws Exception - { - // Alluxio metastore does not support create operations - } - - @Override - public void testBucketedTableEvolutionWithDifferentReadBucketCount() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testEmptyOrcFile() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testPerTransactionDirectoryListerCache() - { - // Alluxio metastore does not support create operations - } - - // specifically disable so that expected exception on the superclass don't fail this test - @Override - @Test(enabled = false) - public void testEmptyRcBinaryFile() - { - // Alluxio metastore does not support create operations - } - - // specifically disable so that expected exception on the superclass don't fail this test - @Override - @Test(enabled = false) - public void testEmptyRcTextFile() - { - // Alluxio metastore does not support create operations - } - - // specifically disable so that expected exception on the superclass don't fail this test - @Override - @Test(enabled = false) - public void testEmptySequenceFile() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testEmptyTableCreation() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testEmptyTextFile() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testGetPartitions() - { - // Alluxio metastore treats null comment as empty comment - } - - @Override - public void testGetPartitionsWithBindings() - { - // Alluxio metastore treats null comment as empty comment - } - - @Override - public void testGetPartitionsWithFilter() - { - // Alluxio metastore returns incorrect results - } - - @Override - public void testHideDeltaLakeTables() - { - // Alluxio metastore does not support create operations - throw new SkipException("not supported"); - } - - @Override - public void testDisallowQueryingOfIcebergTables() - { - // Alluxio metastore does not support create operations - throw new SkipException("not supported"); - } - - @Override - public void testIllegalStorageFormatDuringTableScan() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testInsert() - { - // Alluxio metastore does not support insert/update operations - } - - @Override - public void testInsertIntoExistingPartition() - { - // Alluxio metastore does not support insert/update operations - } - - @Override - public void testInsertIntoExistingPartitionEmptyStatistics() - { - // Alluxio metastore does not support insert/update operations - } - - @Override - public void testInsertIntoNewPartition() - { - // Alluxio metastore does not support insert/update operations - } - - @Override - public void testInsertOverwriteUnpartitioned() - { - // Alluxio metastore does not support insert/update operations - } - - @Override - public void testInsertUnsupportedWriteType() - { - // Alluxio metastore does not support insert/update operations - } - - @Override - public void testMetadataDelete() - { - // Alluxio metastore does not support create/delete operations - } - - @Override - public void testMismatchSchemaTable() - { - // Alluxio metastore does not support create/delete operations - } - - @Override - public void testPartitionStatisticsSampling() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testApplyProjection() - { - // Alluxio metastore does not support create/delete operations - } - - @Override - public void testApplyRedirection() - { - // Alluxio metastore does not support create/delete operations - } - - @Override - public void testMaterializedViewMetadata() - { - // Alluxio metastore does not support create/delete operations - } - - @Override - public void testOrcPageSourceMetrics() - { - // Alluxio metastore does not support create/insert/delete operations - } - - @Override - public void testParquetPageSourceMetrics() - { - // Alluxio metastore does not support create/insert/delete operations - } - - @Override - public void testPreferredInsertLayout() - { - // Alluxio metastore does not support insert layout operations - } - - @Override - public void testInsertBucketedTableLayout() - { - // Alluxio metastore does not support insert layout operations - } - - @Override - public void testInsertPartitionedBucketedTableLayout() - { - // Alluxio metastore does not support insert layout operations - } - - @Override - public void testStorePartitionWithStatistics() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testRenameTable() - { - // Alluxio metastore does not support update operations - } - - @Override - public void testTableCreation() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testTableCreationWithTrailingSpaceInLocation() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testTableCreationIgnoreExisting() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testTableCreationRollback() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testTransactionDeleteInsert() - { - // Alluxio metastore does not support insert/update/delete operations - } - - @Override - public void testTypesOrc() - throws Exception - { - super.testTypesOrc(); - } - - @Override - public void testUpdateBasicPartitionStatistics() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testUpdateBasicTableStatistics() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testUpdatePartitionColumnStatistics() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testUpdatePartitionColumnStatisticsEmptyOptionalFields() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testInputInfoWhenTableIsPartitioned() - { - // Alluxio metastore does not support create/delete operations - } - - @Override - public void testInputInfoWhenTableIsNotPartitioned() - { - // Alluxio metastore does not support create/delete operations - } - - @Override - public void testInputInfoWithParquetTableFormat() - { - // Alluxio metastore does not support create/delete operations - } - - @Override - public void testUpdateTableColumnStatistics() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testUpdateTableColumnStatisticsEmptyOptionalFields() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testViewCreation() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testNewDirectoryPermissions() - { - // Alluxio metastore does not support create operations - } - - @Override - public void testInsertBucketedTransactionalTableLayout() - throws Exception - { - // Alluxio metastore does not support insert/update/delete operations - } - - @Override - public void testInsertPartitionedBucketedTransactionalTableLayout() - throws Exception - { - // Alluxio metastore does not support insert/update/delete operations - } - - @Override - public void testCreateEmptyTableShouldNotCreateStagingDirectory() - { - // Alluxio metastore does not support create operations - } -} diff --git a/plugin/trino-hive-hadoop2/src/test/java/io/trino/plugin/hive/TestHivePlugin.java b/plugin/trino-hive-hadoop2/src/test/java/io/trino/plugin/hive/TestHivePlugin.java index 3717dbfdfa21..b02ae6e60bba 100644 --- a/plugin/trino-hive-hadoop2/src/test/java/io/trino/plugin/hive/TestHivePlugin.java +++ b/plugin/trino-hive-hadoop2/src/test/java/io/trino/plugin/hive/TestHivePlugin.java @@ -127,20 +127,6 @@ public void testGlueMetastore() .hasMessageContaining("Error: Configuration property 'hive.metastore.uri' was not used"); } - @Test - public void testAlluxioMetastore() - { - ConnectorFactory factory = getHiveConnectorFactory(); - - factory.create( - "test", - ImmutableMap.of( - "hive.metastore", "alluxio-deprecated", - "hive.metastore.alluxio.master.address", "dummy:1234"), - new TestingConnectorContext()) - .shutdown(); - } - @Test public void testRecordingMetastore() { diff --git a/plugin/trino-hive/pom.xml b/plugin/trino-hive/pom.xml index ad350cadbfbd..edd46ce00597 100644 --- a/plugin/trino-hive/pom.xml +++ b/plugin/trino-hive/pom.xml @@ -238,11 +238,6 @@ joda-time - - org.alluxio - alluxio-shaded-client - - org.apache.avro avro diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java index d703227a43e3..7735f2cb2bb7 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java @@ -50,10 +50,6 @@ protected void setup(Binder binder) bindMetastoreModule("thrift", new ThriftMetastoreModule()); bindMetastoreModule("file", new FileMetastoreModule()); bindMetastoreModule("glue", new GlueMetastoreModule()); - // Load Alluxio metastore support through reflection. This makes Alluxio effectively an optional dependency - // and allows deploying Trino without the Alluxio jar. Can be useful if the integration is unused and is flagged - // by a security scanner. - bindMetastoreModule("alluxio-deprecated", deferredModule("io.trino.plugin.hive.metastore.alluxio.AlluxioMetastoreModule")); } install(new DecoratedHiveMetastoreModule(true)); @@ -67,26 +63,6 @@ private void bindMetastoreModule(String name, Module module) module)); } - private static Module deferredModule(String moduleClassName) - { - return new AbstractConfigurationAwareModule() - { - @Override - protected void setup(Binder binder) - { - try { - install(Class.forName(moduleClassName) - .asSubclass(Module.class) - .getConstructor() - .newInstance()); - } - catch (ReflectiveOperationException e) { - throw new RuntimeException("Problem loading module class: " + moduleClassName, e); - } - } - }; - } - @HideDeltaLakeTables @Singleton @Provides diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastore.java deleted file mode 100644 index 7a0b2e5a2961..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastore.java +++ /dev/null @@ -1,504 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore.alluxio; - -import alluxio.client.table.TableMasterClient; -import alluxio.exception.status.AlluxioStatusException; -import alluxio.exception.status.NotFoundException; -import alluxio.grpc.table.ColumnStatisticsInfo; -import alluxio.grpc.table.Constraint; -import alluxio.grpc.table.TableInfo; -import alluxio.grpc.table.layout.hive.PartitionInfo; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import io.trino.plugin.hive.HiveBasicStatistics; -import io.trino.plugin.hive.HiveColumnStatisticType; -import io.trino.plugin.hive.HiveType; -import io.trino.plugin.hive.PartitionStatistics; -import io.trino.plugin.hive.acid.AcidTransaction; -import io.trino.plugin.hive.metastore.Column; -import io.trino.plugin.hive.metastore.Database; -import io.trino.plugin.hive.metastore.HiveColumnStatistics; -import io.trino.plugin.hive.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreConfig; -import io.trino.plugin.hive.metastore.HivePrincipal; -import io.trino.plugin.hive.metastore.HivePrivilegeInfo; -import io.trino.plugin.hive.metastore.HivePrivilegeInfo.HivePrivilege; -import io.trino.plugin.hive.metastore.Partition; -import io.trino.plugin.hive.metastore.PartitionWithStatistics; -import io.trino.plugin.hive.metastore.PrincipalPrivileges; -import io.trino.plugin.hive.metastore.Table; -import io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil; -import io.trino.spi.TrinoException; -import io.trino.spi.connector.SchemaTableName; -import io.trino.spi.predicate.TupleDomain; -import io.trino.spi.security.RoleGrant; -import io.trino.spi.type.Type; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableMap.toImmutableMap; -import static io.trino.plugin.hive.HiveErrorCode.HIVE_METASTORE_ERROR; -import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.getHiveBasicStatistics; -import static io.trino.plugin.hive.util.HiveUtil.makePartName; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; -import static java.util.AbstractMap.SimpleEntry; -import static java.util.Objects.requireNonNull; - -/** - * Implementation of the {@link HiveMetastore} interface through Alluxio. - */ -public class AlluxioHiveMetastore - implements HiveMetastore -{ - private final TableMasterClient client; - - public AlluxioHiveMetastore(TableMasterClient client, HiveMetastoreConfig hiveMetastoreConfig) - { - this.client = requireNonNull(client, "client is null"); - checkArgument(!hiveMetastoreConfig.isHideDeltaLakeTables(), "Hiding Delta Lake tables is not supported"); // TODO - } - - @Override - public Optional getDatabase(String databaseName) - { - try { - return Optional.of(ProtoUtils.fromProto(client.getDatabase(databaseName))); - } - catch (AlluxioStatusException e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public List getAllDatabases() - { - try { - return client.getAllDatabases(); - } - catch (AlluxioStatusException e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional getTable(String databaseName, String tableName) - { - try { - return Optional.of(ProtoUtils.fromProto(client.getTable(databaseName, tableName))); - } - catch (NotFoundException e) { - return Optional.empty(); - } - catch (AlluxioStatusException e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Set getSupportedColumnStatistics(Type type) - { - return ThriftMetastoreUtil.getSupportedColumnStatistics(type); - } - - private Map groupStatisticsByColumn(List statistics, OptionalLong rowCount) - { - return statistics.stream() - .collect(toImmutableMap(ColumnStatisticsInfo::getColName, statisticsObj -> ProtoUtils.fromProto(statisticsObj.getData(), rowCount))); - } - - @Override - public PartitionStatistics getTableStatistics(Table table) - { - try { - HiveBasicStatistics basicStats = ThriftMetastoreUtil.getHiveBasicStatistics(table.getParameters()); - List columns = new ArrayList<>(table.getPartitionColumns()); - columns.addAll(table.getDataColumns()); - List columnNames = columns.stream().map(Column::getName).collect(Collectors.toList()); - List colStatsList = client.getTableColumnStatistics(table.getDatabaseName(), table.getTableName(), columnNames); - return new PartitionStatistics(basicStats, groupStatisticsByColumn(colStatsList, basicStats.getRowCount())); - } - catch (Exception e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Map getPartitionStatistics(Table table, List partitions) - { - try { - List dataColumns = table.getDataColumns().stream() - .map(Column::getName) - .collect(toImmutableList()); - List partitionColumns = table.getPartitionColumns().stream() - .map(Column::getName) - .collect(toImmutableList()); - - Map partitionBasicStatistics = partitions.stream() - .collect(toImmutableMap( - partition -> makePartName(partitionColumns, partition.getValues()), - partition -> getHiveBasicStatistics(partition.getParameters()))); - Map partitionRowCounts = partitionBasicStatistics.entrySet().stream() - .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().getRowCount())); - - long tableRowCount = partitionRowCounts.values().stream() - .mapToLong(count -> count.orElse(0)) - .sum(); - if (!partitionRowCounts.isEmpty() && tableRowCount == 0) { - // When the table has partitions, but row count statistics are set to zero, we treat this case as empty - // statistics to avoid underestimation in the CBO. This scenario may be caused when other engines are - // used to ingest data into partitioned hive tables. - partitionBasicStatistics = partitionBasicStatistics.keySet().stream() - .map(key -> new SimpleEntry<>(key, HiveBasicStatistics.createEmptyStatistics())) - .collect(toImmutableMap(SimpleEntry::getKey, SimpleEntry::getValue)); - } - - Map> colStatsMap = client.getPartitionColumnStatistics(table.getDatabaseName(), table.getTableName(), - ImmutableList.copyOf(partitionBasicStatistics.keySet()), dataColumns); - Map> partitionColumnStatistics = colStatsMap.entrySet().stream() - .filter(entry -> !entry.getValue().isEmpty()) - .collect(toImmutableMap( - Map.Entry::getKey, - entry -> groupStatisticsByColumn(entry.getValue(), partitionRowCounts.getOrDefault(entry.getKey(), OptionalLong.empty())))); - ImmutableMap.Builder result = ImmutableMap.builder(); - for (String partitionName : partitionBasicStatistics.keySet()) { - HiveBasicStatistics basicStatistics = partitionBasicStatistics.get(partitionName); - Map columnStatistics = partitionColumnStatistics.getOrDefault(partitionName, ImmutableMap.of()); - result.put(partitionName, new PartitionStatistics(basicStatistics, columnStatistics)); - } - return result.buildOrThrow(); - } - catch (Exception e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public void updateTableStatistics( - String databaseName, - String tableName, - AcidTransaction transaction, - Function update) - { - throw new TrinoException(NOT_SUPPORTED, "updateTableStatistics"); - } - - @Override - public void updatePartitionStatistics( - Table table, - Map> updates) - { - throw new TrinoException(NOT_SUPPORTED, "updatePartitionStatistics"); - } - - @Override - public List getAllTables(String databaseName) - { - try { - return client.getAllTables(databaseName); - } - catch (NotFoundException e) { - return new ArrayList<>(0); - } - catch (AlluxioStatusException e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Optional> getAllTables() - { - return Optional.empty(); - } - - @Override - public List getTablesWithParameter( - String databaseName, - String parameterKey, - String parameterValue) - { - try { - return client.getAllTables(databaseName).stream() - .filter(tableName -> { - // TODO Is there a way to do a bulk RPC? - try { - TableInfo table = client.getTable(databaseName, tableName); - if (table == null) { - return false; - } - String value = table.getParametersMap().get(parameterKey); - return value != null && value.equals(parameterValue); - } - catch (AlluxioStatusException e) { - throw new TrinoException(HIVE_METASTORE_ERROR, "Failed to get info for table: " + tableName, e); - } - }) - .collect(Collectors.toList()); - } - catch (AlluxioStatusException e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public List getAllViews(String databaseName) - { - // TODO: Add views on the server side - return Collections.emptyList(); - } - - @Override - public Optional> getAllViews() - { - return Optional.empty(); - } - - @Override - public void createDatabase(Database database) - { - throw new TrinoException(NOT_SUPPORTED, "createDatabase"); - } - - @Override - public void dropDatabase(String databaseName, boolean deleteData) - { - throw new TrinoException(NOT_SUPPORTED, "dropDatabase"); - } - - @Override - public void renameDatabase(String databaseName, String newDatabaseName) - { - throw new TrinoException(NOT_SUPPORTED, "renameDatabase"); - } - - @Override - public void setDatabaseOwner(String databaseName, HivePrincipal principal) - { - throw new TrinoException(NOT_SUPPORTED, "setDatabaseOwner"); - } - - @Override - public void createTable(Table table, PrincipalPrivileges principalPrivileges) - { - throw new TrinoException(NOT_SUPPORTED, "createTable"); - } - - @Override - public void dropTable(String databaseName, String tableName, boolean deleteData) - { - throw new TrinoException(NOT_SUPPORTED, "dropTable"); - } - - @Override - public void replaceTable(String databaseName, String tableName, Table newTable, - PrincipalPrivileges principalPrivileges) - { - throw new TrinoException(NOT_SUPPORTED, "replaceTable"); - } - - @Override - public void renameTable(String databaseName, String tableName, String newDatabaseName, - String newTableName) - { - throw new TrinoException(NOT_SUPPORTED, "renameTable"); - } - - @Override - public void commentTable(String databaseName, String tableName, Optional comment) - { - throw new TrinoException(NOT_SUPPORTED, "commentTable"); - } - - @Override - public void setTableOwner(String databaseName, String tableName, HivePrincipal principal) - { - throw new TrinoException(NOT_SUPPORTED, "setTableOwner"); - } - - @Override - public void commentColumn(String databaseName, String tableName, String columnName, Optional comment) - { - throw new TrinoException(NOT_SUPPORTED, "commentColumn"); - } - - @Override - public void addColumn(String databaseName, String tableName, String columnName, - HiveType columnType, String columnComment) - { - throw new TrinoException(NOT_SUPPORTED, "addColumn"); - } - - @Override - public void renameColumn(String databaseName, String tableName, String oldColumnName, - String newColumnName) - { - throw new TrinoException(NOT_SUPPORTED, "renameColumn"); - } - - @Override - public void dropColumn(String databaseName, String tableName, String columnName) - { - throw new TrinoException(NOT_SUPPORTED, "dropColumn"); - } - - @Override - public Optional getPartition(Table table, List partitionValues) - { - throw new TrinoException(NOT_SUPPORTED, "getPartition"); - } - - @Override - public Optional> getPartitionNamesByFilter( - String databaseName, - String tableName, - List columnNames, - TupleDomain partitionKeysFilter) - { - try { - List partitionInfos = ProtoUtils.toPartitionInfoList( - client.readTable(databaseName, tableName, Constraint.getDefaultInstance())); - List partitionNames = partitionInfos.stream() - .map(PartitionInfo::getPartitionName) - .collect(Collectors.toList()); - return Optional.of(partitionNames); - } - catch (AlluxioStatusException e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public Map> getPartitionsByNames(Table table, List partitionNames) - { - if (partitionNames.isEmpty()) { - return Collections.emptyMap(); - } - String databaseName = table.getDatabaseName(); - String tableName = table.getTableName(); - - try { - // Get all partitions - List partitionInfos = ProtoUtils.toPartitionInfoList( - client.readTable(databaseName, tableName, Constraint.getDefaultInstance())); - // Check that table name is correct - // TODO also check for database name equality - partitionInfos = partitionInfos.stream() - .filter(partition -> partition.getTableName().equals(tableName)) - .collect(Collectors.toList()); - Map> result = partitionInfos.stream() - .filter(partitionName -> partitionNames.stream() - .anyMatch(partitionName.getPartitionName()::equals)) - .collect(Collectors.toMap( - PartitionInfo::getPartitionName, - partitionInfo -> Optional.of(ProtoUtils.fromProto(partitionInfo)))); - return Collections.unmodifiableMap(result); - } - catch (AlluxioStatusException e) { - throw new TrinoException(HIVE_METASTORE_ERROR, e); - } - } - - @Override - public void addPartitions(String databaseName, String tableName, - List partitions) - { - throw new TrinoException(NOT_SUPPORTED, "addPartitions"); - } - - @Override - public void dropPartition(String databaseName, String tableName, List parts, - boolean deleteData) - { - throw new TrinoException(NOT_SUPPORTED, "dropPartition"); - } - - @Override - public void alterPartition(String databaseName, String tableName, - PartitionWithStatistics partition) - { - throw new TrinoException(NOT_SUPPORTED, "alterPartition"); - } - - @Override - public void createRole(String role, String grantor) - { - throw new TrinoException(NOT_SUPPORTED, "createRole"); - } - - @Override - public void dropRole(String role) - { - throw new TrinoException(NOT_SUPPORTED, "dropRole"); - } - - @Override - public Set listRoles() - { - throw new TrinoException(NOT_SUPPORTED, "listRoles"); - } - - @Override - public void grantRoles(Set roles, Set grantees, boolean withAdminOption, - HivePrincipal grantor) - { - throw new TrinoException(NOT_SUPPORTED, "grantRoles"); - } - - @Override - public void revokeRoles(Set roles, Set grantees, boolean adminOptionFor, - HivePrincipal grantor) - { - throw new TrinoException(NOT_SUPPORTED, "revokeRoles"); - } - - @Override - public Set listGrantedPrincipals(String role) - { - throw new TrinoException(NOT_SUPPORTED, "listRoleGrants"); - } - - @Override - public Set listRoleGrants(HivePrincipal principal) - { - throw new TrinoException(NOT_SUPPORTED, "listRoleGrants"); - } - - @Override - public void grantTablePrivileges(String databaseName, String tableName, String tableOwner, HivePrincipal grantee, HivePrincipal grantor, Set privileges, boolean grantOption) - { - throw new TrinoException(NOT_SUPPORTED, "grantTablePrivileges"); - } - - @Override - public void revokeTablePrivileges(String databaseName, String tableName, String tableOwner, HivePrincipal grantee, HivePrincipal grantor, Set privileges, boolean grantOption) - { - throw new TrinoException(NOT_SUPPORTED, "revokeTablePrivileges"); - } - - @Override - public Set listTablePrivileges(String databaseName, String tableName, Optional tableOwner, Optional principal) - { - throw new TrinoException(NOT_SUPPORTED, "listTablePrivileges"); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastoreConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastoreConfig.java deleted file mode 100644 index 028aef0a607b..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastoreConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore.alluxio; - -import io.airlift.configuration.Config; -import io.airlift.configuration.ConfigDescription; - -/** - * Configuration for the Alluxio compatible hive metastore interface. - */ -public class AlluxioHiveMetastoreConfig -{ - private String masterAddress; - - public String getMasterAddress() - { - return masterAddress; - } - - @Config("hive.metastore.alluxio.master.address") - @ConfigDescription("Alluxio master address") - public AlluxioHiveMetastoreConfig setMasterAddress(String masterAddress) - { - this.masterAddress = masterAddress; - return this; - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastoreFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastoreFactory.java deleted file mode 100644 index 34dd7b426ae5..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioHiveMetastoreFactory.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore.alluxio; - -import alluxio.client.table.TableMasterClient; -import com.google.inject.Inject; -import io.trino.plugin.hive.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreConfig; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; -import io.trino.spi.security.ConnectorIdentity; - -import java.util.Optional; - -public class AlluxioHiveMetastoreFactory - implements HiveMetastoreFactory -{ - private final AlluxioHiveMetastore metastore; - - @Inject - public AlluxioHiveMetastoreFactory(TableMasterClient client, HiveMetastoreConfig hiveMetastoreConfig) - { - // Alluxio metastore does not support impersonation, so just create a single shared instance - metastore = new AlluxioHiveMetastore(client, hiveMetastoreConfig); - } - - @Override - public boolean isImpersonationEnabled() - { - return false; - } - - @Override - public HiveMetastore createMetastore(Optional identity) - { - return metastore; - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioMetastoreModule.java deleted file mode 100644 index 67d737227f23..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/AlluxioMetastoreModule.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore.alluxio; - -import alluxio.ClientContext; -import alluxio.client.table.RetryHandlingTableMasterClient; -import alluxio.client.table.TableMasterClient; -import alluxio.conf.Configuration; -import alluxio.conf.InstancedConfiguration; -import alluxio.conf.PropertyKey; -import alluxio.master.MasterClientContext; -import com.google.inject.Binder; -import com.google.inject.Key; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.plugin.hive.AllowHiveTableRename; -import io.trino.plugin.hive.metastore.HiveMetastore; -import io.trino.plugin.hive.metastore.HiveMetastoreFactory; -import io.trino.plugin.hive.metastore.RawHiveMetastoreFactory; - -import static io.airlift.configuration.ConfigBinder.configBinder; - -/** - * Module for an Alluxio metastore implementation of the {@link HiveMetastore} interface. - */ -public class AlluxioMetastoreModule - extends AbstractConfigurationAwareModule -{ - @Override - protected void setup(Binder binder) - { - configBinder(binder).bindConfig(AlluxioHiveMetastoreConfig.class); - - binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).to(AlluxioHiveMetastoreFactory.class).in(Scopes.SINGLETON); - binder.bind(Key.get(boolean.class, AllowHiveTableRename.class)).toInstance(false); - } - - @Provides - public TableMasterClient provideCatalogMasterClient(AlluxioHiveMetastoreConfig config) - { - return createCatalogMasterClient(config); - } - - public static TableMasterClient createCatalogMasterClient(AlluxioHiveMetastoreConfig config) - { - InstancedConfiguration conf = Configuration.modifiableGlobal(); - String addr = config.getMasterAddress(); - String[] parts = addr.split(":", 2); - conf.set(PropertyKey.MASTER_HOSTNAME, parts[0]); - if (parts.length > 1) { - conf.set(PropertyKey.MASTER_RPC_PORT, parts[1]); - } - MasterClientContext context = MasterClientContext - .newBuilder(ClientContext.create(conf)).build(); - return new RetryHandlingTableMasterClient(context); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/ProtoUtils.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/ProtoUtils.java deleted file mode 100644 index a07b3aad1d9c..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/alluxio/ProtoUtils.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore.alluxio; - -import alluxio.grpc.table.BinaryColumnStatsData; -import alluxio.grpc.table.BooleanColumnStatsData; -import alluxio.grpc.table.ColumnStatisticsData; -import alluxio.grpc.table.Date; -import alluxio.grpc.table.DateColumnStatsData; -import alluxio.grpc.table.Decimal; -import alluxio.grpc.table.DecimalColumnStatsData; -import alluxio.grpc.table.DoubleColumnStatsData; -import alluxio.grpc.table.FieldSchema; -import alluxio.grpc.table.Layout; -import alluxio.grpc.table.LongColumnStatsData; -import alluxio.grpc.table.PrincipalType; -import alluxio.grpc.table.StringColumnStatsData; -import alluxio.grpc.table.layout.hive.PartitionInfo; -import alluxio.shaded.client.com.google.protobuf.InvalidProtocolBufferException; -import com.google.common.collect.Lists; -import io.trino.plugin.hive.HiveBucketProperty; -import io.trino.plugin.hive.HiveType; -import io.trino.plugin.hive.metastore.Column; -import io.trino.plugin.hive.metastore.Database; -import io.trino.plugin.hive.metastore.HiveColumnStatistics; -import io.trino.plugin.hive.metastore.Partition; -import io.trino.plugin.hive.metastore.SortingColumn; -import io.trino.plugin.hive.metastore.StorageFormat; -import io.trino.plugin.hive.metastore.Table; -import io.trino.plugin.hive.util.HiveBucketing; -import io.trino.spi.TrinoException; -import jakarta.annotation.Nullable; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.time.LocalDate; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalDouble; -import java.util.OptionalLong; -import java.util.Set; - -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static io.trino.plugin.hive.HiveErrorCode.HIVE_INVALID_METADATA; -import static io.trino.plugin.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics; -import static io.trino.plugin.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics; -import static io.trino.plugin.hive.metastore.HiveColumnStatistics.createDateColumnStatistics; -import static io.trino.plugin.hive.metastore.HiveColumnStatistics.createDecimalColumnStatistics; -import static io.trino.plugin.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics; -import static io.trino.plugin.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics; -import static io.trino.plugin.hive.metastore.HiveColumnStatistics.createStringColumnStatistics; -import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreDistinctValuesCount; -import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.fromMetastoreNullsCount; -import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.getTotalSizeInBytes; -import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; - -public final class ProtoUtils -{ - private ProtoUtils() {} - - public static Database fromProto(alluxio.grpc.table.Database db) - { - Optional owner = Optional.ofNullable(db.getOwnerName()); - Optional ownerType = owner.map(name -> db.getOwnerType() == PrincipalType.USER ? io.trino.spi.security.PrincipalType.USER : io.trino.spi.security.PrincipalType.ROLE); - return Database.builder() - .setDatabaseName(db.getDbName()) - .setLocation(db.hasLocation() ? Optional.of(db.getLocation()) : Optional.empty()) - .setOwnerName(owner) - .setOwnerType(ownerType) - .setComment(db.hasComment() ? Optional.of(db.getComment()) : Optional.empty()) - .setParameters(db.getParameterMap()) - .build(); - } - - public static Table fromProto(alluxio.grpc.table.TableInfo table) - { - if (!table.hasLayout()) { - throw new TrinoException(NOT_SUPPORTED, "Unsupported table metadata. missing layout.: " + table.getTableName()); - } - Layout layout = table.getLayout(); - if (!alluxio.table.ProtoUtils.isHiveLayout(layout)) { - throw new TrinoException(NOT_SUPPORTED, "Unsupported table layout: " + layout + " for table: " + table.getTableName()); - } - try { - PartitionInfo partitionInfo = alluxio.table.ProtoUtils.toHiveLayout(layout); - - // compute the data columns - Set partitionColumns = table.getPartitionColsList().stream() - .map(FieldSchema::getName) - .collect(toImmutableSet()); - List dataColumns = table.getSchema().getColsList().stream() - .filter((f) -> !partitionColumns.contains(f.getName())) - .collect(toImmutableList()); - - Map tableParameters = table.getParametersMap(); - Table.Builder builder = Table.builder() - .setDatabaseName(table.getDbName()) - .setTableName(table.getTableName()) - .setOwner(Optional.ofNullable(table.getOwner())) - .setTableType(table.getType().toString()) - .setDataColumns(dataColumns.stream() - .map(ProtoUtils::fromProto) - .collect(toImmutableList())) - .setPartitionColumns(table.getPartitionColsList().stream() - .map(ProtoUtils::fromProto) - .collect(toImmutableList())) - .setParameters(tableParameters) - .setViewOriginalText(Optional.empty()) - .setViewExpandedText(Optional.empty()); - alluxio.grpc.table.layout.hive.Storage storage = partitionInfo.getStorage(); - builder.getStorageBuilder() - .setSkewed(storage.getSkewed()) - .setStorageFormat(fromProto(storage.getStorageFormat())) - .setLocation(storage.getLocation()) - .setBucketProperty(storage.hasBucketProperty() ? fromProto(tableParameters, storage.getBucketProperty()) : Optional.empty()) - .setSerdeParameters(storage.getStorageFormat().getSerdelibParametersMap()); - return builder.build(); - } - catch (InvalidProtocolBufferException e) { - throw new IllegalArgumentException("Failed to extract PartitionInfo from TableInfo", e); - } - } - - static SortingColumn fromProto(alluxio.grpc.table.layout.hive.SortingColumn column) - { - if (column.getOrder().equals(alluxio.grpc.table.layout.hive.SortingColumn.SortingOrder.ASCENDING)) { - return new SortingColumn(column.getColumnName(), SortingColumn.Order.ASCENDING); - } - if (column.getOrder().equals(alluxio.grpc.table.layout.hive.SortingColumn.SortingOrder.DESCENDING)) { - return new SortingColumn(column.getColumnName(), SortingColumn.Order.DESCENDING); - } - throw new IllegalArgumentException("Invalid sort order: " + column.getOrder()); - } - - static Optional fromProto(Map tableParameters, alluxio.grpc.table.layout.hive.HiveBucketProperty property) - { - // must return empty if buckets <= 0 - if (!property.hasBucketCount() || property.getBucketCount() <= 0) { - return Optional.empty(); - } - List sortedBy = property.getSortedByList().stream() - .map(ProtoUtils::fromProto) - .collect(toImmutableList()); - HiveBucketing.BucketingVersion bucketingVersion = HiveBucketing.getBucketingVersion(tableParameters); - return Optional.of(new HiveBucketProperty(property.getBucketedByList(), bucketingVersion, (int) property.getBucketCount(), sortedBy)); - } - - static StorageFormat fromProto(alluxio.grpc.table.layout.hive.StorageFormat format) - { - return StorageFormat.create(format.getSerde(), format.getInputFormat(), format.getOutputFormat()); - } - - private static Optional fromMetastoreDecimal(@Nullable Decimal decimal) - { - if (decimal == null) { - return Optional.empty(); - } - return Optional.of(new BigDecimal(new BigInteger(decimal.getUnscaled().toByteArray()), decimal.getScale())); - } - - private static Optional fromMetastoreDate(@Nullable Date date) - { - if (date == null) { - return Optional.empty(); - } - return Optional.of(LocalDate.ofEpochDay(date.getDaysSinceEpoch())); - } - - public static HiveColumnStatistics fromProto(ColumnStatisticsData columnStatistics, OptionalLong rowCount) - { - if (columnStatistics.hasLongStats()) { - LongColumnStatsData longStatsData = columnStatistics.getLongStats(); - OptionalLong min = longStatsData.hasLowValue() ? OptionalLong.of(longStatsData.getLowValue()) : OptionalLong.empty(); - OptionalLong max = longStatsData.hasHighValue() ? OptionalLong.of(longStatsData.getHighValue()) : OptionalLong.empty(); - OptionalLong nullsCount = longStatsData.hasNumNulls() ? fromMetastoreNullsCount(longStatsData.getNumNulls()) : OptionalLong.empty(); - OptionalLong distinctValuesCount = longStatsData.hasNumDistincts() ? OptionalLong.of(longStatsData.getNumDistincts()) : OptionalLong.empty(); - return createIntegerColumnStatistics(min, max, nullsCount, fromMetastoreDistinctValuesCount(distinctValuesCount, nullsCount, rowCount)); - } - if (columnStatistics.hasDoubleStats()) { - DoubleColumnStatsData doubleStatsData = columnStatistics.getDoubleStats(); - OptionalDouble min = doubleStatsData.hasLowValue() ? OptionalDouble.of(doubleStatsData.getLowValue()) : OptionalDouble.empty(); - OptionalDouble max = doubleStatsData.hasHighValue() ? OptionalDouble.of(doubleStatsData.getHighValue()) : OptionalDouble.empty(); - OptionalLong nullsCount = doubleStatsData.hasNumNulls() ? fromMetastoreNullsCount(doubleStatsData.getNumNulls()) : OptionalLong.empty(); - OptionalLong distinctValuesCount = doubleStatsData.hasNumDistincts() ? OptionalLong.of(doubleStatsData.getNumDistincts()) : OptionalLong.empty(); - return createDoubleColumnStatistics(min, max, nullsCount, fromMetastoreDistinctValuesCount(distinctValuesCount, nullsCount, rowCount)); - } - if (columnStatistics.hasDecimalStats()) { - DecimalColumnStatsData decimalStatsData = columnStatistics.getDecimalStats(); - Optional min = decimalStatsData.hasLowValue() ? fromMetastoreDecimal(decimalStatsData.getLowValue()) : Optional.empty(); - Optional max = decimalStatsData.hasHighValue() ? fromMetastoreDecimal(decimalStatsData.getHighValue()) : Optional.empty(); - OptionalLong nullsCount = decimalStatsData.hasNumNulls() ? fromMetastoreNullsCount(decimalStatsData.getNumNulls()) : OptionalLong.empty(); - OptionalLong distinctValuesCount = decimalStatsData.hasNumDistincts() ? OptionalLong.of(decimalStatsData.getNumDistincts()) : OptionalLong.empty(); - return createDecimalColumnStatistics(min, max, nullsCount, fromMetastoreDistinctValuesCount(distinctValuesCount, nullsCount, rowCount)); - } - if (columnStatistics.hasDateStats()) { - DateColumnStatsData dateStatsData = columnStatistics.getDateStats(); - Optional min = dateStatsData.hasLowValue() ? fromMetastoreDate(dateStatsData.getLowValue()) : Optional.empty(); - Optional max = dateStatsData.hasHighValue() ? fromMetastoreDate(dateStatsData.getHighValue()) : Optional.empty(); - OptionalLong nullsCount = dateStatsData.hasNumNulls() ? fromMetastoreNullsCount(dateStatsData.getNumNulls()) : OptionalLong.empty(); - OptionalLong distinctValuesCount = dateStatsData.hasNumDistincts() ? OptionalLong.of(dateStatsData.getNumDistincts()) : OptionalLong.empty(); - return createDateColumnStatistics(min, max, nullsCount, fromMetastoreDistinctValuesCount(distinctValuesCount, nullsCount, rowCount)); - } - if (columnStatistics.hasBooleanStats()) { - BooleanColumnStatsData booleanStatsData = columnStatistics.getBooleanStats(); - OptionalLong trueCount = OptionalLong.empty(); - OptionalLong falseCount = OptionalLong.empty(); - // Impala 'COMPUTE STATS' writes 1 as the numTrue and -1 as the numFalse - if (booleanStatsData.hasNumTrues() && booleanStatsData.hasNumFalses() && (booleanStatsData.getNumFalses() != -1)) { - trueCount = OptionalLong.of(booleanStatsData.getNumTrues()); - falseCount = OptionalLong.of(booleanStatsData.getNumFalses()); - } - return createBooleanColumnStatistics( - trueCount, - falseCount, - booleanStatsData.hasNumNulls() ? fromMetastoreNullsCount(booleanStatsData.getNumNulls()) : OptionalLong.empty()); - } - if (columnStatistics.hasStringStats()) { - StringColumnStatsData stringStatsData = columnStatistics.getStringStats(); - OptionalLong maxColumnLength = stringStatsData.hasMaxColLen() ? OptionalLong.of(stringStatsData.getMaxColLen()) : OptionalLong.empty(); - OptionalDouble averageColumnLength = stringStatsData.hasAvgColLen() ? OptionalDouble.of(stringStatsData.getAvgColLen()) : OptionalDouble.empty(); - OptionalLong nullsCount = stringStatsData.hasNumNulls() ? fromMetastoreNullsCount(stringStatsData.getNumNulls()) : OptionalLong.empty(); - OptionalLong distinctValuesCount = stringStatsData.hasNumDistincts() ? OptionalLong.of(stringStatsData.getNumDistincts()) : OptionalLong.empty(); - return createStringColumnStatistics( - maxColumnLength, - getTotalSizeInBytes(averageColumnLength, rowCount, nullsCount), - nullsCount, - fromMetastoreDistinctValuesCount(distinctValuesCount, nullsCount, rowCount)); - } - if (columnStatistics.hasBinaryStats()) { - BinaryColumnStatsData binaryStatsData = columnStatistics.getBinaryStats(); - OptionalLong maxColumnLength = binaryStatsData.hasMaxColLen() ? OptionalLong.of(binaryStatsData.getMaxColLen()) : OptionalLong.empty(); - OptionalDouble averageColumnLength = binaryStatsData.hasAvgColLen() ? OptionalDouble.of(binaryStatsData.getAvgColLen()) : OptionalDouble.empty(); - OptionalLong nullsCount = binaryStatsData.hasNumNulls() ? fromMetastoreNullsCount(binaryStatsData.getNumNulls()) : OptionalLong.empty(); - return createBinaryColumnStatistics( - maxColumnLength, - getTotalSizeInBytes(averageColumnLength, rowCount, nullsCount), - nullsCount); - } - throw new TrinoException(HIVE_INVALID_METADATA, "Invalid column statistics data: " + columnStatistics); - } - - static Column fromProto(alluxio.grpc.table.FieldSchema column) - { - Optional comment = column.hasComment() ? Optional.of(column.getComment()) : Optional.empty(); - return new Column(column.getName(), HiveType.valueOf(column.getType()), comment); - } - - public static Partition fromProto(alluxio.grpc.table.layout.hive.PartitionInfo info) - { - Map parametersMap = info.getParametersMap(); - Partition.Builder builder = Partition.builder() - .setColumns(info.getDataColsList().stream() - .map(ProtoUtils::fromProto) - .collect(toImmutableList())) - .setDatabaseName(info.getDbName()) - .setParameters(parametersMap) - .setValues(Lists.newArrayList(info.getValuesList())) - .setTableName(info.getTableName()); - - builder.getStorageBuilder() - .setSkewed(info.getStorage().getSkewed()) - .setStorageFormat(fromProto(info.getStorage().getStorageFormat())) - .setLocation(info.getStorage().getLocation()) - .setBucketProperty(info.getStorage().hasBucketProperty() - ? fromProto(parametersMap, info.getStorage().getBucketProperty()) : Optional.empty()) - .setSerdeParameters(info.getStorage().getStorageFormat().getSerdelibParametersMap()); - - return builder.build(); - } - - public static alluxio.grpc.table.layout.hive.PartitionInfo toPartitionInfo(alluxio.grpc.table.Partition part) - { - try { - return alluxio.table.ProtoUtils.extractHiveLayout(part); - } - catch (InvalidProtocolBufferException e) { - throw new IllegalArgumentException("Failed to extract PartitionInfo", e); - } - } - - public static List toPartitionInfoList(List parts) - { - return parts.stream() - .map(ProtoUtils::toPartitionInfo) - .collect(toImmutableList()); - } -} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java index 06cb66a91c2f..391ce377d948 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/AbstractTestHive.java @@ -13,11 +13,11 @@ */ package io.trino.plugin.hive; -import alluxio.collections.ConcurrentHashSet; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; import io.airlift.json.JsonCodec; import io.airlift.log.Logger; @@ -673,7 +673,7 @@ private static RowType toRowType(List columns) private ScheduledExecutorService heartbeatService; private java.nio.file.Path temporaryStagingDirectory; - protected final ConcurrentHashSet materializedViews = new ConcurrentHashSet<>(); + protected final Set materializedViews = Sets.newConcurrentHashSet(); @BeforeClass(alwaysRun = true) public void setupClass() diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestAlluxioHiveMetastoreConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestAlluxioHiveMetastoreConfig.java deleted file mode 100644 index 8b7c19ce79ea..000000000000 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestAlluxioHiveMetastoreConfig.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore.alluxio; - -import com.google.common.collect.ImmutableMap; -import org.testng.annotations.Test; - -import java.util.Map; - -import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; -import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; -import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; - -public class TestAlluxioHiveMetastoreConfig -{ - @Test - public void testDefaults() - { - assertRecordedDefaults(recordDefaults(AlluxioHiveMetastoreConfig.class) - .setMasterAddress(null)); - } - - @Test - public void testExplicitPropertyMapping() - { - Map properties = ImmutableMap.of("hive.metastore.alluxio.master.address", "localhost:19998"); - - AlluxioHiveMetastoreConfig expected = new AlluxioHiveMetastoreConfig() - .setMasterAddress("localhost:19998"); - - assertFullMapping(properties, expected); - } -} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestProtoUtils.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestProtoUtils.java deleted file mode 100644 index 4e0f842030bc..000000000000 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestProtoUtils.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore.alluxio; - -import alluxio.grpc.table.Layout; -import alluxio.shaded.client.com.google.protobuf.ByteString; -import io.trino.plugin.hive.HiveBucketProperty; -import io.trino.plugin.hive.HiveType; -import io.trino.plugin.hive.metastore.Column; -import io.trino.plugin.hive.metastore.Database; -import io.trino.plugin.hive.metastore.Partition; -import io.trino.plugin.hive.metastore.SortingColumn; -import io.trino.plugin.hive.metastore.Storage; -import io.trino.plugin.hive.metastore.StorageFormat; -import io.trino.plugin.hive.metastore.Table; -import io.trino.plugin.hive.util.HiveBucketing; -import io.trino.spi.TrinoException; -import org.testng.annotations.Test; - -import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertTrue; - -public class TestProtoUtils -{ - @Test - public void testDatabaseNameLocation() - { - Database db = ProtoUtils.fromProto(TestingAlluxioMetastoreObjects.getTestingDatabase().build()); - assertEquals(TestingAlluxioMetastoreObjects.DATABASE_NAME, db.getDatabaseName()); - assertEquals("alluxio:///", db.getLocation().get()); - // Intentionally leave location unset - alluxio.grpc.table.Database.Builder alluxioDb = TestingAlluxioMetastoreObjects.getTestingDatabase() - .clearLocation(); - assertEquals(Optional.empty(), ProtoUtils.fromProto(alluxioDb.build()).getLocation()); - } - - @Test(expectedExceptions = TrinoException.class) - public void testTableMissingLayout() - { - ProtoUtils.fromProto(TestingAlluxioMetastoreObjects.getTestingTableInfo().clearLayout().build()); - } - - @Test(expectedExceptions = TrinoException.class) - public void testTableNonHiveLayout() - { - alluxio.grpc.table.TableInfo.Builder alluxioTable = alluxio.grpc.table.TableInfo.newBuilder() - .setLayout(TestingAlluxioMetastoreObjects.getTestingNonHiveLayout()); - ProtoUtils.fromProto(alluxioTable.build()); - } - - @Test(expectedExceptions = IllegalArgumentException.class) - public void testTableBadLayoutBytes() - { - Layout.Builder alluxioLayout = TestingAlluxioMetastoreObjects.getTestingHiveLayout() - .setLayoutData(ByteString.copyFrom(new byte[] {'z', 'z', 'z'})); - alluxio.grpc.table.TableInfo.Builder alluxioTable = TestingAlluxioMetastoreObjects.getTestingTableInfo() - .setLayout(alluxioLayout); - ProtoUtils.fromProto(alluxioTable.build()); - } - - @Test - public void testTable() - { - alluxio.grpc.table.TableInfo.Builder table = TestingAlluxioMetastoreObjects.getTestingTableInfo(); - alluxio.grpc.table.FieldSchema fieldSchema = TestingAlluxioMetastoreObjects.getTestingFieldSchema().build(); - Table t = ProtoUtils.fromProto(table.build()); - Column c = t.getColumn(TestingAlluxioMetastoreObjects.COLUMN_NAME).get(); - assertEquals(table.getDbName(), t.getDatabaseName()); - assertEquals(table.getTableName(), t.getTableName()); - assertEquals(table.getOwner(), t.getOwner().orElse(null)); - assertEquals(table.getType().toString(), t.getTableType()); - assertEquals(0, t.getDataColumns().size()); - assertEquals(1, t.getPartitionColumns().size()); - assertEquals(table.getParametersMap(), t.getParameters()); - assertEquals(Optional.empty(), t.getViewOriginalText()); - assertEquals(Optional.empty(), t.getViewExpandedText()); - assertEquals(fieldSchema.getName(), c.getName()); - assertEquals(fieldSchema.getComment(), c.getComment().get()); - assertEquals(fieldSchema.getType(), c.getType().toString()); - Storage s = t.getStorage(); - alluxio.grpc.table.layout.hive.Storage storage = TestingAlluxioMetastoreObjects.getTestingPartitionInfo().getStorage(); - assertEquals(storage.getSkewed(), s.isSkewed()); - assertEquals(ProtoUtils.fromProto(storage.getStorageFormat()), s.getStorageFormat()); - assertEquals(storage.getLocation(), s.getLocation()); - assertEquals(ProtoUtils.fromProto(table.getParametersMap(), storage.getBucketProperty()), s.getBucketProperty()); - assertEquals(storage.getStorageFormat().getSerdelibParametersMap(), s.getSerdeParameters()); - } - - @Test - public void testSortingColumn() - { - alluxio.grpc.table.layout.hive.SortingColumn.Builder column = TestingAlluxioMetastoreObjects.getTestingSortingColumn(); - SortingColumn c = ProtoUtils.fromProto(column.build()); - assertEquals(column.getColumnName(), c.getColumnName()); - assertEquals(SortingColumn.Order.valueOf(column.getOrder().toString()), c.getOrder()); - } - - @Test - public void testBucketProperty() - { - alluxio.grpc.table.layout.hive.HiveBucketProperty.Builder bucketProperty = TestingAlluxioMetastoreObjects.getTestingHiveBucketProperty(); - Optional bp = ProtoUtils.fromProto(TestingAlluxioMetastoreObjects.getTestingTableInfo().getParametersMap(), bucketProperty.build()); - assertTrue(bp.isPresent()); - assertEquals(Collections.singletonList(ProtoUtils.fromProto(TestingAlluxioMetastoreObjects.getTestingSortingColumn().build())), - bp.get().getSortedBy()); - assertEquals(1, bp.get().getSortedBy().size()); - assertEquals(bucketProperty.getBucketedByCount(), bp.get().getBucketCount()); - assertEquals(HiveBucketing.BucketingVersion.BUCKETING_V1, bp.get().getBucketingVersion()); - } - - @Test - public void testBucketPropertyNoBuckets() - { - alluxio.grpc.table.layout.hive.HiveBucketProperty.Builder bucketProperty = TestingAlluxioMetastoreObjects.getTestingHiveBucketProperty(); - bucketProperty.clearBucketCount(); - Map tableParameters = TestingAlluxioMetastoreObjects.getTestingTableInfo().getParametersMap(); - Optional bp = ProtoUtils.fromProto(tableParameters, bucketProperty.build()); - assertFalse(bp.isPresent()); - - bucketProperty = TestingAlluxioMetastoreObjects.getTestingHiveBucketProperty(); - bucketProperty.setBucketCount(0); - bp = ProtoUtils.fromProto(tableParameters, bucketProperty.build()); - assertFalse(bp.isPresent()); - } - - @Test - public void testStorageFormat() - { - alluxio.grpc.table.layout.hive.StorageFormat.Builder storageFormat = TestingAlluxioMetastoreObjects.getTestingStorageFormat(); - StorageFormat fmt = ProtoUtils.fromProto(storageFormat.build()); - assertEquals(storageFormat.getSerde(), fmt.getSerde()); - assertEquals(storageFormat.getInputFormat(), fmt.getInputFormat()); - assertEquals(storageFormat.getOutputFormat(), fmt.getOutputFormat()); - } - - @Test - public void testColumn() - { - alluxio.grpc.table.FieldSchema.Builder fieldSchema = TestingAlluxioMetastoreObjects.getTestingFieldSchema(); - Column column = ProtoUtils.fromProto(fieldSchema.build()); - assertTrue(column.getComment().isPresent()); - assertEquals(fieldSchema.getComment(), column.getComment().get()); - assertEquals(fieldSchema.getName(), column.getName()); - assertEquals(HiveType.valueOf(fieldSchema.getType()), column.getType()); - } - - @Test - public void testColumnNoComment() - { - alluxio.grpc.table.FieldSchema.Builder fieldSchema = TestingAlluxioMetastoreObjects.getTestingFieldSchema(); - fieldSchema.clearComment(); - Column column = ProtoUtils.fromProto(fieldSchema.build()); - assertFalse(column.getComment().isPresent()); - assertEquals(fieldSchema.getName(), column.getName()); - assertEquals(HiveType.valueOf(fieldSchema.getType()), column.getType()); - } - - @Test - public void testPartition() - { - alluxio.grpc.table.layout.hive.PartitionInfo.Builder partitionInfo = TestingAlluxioMetastoreObjects.getTestingPartitionInfo(); - Partition partition = ProtoUtils.fromProto(partitionInfo.build()); - assertEquals( - partitionInfo.getDataColsList().stream().map(ProtoUtils::fromProto).collect(Collectors.toList()), - partition.getColumns()); - assertEquals(partitionInfo.getDbName(), partition.getDatabaseName()); - assertEquals(partitionInfo.getParametersMap(), partition.getParameters()); - assertEquals(partitionInfo.getValuesList(), partition.getValues()); - assertEquals(partitionInfo.getTableName(), partition.getTableName()); - - Storage s = partition.getStorage(); - alluxio.grpc.table.layout.hive.Storage storage = - TestingAlluxioMetastoreObjects.getTestingPartitionInfo().getStorage(); - assertEquals(storage.getSkewed(), s.isSkewed()); - assertEquals(ProtoUtils.fromProto(storage.getStorageFormat()), s.getStorageFormat()); - assertEquals(storage.getLocation(), s.getLocation()); - assertEquals(ProtoUtils.fromProto(partitionInfo.getParametersMap(), storage.getBucketProperty()), s.getBucketProperty()); - assertEquals(storage.getStorageFormat().getSerdelibParametersMap(), s.getSerdeParameters()); - } -} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestingAlluxioMetastoreObjects.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestingAlluxioMetastoreObjects.java deleted file mode 100644 index 522bd16e7eee..000000000000 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/alluxio/TestingAlluxioMetastoreObjects.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.metastore.alluxio; - -import alluxio.grpc.table.Database; -import alluxio.grpc.table.FieldSchema; -import alluxio.grpc.table.Layout; -import alluxio.grpc.table.LayoutSpec; -import alluxio.grpc.table.Partition; -import alluxio.grpc.table.PartitionSpec; -import alluxio.grpc.table.Schema; -import alluxio.grpc.table.TableInfo; -import alluxio.grpc.table.layout.hive.HiveBucketProperty; -import alluxio.grpc.table.layout.hive.PartitionInfo; -import alluxio.grpc.table.layout.hive.SortingColumn; -import alluxio.grpc.table.layout.hive.Storage; -import alluxio.grpc.table.layout.hive.StorageFormat; -import alluxio.shaded.client.com.google.protobuf.ByteString; - -import static java.lang.String.format; - -public final class TestingAlluxioMetastoreObjects -{ - private TestingAlluxioMetastoreObjects() {} - - public static final String DATABASE_NAME = "test_db"; - public static final String OWNER_NAME = "test_owner"; - public static final String COLUMN_NAME = "test_owner"; - public static final String TABLE_NAME = "test_table"; - public static final String LOCATION = "alluxio:///"; - public static final String SPEC_NAME = "spec"; - - public static Database.Builder getTestingDatabase() - { - return alluxio.grpc.table.Database.newBuilder() - .setDbName(DATABASE_NAME) - .setDescription("test") - .setLocation(LOCATION); - } - - public static FieldSchema.Builder getTestingFieldSchema() - { - return FieldSchema.newBuilder() - .setId(0) - .setName(COLUMN_NAME) - .setType("int") - .setComment(""); - } - - public static PartitionInfo.Builder getTestingPartitionInfo() - { - return PartitionInfo.newBuilder() - .setDbName(DATABASE_NAME) - .setTableName(TABLE_NAME) - .addValues("1") - .setPartitionName(format("%s=1", COLUMN_NAME)) - .setStorage(getTestingStorage()) - .addDataCols(getTestingFieldSchema()); - } - - public static Layout.Builder getTestingHiveLayout() - { - return Layout.newBuilder() - .setLayoutSpec(LayoutSpec.newBuilder().setSpec(SPEC_NAME).build()) - .setLayoutData(getTestingPartitionInfo().build().toByteString()) - .setLayoutType("hive"); - } - - public static Layout.Builder getTestingNonHiveLayout() - { - return Layout.newBuilder() - .setLayoutData(ByteString.EMPTY) - .setLayoutSpec(LayoutSpec.newBuilder().setSpec(SPEC_NAME).build()) - .setLayoutType("not-hive"); - } - - public static TableInfo.Builder getTestingTableInfo() - { - return TableInfo.newBuilder() - .setLayout(getTestingHiveLayout()) - .setTableName(TABLE_NAME) - .setOwner(OWNER_NAME) - .setType(TableInfo.TableType.IMPORTED) - // Single column partition, no data columns - .addPartitionCols(getTestingFieldSchema()) - .setSchema(getTestingSchema()) - .putParameters("table", "parameter"); - } - - public static Schema.Builder getTestingSchema() - { - return Schema.newBuilder() - .addCols(getTestingFieldSchema()); - } - - public static Storage.Builder getTestingStorage() - { - return Storage.newBuilder() - .setStorageFormat(getTestingStorageFormat()) - .setLocation(LOCATION) - .setBucketProperty(getTestingHiveBucketProperty()) - .setSkewed(false) - .putSerdeParameters("serde_param_key", "serde_param_value"); - } - - public static StorageFormat.Builder getTestingStorageFormat() - { - return StorageFormat.newBuilder() - .setSerde("serde") - .setInputFormat("TextFile") - .setOutputFormat("TextFile") - .putSerdelibParameters("serdelib_key", "serdelib_value"); - } - - public static HiveBucketProperty.Builder getTestingHiveBucketProperty() - { - return HiveBucketProperty.newBuilder() - .addBucketedBy(COLUMN_NAME) - .setBucketCount(1) - .addSortedBy(getTestingSortingColumn()); - } - - public static SortingColumn.Builder getTestingSortingColumn() - { - return SortingColumn.newBuilder() - .setColumnName(COLUMN_NAME) - .setOrder(SortingColumn.SortingOrder.ASCENDING); - } - - public static Partition.Builder getTestingPartition() - { - return Partition.newBuilder() - .setBaseLayout(getTestingHiveLayout()) - .setPartitionSpec(getTestingPartitionSpec()); - } - - public static PartitionSpec.Builder getTestingPartitionSpec() - { - return PartitionSpec.newBuilder() - .setSpec(SPEC_NAME); - } -} diff --git a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java index c0d42e119967..7b13f5191e25 100644 --- a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java +++ b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiPlugin.java @@ -94,25 +94,6 @@ public void testGlueMetastore() .hasMessageContaining("Error: Configuration property 'hive.metastore.uri' was not used"); } - @Test - public void testAlluxioMetastore() - { - ConnectorFactory factory = getConnectorFactory(); - assertThatThrownBy(() -> factory.create( - "test", - Map.of("hive.metastore", "alluxio"), - new TestingConnectorContext())) - .hasMessageMatching("(?s)Unable to create injector, see the following errors:.*" + - "Explicit bindings are required and HiveMetastoreFactory .* is not explicitly bound.*"); - - assertThatThrownBy(() -> factory.create( - "test", - Map.of("hive.metastore", "alluxio-deprecated"), - new TestingConnectorContext())) - // TODO: this should fail cleanly. We do not plan to support Alluxio metastore with Hudi - .hasStackTraceContaining("Cannot invoke \"String.split(String, int)\" because \"addr\" is null"); - } - @Test public void testHiveConfigIsNotBound() { diff --git a/plugin/trino-session-property-managers/pom.xml b/plugin/trino-session-property-managers/pom.xml index 74c1108136b8..0ef4b4630b5f 100644 --- a/plugin/trino-session-property-managers/pom.xml +++ b/plugin/trino-session-property-managers/pom.xml @@ -155,12 +155,6 @@ trino-hive jar test - - - org.alluxio - alluxio-shaded-client - - @@ -168,12 +162,6 @@ trino-hive test-jar test - - - org.alluxio - alluxio-shaded-client - - diff --git a/testing/trino-faulttolerant-tests/pom.xml b/testing/trino-faulttolerant-tests/pom.xml index 7d6c2f6d6994..c355edcbf776 100644 --- a/testing/trino-faulttolerant-tests/pom.xml +++ b/testing/trino-faulttolerant-tests/pom.xml @@ -184,12 +184,6 @@ io.trino trino-hive test - - - org.alluxio - alluxio-shaded-client - - @@ -197,12 +191,6 @@ trino-hive test-jar test - - - org.alluxio - alluxio-shaded-client - - diff --git a/testing/trino-tests/pom.xml b/testing/trino-tests/pom.xml index 2bc70d7040e1..7651799d1b98 100644 --- a/testing/trino-tests/pom.xml +++ b/testing/trino-tests/pom.xml @@ -200,11 +200,6 @@ com.linkedin.calcite calcite-core - - - org.alluxio - alluxio-shaded-client - @@ -219,11 +214,6 @@ com.linkedin.calcite calcite-core - - - org.alluxio - alluxio-shaded-client -