From cd1d4813bfca39e0715164a0cbb17da5b6a56435 Mon Sep 17 00:00:00 2001 From: Vlad Rozov Date: Wed, 5 Feb 2020 19:00:10 -0800 Subject: [PATCH 1/2] Add iceberg to product tests suite --- .../product/launcher/env/common/Hadoop.java | 4 +- .../launcher/env/environment/Multinode.java | 4 +- .../env/environment/MultinodeTls.java | 4 +- .../env/environment/MultinodeTlsKerberos.java | 7 +- .../SinglenodeHdfsImpersonation.java | 4 +- .../SinglenodeHiveImpersonation.java | 4 +- .../SinglenodeKerberosHdfsImpersonation.java | 4 +- ...deKerberosHdfsImpersonationCrossRealm.java | 5 ++ ...osHdfsImpersonationWithWireEncryption.java | 7 +- ...SinglenodeKerberosHdfsNoImpersonation.java | 4 +- .../SinglenodeKerberosHiveImpersonation.java | 4 +- ...inglenodeKerberosKmsHdfsImpersonation.java | 5 ++ ...glenodeKerberosKmsHdfsNoImpersonation.java | 5 ++ .../env/environment/TwoKerberosHives.java | 10 +++ .../env/environment/TwoMixedHives.java | 8 ++ .../common/hadoop/iceberg.properties | 4 + .../multinode-tls-kerberos/iceberg.properties | 15 ++++ .../iceberg.properties | 8 ++ .../iceberg.properties | 8 ++ .../iceberg.properties | 22 ++++++ .../iceberg.properties | 17 +++++ .../iceberg.properties | 15 ++++ .../iceberg.properties | 15 ++++ .../iceberg.properties | 16 ++++ .../iceberg.properties | 21 ++++++ .../iceberg.properties | 16 ++++ .../two-kerberos-hives/iceberg1.properties | 16 ++++ .../two-kerberos-hives/iceberg2.properties | 15 ++++ .../two-mixed-hives/iceberg1.properties | 16 ++++ .../two-mixed-hives/iceberg2.properties | 5 ++ .../presto/etc/catalog/iceberg.properties | 3 + .../presto/etc/catalog/iceberg1.properties | 3 + .../presto/etc/catalog/iceberg2.properties | 3 + .../java/io/prestosql/tests/TestGroups.java | 1 + .../tests/iceberg/TestIcebergCreateTable.java | 74 +++++++++++++++++++ 35 files changed, 361 insertions(+), 11 deletions(-) create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls-kerberos/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/iceberg1.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/iceberg2.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/iceberg1.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/iceberg2.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg1.properties create mode 100644 presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg2.properties create mode 100644 presto-product-tests/src/main/java/io/prestosql/tests/iceberg/TestIcebergCreateTable.java diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/common/Hadoop.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/common/Hadoop.java index ddc3c2b7b40c..a95a0e128a91 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/common/Hadoop.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/common/Hadoop.java @@ -33,6 +33,7 @@ public final class Hadoop implements EnvironmentExtender { public static final String CONTAINER_PRESTO_HIVE_PROPERTIES = CONTAINER_PRESTO_ETC + "/catalog/hive.properties"; + public static final String CONTAINER_PRESTO_ICEBERG_PROPERTIES = CONTAINER_PRESTO_ETC + "/catalog/iceberg.properties"; private final DockerFiles dockerFiles; @@ -56,7 +57,8 @@ public void extendEnvironment(Environment.Builder builder) builder.addContainer("hadoop-master", createHadoopMaster()); builder.configureContainer("presto-master", container -> container - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY)); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY)); } @SuppressWarnings("resource") diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/Multinode.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/Multinode.java index 3332643621fa..c3ce9a7a8a06 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/Multinode.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/Multinode.java @@ -29,6 +29,7 @@ import java.io.File; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static io.prestosql.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_CONFIG_PROPERTIES; import static io.prestosql.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_JVM_CONFIG; import static io.prestosql.tests.product.launcher.env.common.Standard.createPrestoContainer; @@ -78,7 +79,8 @@ private DockerContainer createPrestoWorker() DockerContainer container = createPrestoContainer(dockerFiles, pathResolver, serverPackage, "prestodev/centos6-oj8:" + imagesVersion) .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode/multinode-worker-jvm.config"), CONTAINER_PRESTO_JVM_CONFIG, READ_ONLY) .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode/multinode-worker-config.properties"), CONTAINER_PRESTO_CONFIG_PROPERTIES, READ_ONLY) - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY); return container; } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/MultinodeTls.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/MultinodeTls.java index 90fe16915e49..5827e3208bd9 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/MultinodeTls.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/MultinodeTls.java @@ -29,6 +29,7 @@ import java.io.File; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static io.prestosql.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_CONFIG_PROPERTIES; import static io.prestosql.tests.product.launcher.env.common.Standard.CONTAINER_TEMPTO_PROFILE_CONFIG; import static io.prestosql.tests.product.launcher.env.common.Standard.createPrestoContainer; @@ -88,7 +89,8 @@ private void addPrestoWorker(Environment.Builder builder, String workerName) .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withDomainName("docker.cluster")) .withNetworkAliases(workerName + ".docker.cluster") .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls/config-worker.properties"), CONTAINER_PRESTO_CONFIG_PROPERTIES, READ_ONLY) - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("common/hadoop/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY); builder.addContainer(workerName, container); } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/MultinodeTlsKerberos.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/MultinodeTlsKerberos.java index b9e6d1f0f70a..78b123876aa9 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/MultinodeTlsKerberos.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/MultinodeTlsKerberos.java @@ -32,6 +32,7 @@ import static com.google.common.base.Verify.verify; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static io.prestosql.tests.product.launcher.env.common.Standard.CONTAINER_PRESTO_CONFIG_PROPERTIES; import static io.prestosql.tests.product.launcher.env.common.Standard.createPrestoContainer; import static java.util.Objects.requireNonNull; @@ -73,7 +74,8 @@ protected void extendEnvironment(Environment.Builder builder) verify(Objects.equals(container.getDockerImageName(), prestoDockerImageName), "Expected image '%s', but is '%s'", prestoDockerImageName, container.getDockerImageName()); container .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/config-master.properties"), CONTAINER_PRESTO_CONFIG_PROPERTIES, READ_ONLY) - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY); }); addPrestoWorker(builder, "presto-worker-1"); @@ -87,7 +89,8 @@ private void addPrestoWorker(Environment.Builder builder, String workerName) .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withDomainName("docker.cluster")) .withNetworkAliases(workerName + ".docker.cluster") .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/config-worker.properties"), CONTAINER_PRESTO_CONFIG_PROPERTIES, READ_ONLY) - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-tls-kerberos/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY); builder.addContainer(workerName, container); } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHdfsImpersonation.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHdfsImpersonation.java index fe741638262c..63731e746610 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHdfsImpersonation.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHdfsImpersonation.java @@ -24,6 +24,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -44,6 +45,7 @@ public SinglenodeHdfsImpersonation(DockerFiles dockerFiles, Standard standard, H protected void extendEnvironment(Environment.Builder builder) { builder.configureContainer("presto-master", container -> container - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hdfs-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY)); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hdfs-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hdfs-impersonation/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY)); } } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHiveImpersonation.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHiveImpersonation.java index 059263761f15..b1a3e3bb85d4 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHiveImpersonation.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeHiveImpersonation.java @@ -24,6 +24,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -44,6 +45,7 @@ public SinglenodeHiveImpersonation(DockerFiles dockerFiles, Standard standard, H protected void extendEnvironment(Environment.Builder builder) { builder.configureContainer("presto-master", container -> container - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hive-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY)); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hive-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-hive-impersonation/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY)); } } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonation.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonation.java index 18631d7a1448..4ca84d2cba9a 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonation.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonation.java @@ -25,6 +25,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -45,6 +46,7 @@ public SinglenodeKerberosHdfsImpersonation(DockerFiles dockerFiles, Standard sta protected void extendEnvironment(Environment.Builder builder) { builder.configureContainer("presto-master", container -> container - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY)); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-impersonation/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY)); } } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonationCrossRealm.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonationCrossRealm.java index 0a8e003dc67a..db010b1ff267 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonationCrossRealm.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonationCrossRealm.java @@ -25,6 +25,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -50,6 +51,10 @@ protected void extendEnvironment(Environment.Builder builder) .withFileSystemBind( dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, + READ_ONLY) + .withFileSystemBind( + dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/iceberg.properties"), + CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY); }); } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonationWithWireEncryption.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonationWithWireEncryption.java index a5a6be82c218..5bb50e684d31 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonationWithWireEncryption.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsImpersonationWithWireEncryption.java @@ -25,6 +25,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -62,7 +63,11 @@ protected void extendEnvironment(Environment.Builder builder) .withFileSystemBind( dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, - READ_ONLY); + READ_ONLY) + .withFileSystemBind( + dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/iceberg.properties"), + CONTAINER_PRESTO_ICEBERG_PROPERTIES, + READ_ONLY); }); } } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsNoImpersonation.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsNoImpersonation.java index 73a853ed0ae2..4d781b31f251 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsNoImpersonation.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHdfsNoImpersonation.java @@ -25,6 +25,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -45,6 +46,7 @@ public SinglenodeKerberosHdfsNoImpersonation(DockerFiles dockerFiles, Standard s protected void extendEnvironment(Environment.Builder builder) { builder.configureContainer("presto-master", container -> container - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY)); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-no-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hdfs-no-impersonation/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY)); } } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHiveImpersonation.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHiveImpersonation.java index 3d9d39c36c89..38422c08d3d8 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHiveImpersonation.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosHiveImpersonation.java @@ -25,6 +25,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -45,6 +46,7 @@ public SinglenodeKerberosHiveImpersonation(DockerFiles dockerFiles, Standard sta protected void extendEnvironment(Environment.Builder builder) { builder.configureContainer("presto-master", container -> container - .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hive-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY)); + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hive-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, READ_ONLY) + .withFileSystemBind(dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-hive-impersonation/iceberg.properties"), CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY)); } } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosKmsHdfsImpersonation.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosKmsHdfsImpersonation.java index b07bf40f9b97..3284bf7dc246 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosKmsHdfsImpersonation.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosKmsHdfsImpersonation.java @@ -26,6 +26,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -63,6 +64,10 @@ protected void extendEnvironment(Environment.Builder builder) .withFileSystemBind( dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-kms-hdfs-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, + READ_ONLY) + .withFileSystemBind( + dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-kms-hdfs-impersonation/iceberg.properties"), + CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY); }); } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosKmsHdfsNoImpersonation.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosKmsHdfsNoImpersonation.java index 04e3847a2df8..b0ec677ac611 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosKmsHdfsNoImpersonation.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/SinglenodeKerberosKmsHdfsNoImpersonation.java @@ -26,6 +26,7 @@ import javax.inject.Inject; import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_HIVE_PROPERTIES; +import static io.prestosql.tests.product.launcher.env.common.Hadoop.CONTAINER_PRESTO_ICEBERG_PROPERTIES; import static java.util.Objects.requireNonNull; import static org.testcontainers.containers.BindMode.READ_ONLY; @@ -51,6 +52,10 @@ protected void extendEnvironment(Environment.Builder builder) .withFileSystemBind( dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/hive.properties"), CONTAINER_PRESTO_HIVE_PROPERTIES, + READ_ONLY) + .withFileSystemBind( + dockerFiles.getDockerFilesHostPath("conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/iceberg.properties"), + CONTAINER_PRESTO_ICEBERG_PROPERTIES, READ_ONLY); }); } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/TwoKerberosHives.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/TwoKerberosHives.java index 838bd9842d1b..0a0687a53ae5 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/TwoKerberosHives.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/TwoKerberosHives.java @@ -102,6 +102,16 @@ protected void extendEnvironment(Environment.Builder builder) .withFileSystemBind( dockerFiles.getDockerFilesHostPath("conf/environment/two-kerberos-hives/hive2.properties"), CONTAINER_PRESTO_ETC + "/catalog/hive2.properties", + READ_ONLY) + + .withFileSystemBind( + dockerFiles.getDockerFilesHostPath("conf/environment/two-kerberos-hives/iceberg1.properties"), + CONTAINER_PRESTO_ETC + "/catalog/iceberg1.properties", + READ_ONLY) + + .withFileSystemBind( + dockerFiles.getDockerFilesHostPath("conf/environment/two-kerberos-hives/iceberg2.properties"), + CONTAINER_PRESTO_ETC + "/catalog/iceberg2.properties", READ_ONLY); }); diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/TwoMixedHives.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/TwoMixedHives.java index c5b6dbe777ad..2e05647a3b5f 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/TwoMixedHives.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/environment/TwoMixedHives.java @@ -73,6 +73,14 @@ protected void extendEnvironment(Environment.Builder builder) dockerFiles.getDockerFilesHostPath("conf/environment/two-mixed-hives/hive2.properties"), CONTAINER_PRESTO_ETC + "/catalog/hive2.properties", READ_ONLY); + container.withFileSystemBind( + dockerFiles.getDockerFilesHostPath("conf/environment/two-mixed-hives/iceberg1.properties"), + CONTAINER_PRESTO_ETC + "/catalog/iceberg1.properties", + READ_ONLY); + container.withFileSystemBind( + dockerFiles.getDockerFilesHostPath("conf/environment/two-mixed-hives/iceberg2.properties"), + CONTAINER_PRESTO_ETC + "/catalog/iceberg2.properties", + READ_ONLY); }); builder.addContainer("hadoop-master-2", createHadoopMaster2()); diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/iceberg.properties new file mode 100644 index 000000000000..7ec547c05f48 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/common/hadoop/iceberg.properties @@ -0,0 +1,4 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls-kerberos/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls-kerberos/iceberg.properties new file mode 100644 index 000000000000..88356e4e91b1 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-tls-kerberos/iceberg.properties @@ -0,0 +1,15 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/hive/conf/hive.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.presto.principal=hdfs/hadoop-master@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/hadoop/conf/hdfs.keytab + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/iceberg.properties new file mode 100644 index 000000000000..6cfd95f0a6eb --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hdfs-impersonation/iceberg.properties @@ -0,0 +1,8 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.hdfs.authentication.type=NONE +hive.hdfs.impersonation.enabled=true + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/iceberg.properties new file mode 100644 index 000000000000..9230951cec25 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-hive-impersonation/iceberg.properties @@ -0,0 +1,8 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml +hive.metastore.thrift.impersonation.enabled=true +hive.hdfs.authentication.type=NONE +hive.hdfs.impersonation.enabled=true + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/iceberg.properties new file mode 100644 index 000000000000..1b7fe714c478 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-cross-realm/iceberg.properties @@ -0,0 +1,22 @@ +# +# WARNING +# ^^^^^^^ +# This configuration file is for development only and should NOT be used +# in production. For example configuration, see the Presto documentation. +# + +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/hadoop-master@OTHERLABS.TERADATA.COM +hive.metastore.client.keytab=/etc/hive/conf/hive-other.keytab +hive.config.resources = /etc/hadoop/conf/core-site.xml + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.presto.principal=hdfs/hadoop-master@OTHERLABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/hadoop/conf/hdfs-other.keytab + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/iceberg.properties new file mode 100644 index 000000000000..e42fd3abf7cc --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation-with-wire-encryption/iceberg.properties @@ -0,0 +1,17 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/presto/conf/hive-presto-master.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.presto.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/presto/conf/presto-server.keytab + +hive.hdfs.wire-encryption.enabled=true + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/iceberg.properties new file mode 100644 index 000000000000..ff4cc8f826cb --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-impersonation/iceberg.properties @@ -0,0 +1,15 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/presto/conf/hive-presto-master.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.presto.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/presto/conf/presto-server.keytab + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/iceberg.properties new file mode 100644 index 000000000000..88356e4e91b1 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hdfs-no-impersonation/iceberg.properties @@ -0,0 +1,15 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/hive/conf/hive.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.presto.principal=hdfs/hadoop-master@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/hadoop/conf/hdfs.keytab + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/iceberg.properties new file mode 100644 index 000000000000..fbf1e30e1d69 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-hive-impersonation/iceberg.properties @@ -0,0 +1,16 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.thrift.impersonation.enabled=true +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/presto/conf/hive-presto-master.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.presto.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/presto/conf/presto-server.keytab + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/iceberg.properties new file mode 100644 index 000000000000..e131a1e9b19b --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/iceberg.properties @@ -0,0 +1,21 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +# When using HDFS impersonation we talk to HDFS as session user configured in tempto (which is 'hive') +# However, TODO we don't have Metastore impersonation yet; when DROP TABLE is issued, +# Metastore needs to be able to delete the table files on HDFS, otherwise they be silently left behind, +# potentially causing some further tests to fail. For this reason, `hive.metastore.client.principal` needs +# to match session user configured in tempto. +hive.metastore.client.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/hive/conf/hive.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.presto.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/presto/conf/presto-server.keytab +hive.fs.cache.max-size=10 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-impersonation/hive-disable-key-provider-cache-site.xml + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/iceberg.properties new file mode 100644 index 000000000000..3663d9507964 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/iceberg.properties @@ -0,0 +1,16 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/presto/conf/presto-server.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=false +hive.hdfs.presto.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/presto/conf/presto-server.keytab +hive.fs.cache.max-size=10 +hive.config.resources=/etc/hadoop/conf/core-site.xml,/docker/presto-product-tests/conf/environment/singlenode-kerberos-kms-hdfs-no-impersonation/hive-disable-key-provider-cache-site.xml + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/iceberg1.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/iceberg1.properties new file mode 100644 index 000000000000..fbf1e30e1d69 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/iceberg1.properties @@ -0,0 +1,16 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.thrift.impersonation.enabled=true +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/presto/conf/hive-presto-master.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.presto.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/presto/conf/presto-server.keytab + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/iceberg2.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/iceberg2.properties new file mode 100644 index 000000000000..77c782d17633 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-kerberos-hives/iceberg2.properties @@ -0,0 +1,15 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master-2:9083 +hive.config.resources=/docker/presto-product-tests/conf/environment/two-kerberos-hives/hive2-default-fs-site.xml,\ + /docker/presto-product-tests/conf/environment/two-kerberos-hives/auth-to-local.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.service.principal=hive/hadoop-master-2@OTHERREALM.COM +hive.metastore.client.principal=hive/_HOST@OTHERREALM.COM +hive.metastore.client.keytab=/etc/presto/conf/other-hive-presto-master.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.presto.principal=presto-server/_HOST@OTHERREALM.COM +hive.hdfs.presto.keytab=/etc/presto/conf/other-presto-server.keytab + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/iceberg1.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/iceberg1.properties new file mode 100644 index 000000000000..fbf1e30e1d69 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/iceberg1.properties @@ -0,0 +1,16 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master:9083 +hive.config.resources=/docker/presto-product-tests/conf/presto/etc/hive-default-fs-site.xml + +hive.metastore.authentication.type=KERBEROS +hive.metastore.thrift.impersonation.enabled=true +hive.metastore.service.principal=hive/hadoop-master@LABS.TERADATA.COM +hive.metastore.client.principal=hive/_HOST@LABS.TERADATA.COM +hive.metastore.client.keytab=/etc/presto/conf/hive-presto-master.keytab + +hive.hdfs.authentication.type=KERBEROS +hive.hdfs.impersonation.enabled=true +hive.hdfs.presto.principal=presto-server/_HOST@LABS.TERADATA.COM +hive.hdfs.presto.keytab=/etc/presto/conf/presto-server.keytab + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/iceberg2.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/iceberg2.properties new file mode 100644 index 000000000000..318b918a4a61 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/two-mixed-hives/iceberg2.properties @@ -0,0 +1,5 @@ +connector.name=iceberg +hive.metastore.uri=thrift://hadoop-master-2:9083 +hive.config.resources=/docker/presto-product-tests/conf/environment/two-mixed-hives/hive2-default-fs-site.xml + +iceberg.file-format=PARQUET diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg.properties new file mode 100644 index 000000000000..a2ff2918dee5 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg.properties @@ -0,0 +1,3 @@ +# This file exists so that we can install iceberg connector only in tests using it. +# Docker does not allow to re-bind a file inside a folder that is also a volume, unless the file already exists. +connector.name=noop diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg1.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg1.properties new file mode 100644 index 000000000000..a2ff2918dee5 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg1.properties @@ -0,0 +1,3 @@ +# This file exists so that we can install iceberg connector only in tests using it. +# Docker does not allow to re-bind a file inside a folder that is also a volume, unless the file already exists. +connector.name=noop diff --git a/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg2.properties b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg2.properties new file mode 100644 index 000000000000..a2ff2918dee5 --- /dev/null +++ b/presto-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/presto/etc/catalog/iceberg2.properties @@ -0,0 +1,3 @@ +# This file exists so that we can install iceberg connector only in tests using it. +# Docker does not allow to re-bind a file inside a folder that is also a volume, unless the file already exists. +connector.name=noop diff --git a/presto-product-tests/src/main/java/io/prestosql/tests/TestGroups.java b/presto-product-tests/src/main/java/io/prestosql/tests/TestGroups.java index 2f320c08737b..2ca7d2e1a864 100644 --- a/presto-product-tests/src/main/java/io/prestosql/tests/TestGroups.java +++ b/presto-product-tests/src/main/java/io/prestosql/tests/TestGroups.java @@ -66,6 +66,7 @@ public final class TestGroups public static final String BIG_QUERY = "big_query"; public static final String KAFKA = "kafka"; public static final String TWO_HIVES = "two_hives"; + public static final String ICEBERG = "iceberg"; private TestGroups() {} } diff --git a/presto-product-tests/src/main/java/io/prestosql/tests/iceberg/TestIcebergCreateTable.java b/presto-product-tests/src/main/java/io/prestosql/tests/iceberg/TestIcebergCreateTable.java new file mode 100644 index 000000000000..4037d2b3a417 --- /dev/null +++ b/presto-product-tests/src/main/java/io/prestosql/tests/iceberg/TestIcebergCreateTable.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.tests.iceberg; + +import io.prestosql.tempto.ProductTest; +import io.prestosql.tempto.query.QueryExecutor; +import org.testng.annotations.Test; + +import java.sql.SQLException; + +import static io.prestosql.tempto.assertions.QueryAssert.Row.row; +import static io.prestosql.tempto.assertions.QueryAssert.assertThat; +import static io.prestosql.tests.TestGroups.ICEBERG; +import static io.prestosql.tests.TestGroups.STORAGE_FORMATS; +import static io.prestosql.tests.utils.QueryExecutors.onPresto; + +public class TestIcebergCreateTable + extends ProductTest +{ + @Test(groups = {ICEBERG, STORAGE_FORMATS}) + public void testCreateTable() + throws SQLException + { + QueryExecutor queryExecutor = onPresto(); + queryExecutor.executeQuery("CREATE SCHEMA iceberg.iceberg"); + queryExecutor.executeQuery("use iceberg.iceberg"); + queryExecutor.executeQuery("CREATE TABLE test_create_table(a bigint, b varchar)"); + queryExecutor.executeQuery("INSERT INTO test_create_table(a, b) VALUES " + + "(NULL, NULL), " + + "(-42, 'abc'), " + + "(9223372036854775807, 'abcdefghijklmnopqrstuvwxyz')"); + assertThat(queryExecutor.executeQuery("SELECT * FROM test_create_table")) + .containsOnly( + row(null, null), + row(-42, "abc"), + row(9223372036854775807L, "abcdefghijklmnopqrstuvwxyz")); + queryExecutor.executeQuery("DROP TABLE test_create_table"); + queryExecutor.executeQuery("DROP SCHEMA iceberg.iceberg"); + } + + @Test(groups = {ICEBERG, STORAGE_FORMATS}) + public void testCreateTableAsSelect() + throws SQLException + { + QueryExecutor queryExecutor = onPresto(); + queryExecutor.executeQuery("CREATE SCHEMA iceberg.iceberg"); + queryExecutor.executeQuery("use iceberg.iceberg"); + queryExecutor.executeQuery("" + + "CREATE TABLE test_create_table_as_select AS " + + "SELECT * FROM (VALUES " + + " (NULL, NULL), " + + " (-42, 'abc'), " + + " (9223372036854775807, 'abcdefghijklmnopqrstuvwxyz')" + + ") t(a, b)"); + assertThat(queryExecutor.executeQuery("SELECT * FROM test_create_table_as_select")) + .containsOnly( + row(null, null), + row(-42, "abc"), + row(9223372036854775807L, "abcdefghijklmnopqrstuvwxyz")); + queryExecutor.executeQuery("DROP TABLE test_create_table_as_select"); + queryExecutor.executeQuery("DROP SCHEMA iceberg.iceberg"); + } +} From 01e4286bb1e3098766b8e2008da4d4a9c8a56953 Mon Sep 17 00:00:00 2001 From: Vlad Rozov Date: Fri, 20 Mar 2020 08:50:03 -0700 Subject: [PATCH 2/2] Temporarily exclude iceberg from Kerberos product tests --- presto-product-tests/bin/product-tests-suite-2.sh | 4 ++-- presto-product-tests/bin/product-tests-suite-3.sh | 2 +- presto-product-tests/bin/product-tests-suite-5.sh | 2 +- presto-product-tests/bin/product-tests-suite-6-non-generic.sh | 4 ++-- presto-product-tests/bin/product-tests-suite-7-non-generic.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/presto-product-tests/bin/product-tests-suite-2.sh b/presto-product-tests/bin/product-tests-suite-2.sh index aa1547f7722e..5ac0c2402421 100755 --- a/presto-product-tests/bin/product-tests-suite-2.sh +++ b/presto-product-tests/bin/product-tests-suite-2.sh @@ -11,7 +11,7 @@ presto-product-tests-launcher/bin/run-launcher test run \ presto-product-tests-launcher/bin/run-launcher test run \ --environment singlenode-kerberos-hdfs-no-impersonation \ - -- -g storage_formats,hdfs_no_impersonation \ + -- -g storage_formats,hdfs_no_impersonation -x iceberg \ || suite_exit_code=1 presto-product-tests-launcher/bin/run-launcher test run \ @@ -21,7 +21,7 @@ presto-product-tests-launcher/bin/run-launcher test run \ presto-product-tests-launcher/bin/run-launcher test run \ --environment singlenode-kerberos-hdfs-impersonation \ - -- -g storage_formats,cli,hdfs_impersonation,authorization,hive_file_header \ + -- -g storage_formats,cli,hdfs_impersonation,authorization,hive_file_header -x iceberg \ || suite_exit_code=1 echo "$0: exiting with ${suite_exit_code}" diff --git a/presto-product-tests/bin/product-tests-suite-3.sh b/presto-product-tests/bin/product-tests-suite-3.sh index d73104cf7d3f..cdfa7ec6b7e9 100755 --- a/presto-product-tests/bin/product-tests-suite-3.sh +++ b/presto-product-tests/bin/product-tests-suite-3.sh @@ -16,7 +16,7 @@ presto-product-tests-launcher/bin/run-launcher test run \ presto-product-tests-launcher/bin/run-launcher test run \ --environment singlenode-kerberos-hdfs-impersonation-with-wire-encryption \ - -- -g storage_formats,cli,hdfs_impersonation,authorization \ + -- -g storage_formats,cli,hdfs_impersonation,authorization -x iceberg \ || suite_exit_code=1 echo "$0: exiting with ${suite_exit_code}" diff --git a/presto-product-tests/bin/product-tests-suite-5.sh b/presto-product-tests/bin/product-tests-suite-5.sh index d277c8611e5f..e80767791af3 100755 --- a/presto-product-tests/bin/product-tests-suite-5.sh +++ b/presto-product-tests/bin/product-tests-suite-5.sh @@ -11,7 +11,7 @@ presto-product-tests-launcher/bin/run-launcher test run \ presto-product-tests-launcher/bin/run-launcher test run \ --environment singlenode-kerberos-hive-impersonation \ - -- -g storage_formats,hdfs_impersonation,authorization \ + -- -g storage_formats,hdfs_impersonation,authorization -x iceberg \ || suite_exit_code=1 echo "$0: exiting with ${suite_exit_code}" diff --git a/presto-product-tests/bin/product-tests-suite-6-non-generic.sh b/presto-product-tests/bin/product-tests-suite-6-non-generic.sh index d90c243bcd38..9eab67f9f34a 100755 --- a/presto-product-tests/bin/product-tests-suite-6-non-generic.sh +++ b/presto-product-tests/bin/product-tests-suite-6-non-generic.sh @@ -26,13 +26,13 @@ presto-product-tests-launcher/bin/run-launcher test run \ # We have docker images with KMS on CDH only. TODO (https://github.com/prestosql/presto/issues/1652) create images with HDP and KMS presto-product-tests-launcher/bin/run-launcher test run \ --environment singlenode-kerberos-kms-hdfs-no-impersonation \ - -- -g storage_formats \ + -- -g storage_formats -x iceberg \ || suite_exit_code=1 # We have docker images with KMS on CDH only. TODO (https://github.com/prestosql/presto/issues/1652) create images with HDP and KMS presto-product-tests-launcher/bin/run-launcher test run \ --environment singlenode-kerberos-kms-hdfs-impersonation \ - -- -g storage_formats \ + -- -g storage_formats -x iceberg \ || suite_exit_code=1 presto-product-tests-launcher/bin/run-launcher test run \ diff --git a/presto-product-tests/bin/product-tests-suite-7-non-generic.sh b/presto-product-tests/bin/product-tests-suite-7-non-generic.sh index ddbe51209891..864a69348537 100755 --- a/presto-product-tests/bin/product-tests-suite-7-non-generic.sh +++ b/presto-product-tests/bin/product-tests-suite-7-non-generic.sh @@ -33,7 +33,7 @@ presto-product-tests-launcher/bin/run-launcher test run \ # Environment not set up on CDH. (TODO run on HDP 2.6 and HDP 3.1) presto-product-tests-launcher/bin/run-launcher test run \ --environment singlenode-kerberos-hdfs-impersonation-cross-realm \ - -- -g storage_formats,cli,hdfs_impersonation \ + -- -g storage_formats,cli,hdfs_impersonation -x iceberg \ || suite_exit_code=1 presto-product-tests-launcher/bin/run-launcher test run \