From f8413a20e63ce2330ade76b3cb924ef1dc90be5e Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 26 Sep 2024 18:38:28 +0800 Subject: [PATCH 1/7] (chore)(test)Adding Kerberos DNS check to the Hive connection case --- .../kerberos/test_two_hive_kerberos.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index 2c94c08a6ed29c..201b26247ffa93 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -24,6 +24,9 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_two_hive_kerberos" sql """drop catalog if exists ${hms_catalog_name};""" + println 'check hadoop-master:88 dns' + def hadoopMaster88DnsCheckresult = "nslookup hadoop-master:88".execute().text + println hadoopMaster88DnsCheckresult sql """ CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} PROPERTIES ( @@ -40,6 +43,9 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d """ sql """drop catalog if exists other_${hms_catalog_name};""" + println 'check hadoop-master-2:88 dns' + def hadoopMaster288Dnsresult = "nslookup hadoop-master-2:88".execute().text + println hadoopMaster288Dnsresult sql """ CREATE CATALOG IF NOT EXISTS other_${hms_catalog_name} PROPERTIES ( From 3bf400b641e3f0921ba811044d70208a81821ca9 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 26 Sep 2024 20:45:26 +0800 Subject: [PATCH 2/7] (chore)(test)Adding Kerberos DNS check to the Hive connection case --- .../kerberos/test_single_hive_kerberos.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index dfe5532fefdb44..a23e3ef190afb7 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -20,6 +20,9 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_single_hive_kerberos" sql """drop catalog if exists hms_kerberos;""" + println 'check hadoop-master:88 dns' + def hadoopMaster88Dnsresult = "nslookup hadoop-master:88".execute().text + println hadoopMaster88Dnsresult sql """ CREATE CATALOG IF NOT EXISTS hms_kerberos PROPERTIES ( From 40763a53199e0a503eebd4e4bae5f5b69c7d521b Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 27 Sep 2024 15:07:50 +0800 Subject: [PATCH 3/7] test --- .../kerberos/test_single_hive_kerberos.groovy | 3 --- .../kerberos/test_two_hive_kerberos.groovy | 9 +++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index a23e3ef190afb7..dfe5532fefdb44 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -20,9 +20,6 @@ suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,externa if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_single_hive_kerberos" sql """drop catalog if exists hms_kerberos;""" - println 'check hadoop-master:88 dns' - def hadoopMaster88Dnsresult = "nslookup hadoop-master:88".execute().text - println hadoopMaster88Dnsresult sql """ CREATE CATALOG IF NOT EXISTS hms_kerberos PROPERTIES ( diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index 201b26247ffa93..f38769a01e5ea8 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -24,9 +24,6 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_two_hive_kerberos" sql """drop catalog if exists ${hms_catalog_name};""" - println 'check hadoop-master:88 dns' - def hadoopMaster88DnsCheckresult = "nslookup hadoop-master:88".execute().text - println hadoopMaster88DnsCheckresult sql """ CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} PROPERTIES ( @@ -38,7 +35,11 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@LABS.TERADATA.COM", "hadoop.kerberos.keytab" = "/keytabs/hive-presto-master.keytab", "hive.metastore.sasl.enabled " = "true", - "hive.metastore.kerberos.principal" = "hive/_HOST@LABS.TERADATA.COM" + "hive.metastore.kerberos.principal" = "hive/_HOST@LABS.TERADATA.COM", + "hadoop.security.auth_to_local" = "RULE:[2:\$1@\$0](.*@LABS.TERADATA.COM)s/@.*// + RULE:[2:\$1@\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*// + DEFAULT", ); """ From 3460ea1de4f78db5abd1ece33ca13092b241ee17 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 27 Sep 2024 16:49:32 +0800 Subject: [PATCH 4/7] test --- .../kerberos/test_two_hive_kerberos.groovy | 55 +++++++++---------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index f38769a01e5ea8..cbdb085787a085 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -23,6 +23,25 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d String enabled = context.config.otherConfigs.get("enableKerberosTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_two_hive_kerberos" + sql """drop catalog if exists other_${hms_catalog_name};""" + + sql """ + CREATE CATALOG IF NOT EXISTS other_${hms_catalog_name} + PROPERTIES ( + "type" = "hms", + "hive.metastore.uris" = "thrift://172.31.71.26:9083", + "fs.defaultFS" = "hdfs://172.31.71.26:8020", + "hadoop.kerberos.min.seconds.before.relogin" = "5", + "hadoop.security.authentication" = "kerberos", + "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@OTHERREALM.COM", + "hadoop.kerberos.keytab" = "/keytabs/other-hive-presto-master.keytab", + "hive.metastore.sasl.enabled " = "true", + "hive.metastore.kerberos.principal" = "hive/_HOST@OTHERREALM.COM", + "hadoop.security.auth_to_local" ="RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*// + RULE:[2:\$1@\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// + DEFAULT" + ); + """ sql """drop catalog if exists ${hms_catalog_name};""" sql """ CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} @@ -43,34 +62,9 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d ); """ - sql """drop catalog if exists other_${hms_catalog_name};""" - println 'check hadoop-master-2:88 dns' - def hadoopMaster288Dnsresult = "nslookup hadoop-master-2:88".execute().text - println hadoopMaster288Dnsresult - sql """ - CREATE CATALOG IF NOT EXISTS other_${hms_catalog_name} - PROPERTIES ( - "type" = "hms", - "hive.metastore.uris" = "thrift://172.31.71.26:9083", - "fs.defaultFS" = "hdfs://172.31.71.26:8020", - "hadoop.kerberos.min.seconds.before.relogin" = "5", - "hadoop.security.authentication" = "kerberos", - "hadoop.kerberos.principal"="hive/presto-master.docker.cluster@OTHERREALM.COM", - "hadoop.kerberos.keytab" = "/keytabs/other-hive-presto-master.keytab", - "hive.metastore.sasl.enabled " = "true", - "hive.metastore.kerberos.principal" = "hive/_HOST@OTHERREALM.COM", - "hadoop.security.auth_to_local" ="RULE:[2:\$1@\$0](.*@OTHERREALM.COM)s/@.*// - RULE:[2:\$1@\$0](.*@OTHERLABS.TERADATA.COM)s/@.*// - DEFAULT" - ); - """ + - // 1. catalogA - sql """switch ${hms_catalog_name};""" - logger.info("switched to catalog " + hms_catalog_name) - sql """ show databases """ - sql """ use test_krb_hive_db """ - order_qt_q01 """ select * from test_krb_hive_db.test_krb_hive_tbl """ + // 2. catalogB sql """switch other_${hms_catalog_name};""" @@ -78,7 +72,12 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d sql """ show databases """ sql """ use test_krb_hive_db """ order_qt_q02 """ select * from test_krb_hive_db.test_krb_hive_tbl """ - + // 1. catalogA + sql """switch ${hms_catalog_name};""" + logger.info("switched to catalog " + hms_catalog_name) + sql """ show databases """ + sql """ use test_krb_hive_db """ + order_qt_q01 """ select * from test_krb_hive_db.test_krb_hive_tbl """ // 3. write back test case sql """ switch ${hms_catalog_name}; """ sql """ CREATE DATABASE IF NOT EXISTS `test_krb_hms_db`; """ From 742c48f63de3fc515a2e7e4d468dc65d3ed6277e Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sun, 29 Sep 2024 10:03:42 +0800 Subject: [PATCH 5/7] test --- fe/fe-core/src/main/java/org/apache/doris/DorisFE.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index d028f3aeae1437..f50aab1a6749d7 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -81,6 +81,7 @@ public class DorisFE { private static FileLock processFileLock; public static void main(String[] args) { + System.setProperty("sun.security.krb5.debug", "true"); // Every doris version should have a final meta version, it should not change // between small releases. Add a check here to avoid mistake. if (Version.DORIS_FE_META_VERSION > 0 From 7c87a26b5cdfb78b92bd85f0611784a8a6f1b843 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sun, 29 Sep 2024 14:44:47 +0800 Subject: [PATCH 6/7] add docs --- .../kerberos/test_two_hive_kerberos.groovy | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index cbdb085787a085..dc2c1246b97e4d 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -18,8 +18,23 @@ import groovyjarjarantlr4.v4.codegen.model.ExceptionClause // under the License. import org.junit.Assert; - +import java.net.Socket suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") { + def kdcServers = [ + [host: 'hadoop-master', port: 88], + [host: 'hadoop-master-2', port: 88], + [host: 'hadoop-master', port: 89] + ] + + kdcServers.each { server -> + try { + def socket = new Socket(server.host, server.port) + socket.close() + println "KDC server at ${server.host}:${server.port} is reachable" + } catch (Exception e) { + println "Failed to reach KDC server at ${server.host}:${server.port}: ${e.message}" + } + } String enabled = context.config.otherConfigs.get("enableKerberosTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_two_hive_kerberos" From aca6c5edf0d25446a0ca1e0442a0e8a09162c655 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 30 Sep 2024 16:48:48 +0800 Subject: [PATCH 7/7] test --- .../kerberos/test_single_hive_kerberos.groovy | 8 +++++++ .../kerberos/test_two_hive_kerberos.groovy | 23 +++++++------------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy index dfe5532fefdb44..505c5208c9988c 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy @@ -16,6 +16,14 @@ // under the License. suite("test_single_hive_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") { + def command = "sudo docker ps" + def process = command.execute() + process.waitFor() + + def output = process.in.text + + println "Docker containers:" + println output String enabled = context.config.otherConfigs.get("enableKerberosTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_single_hive_kerberos" diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index dc2c1246b97e4d..f5036c2f7d53e5 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -20,21 +20,14 @@ import groovyjarjarantlr4.v4.codegen.model.ExceptionClause import org.junit.Assert; import java.net.Socket suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") { - def kdcServers = [ - [host: 'hadoop-master', port: 88], - [host: 'hadoop-master-2', port: 88], - [host: 'hadoop-master', port: 89] - ] - - kdcServers.each { server -> - try { - def socket = new Socket(server.host, server.port) - socket.close() - println "KDC server at ${server.host}:${server.port} is reachable" - } catch (Exception e) { - println "Failed to reach KDC server at ${server.host}:${server.port}: ${e.message}" - } - } + def command = "sudo docker ps" + def process = command.execute() + process.waitFor() + + def output = process.in.text + + println "Docker containers:" + println output String enabled = context.config.otherConfigs.get("enableKerberosTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { String hms_catalog_name = "test_two_hive_kerberos"