diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 8c024901352fc..5eec6c490cb1f 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -490,7 +490,6 @@ To use a custom metrics.properties for the application master and executors, upd
spark.yarn.am.tokenConfRegex |
(none) |
- This config is only supported when Hadoop version is 2.9+ or 3.x (e.g., when using the Hadoop 3.x profile).
The value of this config is a regex expression used to grep a list of config entries from the job's configuration file (e.g., hdfs-site.xml)
and send to RM, which uses them when renewing delegation tokens. A typical use case of this feature is to support delegation
tokens in an environment where a YARN cluster needs to talk to multiple downstream HDFS clusters, where the YARN RM may not have configs
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 7010067e1ae03..4619d65f136d5 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -38,7 +38,6 @@ import org.apache.hadoop.io.{DataOutputBuffer, Text}
import org.apache.hadoop.mapreduce.MRJobConfig
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.util.StringUtils
-import org.apache.hadoop.util.VersionInfo
import org.apache.hadoop.yarn.api._
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
import org.apache.hadoop.yarn.api.protocolrecords._
@@ -62,7 +61,7 @@ import org.apache.spark.internal.config.Python._
import org.apache.spark.launcher.{JavaModuleOptions, LauncherBackend, SparkAppHandle, YarnCommandBuilderUtils}
import org.apache.spark.resource.ResourceProfile
import org.apache.spark.rpc.RpcEnv
-import org.apache.spark.util.{CallerContext, Utils, VersionUtils, YarnContainerInfoHelper}
+import org.apache.spark.util.{CallerContext, Utils, YarnContainerInfoHelper}
private[spark] class Client(
val args: ClientArguments,
@@ -358,20 +357,13 @@ private[spark] class Client(
private def setTokenConf(amContainer: ContainerLaunchContext): Unit = {
// SPARK-37205: this regex is used to grep a list of configurations and send them to YARN RM
// for fetching delegation tokens. See YARN-5910 for more details.
- val regex = sparkConf.get(config.AM_TOKEN_CONF_REGEX)
- // The feature is only supported in Hadoop 2.9+ and 3.x, hence the check below.
- val isSupported = VersionUtils.majorMinorVersion(VersionInfo.getVersion) match {
- case (2, n) if n >= 9 => true
- case (3, _) => true
- case _ => false
- }
- if (regex.nonEmpty && isSupported) {
+ sparkConf.get(config.AM_TOKEN_CONF_REGEX).foreach { regex =>
logInfo(s"Processing token conf (spark.yarn.am.tokenConfRegex) with regex $regex")
- val dob = new DataOutputBuffer();
- val copy = new Configuration(false);
- copy.clear();
+ val dob = new DataOutputBuffer()
+ val copy = new Configuration(false)
+ copy.clear()
hadoopConf.asScala.foreach { entry =>
- if (entry.getKey.matches(regex.get)) {
+ if (entry.getKey.matches(regex)) {
copy.set(entry.getKey, entry.getValue)
logInfo(s"Captured key: ${entry.getKey} -> value: ${entry.getValue}")
}
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
index 46f78c17429b3..96ebc03bf163e 100644
--- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
+++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
@@ -80,8 +80,7 @@ package object config extends Logging {
private[spark] val AM_TOKEN_CONF_REGEX =
ConfigBuilder("spark.yarn.am.tokenConfRegex")
- .doc("This config is only supported when Hadoop version is 2.9+ or 3.x (e.g., when using " +
- "the Hadoop 3.x profile). The value of this config is a regex expression used to grep a " +
+ .doc("The value of this config is a regex expression used to grep a " +
"list of config entries from the job's configuration file (e.g., hdfs-site.xml) and send " +
"to RM, which uses them when renewing delegation tokens. A typical use case of this " +
"feature is to support delegation tokens in an environment where a YARN cluster needs to " +
|