Skip to content

Commit

Permalink
### What changes were proposed in this pull request?
Browse files Browse the repository at this point in the history
The pr is add AVA_OPTION_PURE_MODE configuration item. When the configuration item is set to true, JavaOptions cannot be configured.

### Why are the changes needed?
Adding JAVA_OPTION_PURE_MODE can completely prevent command injection caused by Java parameters in Yarn mode. This is more secure when JVM parameters do not need to be customized.

### Does this PR introduce _any_ user-facing change?
Yes. If JAVA_OPTION_PURE_MODE is set to true, JavaOptions cannot be set.

### How was this patch tested?
Added a new UT.

### Was this patch authored or co-authored using generative AI tooling?
No.
  • Loading branch information
cliffchen12 committed Oct 15, 2024
1 parent c3176a7 commit 04e9b19
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,12 @@ package object config {
.stringConf
.createOptional

private[spark] val JAVA_OPTION_PURE_MODE =
ConfigBuilder("spark.executor.javaOptionPureMode")
.version("3.5.1")
.booleanConf
.createWithDefault(false)

private[spark] val EXECUTOR_LIBRARY_PATH =
ConfigBuilder(SparkLauncher.EXECUTOR_EXTRA_LIBRARY_PATH)
.version("1.0.0")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ private[spark] class Client(
*/
def submitApplication(): Unit = {
ResourceRequestHelper.validateResources(sparkConf)
ResourceRequestHelper.validateJavaOptions(sparkConf)

try {
launcherBackend.connect()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,28 @@ private object ResourceRequestHelper extends Logging {
}
}

/**
* Validates sparkConf and throw a IllegalArgumentException if JAVA_OPTION_PURE_MODE is
* true
* @param sparkConf
*/
def validateJavaOptions(sparkConf: SparkConf): Unit = {
if (sparkConf.get(JAVA_OPTION_PURE_MODE)) {
val javaOptionsToCheck = List(
"spark.driver.defaultJavaOptions",
"spark.driver.extraJavaOptions",
"spark.executor.defaultJavaOptions",
"spark.executor.extraJavaOptions"
).foreach { option =>
if (sparkConf.getOption(option).isDefined) {
val msg = s"$option parameters cannot be set, because the pure mode config " +
s"spark.executor.javaOptionPureMode is enabled. "
throw new IllegalArgumentException(msg)
}
}
}
}

/**
* Sets resource amount with the corresponding unit to the passed resource object.
* @param resources resource values to set
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -671,6 +671,34 @@ class ClientSuite extends SparkFunSuite
assertUserClasspathUrls(cluster = true, replacementRootPath)
}

test("SPARK-49923: test JAVA_OPTION_PURE_MODE enabled, JVM parameters cannot be set"){
val sparkConf = new SparkConf()

sparkConf.set("spark.driver.defaultJavaOptions", "-Xmx1024m")
.set("spark.executor.defaultJavaOptions", "-Xmx1024m")
.set("spark.driver.extraJavaOptions", "-Xmx1024m")
.set("spark.executor.extraJavaOptions", "-Xmx1024m")
.set(JAVA_OPTION_PURE_MODE, true)

intercept[IllegalArgumentException] {
ResourceRequestHelper.validateJavaOptions(sparkConf)
}
}

test("SPARK-49923: test JAVA_OPTION_PURE_MODE disabled, JVM parameters can be set"){
val sparkConf = new SparkConf()

sparkConf.set("spark.driver.defaultJavaOptions", "-Xmx1024m")
.set("spark.executor.defaultJavaOptions", "-Xmx1024m")
.set("spark.driver.extraJavaOptions", "-Xmx1024m")
.set("spark.executor.extraJavaOptions", "-Xmx1024m")
.set(JAVA_OPTION_PURE_MODE, false)

noException should be thrownBy {
ResourceRequestHelper.validateJavaOptions(sparkConf)
}
}

test("SPARK-44306: test directoriesToBePreloaded") {
val sparkConf = new SparkConf()
.set(YARN_CLIENT_STAT_CACHE_PRELOAD_PER_DIRECTORY_THRESHOLD, 3)
Expand Down

0 comments on commit 04e9b19

Please sign in to comment.