diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java index 172fb8c560876..ff729cd1cb6dc 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java +++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java @@ -21,6 +21,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Helper methods for command builders. @@ -30,6 +33,11 @@ class CommandBuilderUtils { static final String DEFAULT_MEM = "1g"; static final String DEFAULT_PROPERTIES_FILE = "spark-defaults.conf"; static final String ENV_SPARK_HOME = "SPARK_HOME"; + // This should be consistent with org.apache.spark.internal.config.SECRET_REDACTION_PATTERN + // We maintain this copy to avoid depending on `core` module. + static final String SECRET_REDACTION_PATTERN = "(?i)secret|password|token|access[.]?key"; + static final Pattern redactPattern = Pattern.compile(SECRET_REDACTION_PATTERN); + static final Pattern keyValuePattern = Pattern.compile("-D(.+?)=(.+)"); /** Returns whether the given string is null or empty. */ static boolean isEmpty(String s) { @@ -328,4 +336,23 @@ static String findJarsDir(String sparkHome, String scalaVersion, boolean failIfN return libdir.getAbsolutePath(); } + /** + * Redact a command-line argument's value part which matches `-Dkey=value` pattern. + * Note that this should be consistent with `org.apache.spark.util.Utils.redactCommandLineArgs`. + */ + static List redactCommandLineArgs(List args) { + return args.stream().map(CommandBuilderUtils::redact).collect(Collectors.toList()); + } + + /** + * Redact a command-line argument's value part which matches `-Dkey=value` pattern. + */ + static String redact(String arg) { + Matcher m = keyValuePattern.matcher(arg); + if (m.find() && redactPattern.matcher(m.group(1)).find()) { + return String.format("-D%s=%s", m.group(1), "*********(redacted)"); + } else { + return arg; + } + } } diff --git a/launcher/src/main/java/org/apache/spark/launcher/Main.java b/launcher/src/main/java/org/apache/spark/launcher/Main.java index 6501fc1764c25..321fca0912704 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/Main.java +++ b/launcher/src/main/java/org/apache/spark/launcher/Main.java @@ -114,7 +114,7 @@ private static List buildCommand( boolean printLaunchCommand) throws IOException, IllegalArgumentException { List cmd = builder.buildCommand(env); if (printLaunchCommand) { - System.err.println("Spark Command: " + join(" ", cmd)); + System.err.println("Spark Command: " + join(" ", redactCommandLineArgs(cmd))); System.err.println("========================================"); } return cmd; diff --git a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java index 46cdffc190d52..1b2c683880c25 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java @@ -68,6 +68,16 @@ public void testInvalidOptionStrings() { testInvalidOpt("'abcde"); } + @Test + public void testRedactCommandLineArgs() { + assertEquals(redact("secret"), "secret"); + assertEquals(redact("-Dk=v"), "-Dk=v"); + assertEquals(redact("-Dk=secret"), "-Dk=secret"); + assertEquals(redact("-DsecretKey=my-secret"), "-DsecretKey=*********(redacted)"); + assertEquals(redactCommandLineArgs(Arrays.asList("-DsecretKey=my-secret")), + Arrays.asList("-DsecretKey=*********(redacted)")); + } + @Test public void testWindowsBatchQuoting() { assertEquals("abc", quoteForBatchScript("abc"));