From 009bad387c1ef90656f179b0bf96261c6ac2477b Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 14 Jul 2020 11:16:12 -0700 Subject: [PATCH 01/76] added unit test showing leaked password --- pom.xml | 6 ++ .../plugins/workflow/cps/DSLTest.java | 74 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/pom.xml b/pom.xml index 6a69208c3..bbae2c201 100644 --- a/pom.xml +++ b/pom.xml @@ -256,6 +256,12 @@ 2.3 test + + org.jenkins-ci.plugins + credentials-binding + 1.23 + test + diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 9ec99a78c..8e611172c 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -24,6 +24,11 @@ package org.jenkinsci.plugins.workflow.cps; +import com.cloudbees.plugins.credentials.CredentialsProvider; +import com.cloudbees.plugins.credentials.CredentialsScope; +import com.cloudbees.plugins.credentials.domains.Domain; +import com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl; +import hudson.Functions; import hudson.model.AbstractDescribableImpl; import hudson.model.Descriptor; import hudson.model.Result; @@ -46,6 +51,7 @@ import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import org.jenkinsci.plugins.workflow.steps.StepExecution; import org.jenkinsci.plugins.workflow.steps.SynchronousStepExecution; +import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep; import org.jenkinsci.plugins.workflow.testMetaStep.AmbiguousEchoLowerStep; import org.jenkinsci.plugins.workflow.testMetaStep.AmbiguousEchoUpperStep; @@ -164,6 +170,74 @@ private static class Exec extends SynchronousStepExecution { } } + @Test public void flattenGStringLeak1() throws Exception { + p.setDefinition(new CpsFlowDefinition("echo \"${env.JOB_NAME}\"", true)); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + @Test public void flattenGStringLeak2() throws Exception { + p.setDefinition(new CpsFlowDefinition("def script = \"hello ${env.JOB_NAME}\"; echo(script)", true)); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + @Test public void flattenGStringLeak3() throws Exception { + p.setDefinition(new CpsFlowDefinition("echo \"${'job name is ' + env.JOB_NAME}\"", true)); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + @Test public void flattenGStringLeak4() throws Exception { + p.setDefinition(new CpsFlowDefinition("def s = { -> \"${env.JOB_NAME}\" }; echo(s())", true)); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + @Test public void flattenGStringLeak5() throws Exception { + p.setDefinition(new CpsFlowDefinition( + "parallel(one: {withEnv(['foo=bar']){echo env.foo; sleep 5}}," + + "two: {sleep 2; echo env.foo})", true)); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + // goes to DSL, but not a gstring + @Test public void flattenGStringLeak6() throws Exception { + p.setDefinition(new CpsFlowDefinition("def name = \"${env.JOB_NAME}\"; def script = 'hello' + name; echo(script)", true)); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + @Test public void flattenGStringLeak7() throws Exception { + p.setDefinition(new CpsFlowDefinition("echo \"${ -> env.JOB_NAME }\"", true)); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + @Test public void flattenGStringLeak8() throws Exception { + p.setDefinition(new CpsFlowDefinition( + "withEnv(['foo=bar']){echo env.foo}", true)); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + @Test public void withCredentials() throws Exception { + final String credentialsId = "creds"; + final String username = "bob"; + final String password = "s$$cr3t"; + UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); + CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" + + " // available as an env variable, but will be masked if you try to print it out any which way\n" + + " // note: single quotes prevent Groovy interpolation; expansion is by Bourne Shell, which is what you want\n" +// + " sh 'echo $PASSWORD'\n" + + " // this is bad!\n" + + " sh \"echo $PASSWORD\"\n" + + " // also available as a Groovy variable\n" + + " echo USERNAME\n" + + " // or inside double quotes for string interpolation\n" + + " echo \"username is $USERNAME\"\n" + + "}\n" + + "}", true)); + r.assertLogNotContains("cr3t", r.assertBuildStatusSuccess(p.scheduleBuild2(0))); + } + /** * Tests the ability to execute meta-step with clean syntax */ From 52f6f4b6aa567dafb3d49f9e8295c9bd21647516 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Fri, 24 Jul 2020 11:48:38 -0700 Subject: [PATCH 02/76] working PoC --- pom.xml | 15 ++-- .../plugins/workflow/cps/CpsStepContext.java | 10 +++ .../jenkinsci/plugins/workflow/cps/DSL.java | 76 ++++++++++++------- .../workflow/cps/SnippetizerTester.java | 2 +- 4 files changed, 68 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index bbae2c201..5f455aae8 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,8 @@ 2.83 -SNAPSHOT - 2.176.4 + + 2.246-SNAPSHOT 8 false 1.32 @@ -78,6 +79,11 @@ import pom + + org.jenkins-ci.plugins + credentials + 2.3.7 + @@ -157,6 +163,7 @@ org.jenkins-ci.plugins credentials-binding + 1.24-SNAPSHOT test @@ -256,12 +263,6 @@ 2.3 test - - org.jenkins-ci.plugins - credentials-binding - 1.23 - test - diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java index fe1dcacc2..5e0ffb6ef 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java @@ -162,6 +162,7 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla private @CheckForNull BodyReference body; private final int threadId; +// private CpsThread thread; /** * {@linkplain Descriptor#getId() step descriptor ID}. @@ -183,6 +184,7 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla @CpsVmThreadOnly CpsStepContext(StepDescriptor step, CpsThread thread, FlowExecutionOwner executionRef, FlowNode node, @CheckForNull Closure body) { this.threadId = thread.id; +// this.thread = thread; this.executionRef = executionRef; this.id = node.getId(); this.node = node; @@ -190,6 +192,14 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla this.stepDescriptorId = step.getId(); } + public void setBody(@Nonnull Closure body, @Nonnull CpsThread thread) { + if (this.body == null) { + this.body = thread.group.export(body); + } else { + throw new IllegalStateException("Context already has a body"); + } + } + /** * Obtains {@link StepDescriptor} that represents the step this context is invoking. * diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 46b6c79e7..b25a3eb59 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -88,6 +88,8 @@ import org.kohsuke.stapler.ClassDescriptor; import org.kohsuke.stapler.NoStaplerConstructorException; +import javax.annotation.Nullable; + /** * Calls {@link Step}s and other DSL objects. */ @@ -217,23 +219,29 @@ protected Object invokeStep(StepDescriptor d, Object args) { * @param args The arguments passed to the step. */ protected Object invokeStep(StepDescriptor d, String name, Object args) { - final NamedArgsAndClosure ps = parseArgs(args, d); - - CpsThread thread = CpsThread.current(); - - FlowNode an; // TODO: generalize the notion of Step taking over the FlowNode creation. boolean hack = d instanceof ParallelStep.DescriptorImpl || d instanceof LoadStep.DescriptorImpl; - if (ps.body == null && !hack) { + FlowNode an = null; + CpsThread thread = CpsThread.current(); + // TODO: is this okay? + if (/*ps.body == null*/!d.takesImplicitBlockArgument() && !hack) { an = new StepAtomNode(exec, d, thread.head.get()); } else { an = new StepStartNode(exec, d, thread.head.get()); } + CpsStepContext context = new CpsStepContext(d, thread, handle, an, null); + EnvVars envVars = null; + try { + envVars = context.get(EnvVars.class); + } catch (IOException | InterruptedException e) { + LOGGER.log(Level.FINE, "No environment variables found for the step context"); + } + NamedArgsAndClosure ps = parseArgs(args, d, envVars); + context.setBody(ps.body, thread); // Ensure ArgumentsAction is attached before we notify even synchronous listeners: - final CpsStepContext context = new CpsStepContext(d, thread, handle, an, ps.body); try { // No point storing empty arguments, and ParallelStep is a special case where we can't store its closure arguments if (ps.namedArgs != null && !(ps.namedArgs.isEmpty()) && isKeepStepArguments() && !(d instanceof ParallelStep.DescriptorImpl)) { @@ -354,7 +362,7 @@ protected Object invokeDescribable(String symbol, Object _args) { // The only time a closure is valid is when the resulting Describable is immediately executed via a meta-step NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), - UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly); + UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, null); UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); if (metaStep==null) { @@ -414,7 +422,7 @@ protected Object invokeDescribable(String symbol, Object _args) { ud = new UninstantiatedDescribable(symbol, null, dargs); margs.put(p.getName(),ud); - return invokeStep(metaStep, symbol, new NamedArgsAndClosure(margs, args.body)); + return invokeStep(metaStep, symbol, new NamedArgsAndClosure(margs, args.body, null)); } catch (Exception e) { throw new IllegalArgumentException("Failed to prepare "+symbol+" step",e); } @@ -459,13 +467,13 @@ static class NamedArgsAndClosure { final Map namedArgs; final Closure body; - private NamedArgsAndClosure(Map namedArgs, Closure body) { + private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvVars envVars) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more - Object v = flattenGString(entry.getValue()); + Object v = flattenGString(entry.getValue(), envVars); this.namedArgs.put(k, v); } } @@ -481,14 +489,23 @@ private NamedArgsAndClosure(Map namedArgs, Closure body) { * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v) { + private static Object flattenGString(Object v, @Nullable EnvVars envVars) { if (v instanceof GString) { - return v.toString(); + String flattened = v.toString(); + List watchedVars = null; + if (envVars != null) { + watchedVars = envVars.isValueWatched(flattened); + } + if (watchedVars != null && !watchedVars.isEmpty()) { + LOGGER.log(Level.WARNING, "Use single quotes to prevent leaking via Groovy interpolation. The following variables are at risk:"); + LOGGER.log(Level.WARNING, watchedVars.toString()); + } + return flattened; } else if (v instanceof List) { boolean mutated = false; List r = new ArrayList<>(); for (Object o : ((List) v)) { - Object o2 = flattenGString(o); + Object o2 = flattenGString(o, envVars); mutated |= o != o2; r.add(o2); } @@ -498,9 +515,9 @@ private static Object flattenGString(Object v) { Map r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) v).size())); for (Map.Entry e : ((Map) v).entrySet()) { Object k = e.getKey(); - Object k2 = flattenGString(k); + Object k2 = flattenGString(k, envVars); Object o = e.getValue(); - Object o2 = flattenGString(o); + Object o2 = flattenGString(o, envVars); mutated |= k != k2 || o != o2; r.put(k2, o2); } @@ -510,7 +527,7 @@ private static Object flattenGString(Object v) { } } - static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { + static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, @Nullable EnvVars envVars) { boolean singleArgumentOnly = false; try { DescribableModel stepModel = DescribableModel.of(d.clazz); @@ -518,14 +535,17 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { if (singleArgumentOnly) { // Can fetch the one argument we need DescribableParameter dp = stepModel.getSoleRequiredParameter(); String paramName = (dp != null) ? dp.getName() : null; - return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly); + return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, envVars); } } catch (NoStaplerConstructorException e) { // Ignore steps without databound constructors and treat them as normal. } - return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly); + return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, envVars); } +// static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg) { +// return parseArgs(arg, expectsBlock, soleArgumentKey, singleRequiredArg, null); +// } /** * Given the Groovy style argument packing used in the sole object parameter of {@link GroovyObject#invokeMethod(String, Object)}, * compute the named argument map and an optional closure that represents the body. @@ -548,19 +568,21 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { * @param soleArgumentKey * If the context in which this method call happens allow implicit sole default argument, specify its name. * If null, the call must be with names arguments. + * @param envVars + * The environment variables of the context */ - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg) { + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable EnvVars envVars) { if (arg instanceof NamedArgsAndClosure) return (NamedArgsAndClosure) arg; if (arg instanceof Map) // TODO is this clause actually used? - return new NamedArgsAndClosure((Map) arg, null); + return new NamedArgsAndClosure((Map) arg, null, envVars); if (arg instanceof Closure && expectsBlock) - return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg); + return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, envVars); if (arg instanceof Object[]) {// this is how Groovy appears to pack argument list into one Object for invokeMethod List a = Arrays.asList((Object[])arg); if (a.size()==0) - return new NamedArgsAndClosure(Collections.emptyMap(),null); + return new NamedArgsAndClosure(Collections.emptyMap(),null, envVars); Closure c=null; @@ -575,21 +597,21 @@ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String so if (!singleRequiredArg || (soleArgumentKey != null && mapArg.size() == 1 && mapArg.containsKey(soleArgumentKey))) { // this is how Groovy passes in Map - return new NamedArgsAndClosure(mapArg, c); + return new NamedArgsAndClosure(mapArg, c, envVars); } } switch (a.size()) { case 0: - return new NamedArgsAndClosure(Collections.emptyMap(),c); + return new NamedArgsAndClosure(Collections.emptyMap(),c, envVars); case 1: - return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, envVars); default: throw new IllegalArgumentException("Expected named arguments but got "+a); } } - return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, envVars); } private static Map singleParam(String soleArgumentKey, Object arg) { if (soleArgumentKey != null) { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/SnippetizerTester.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/SnippetizerTester.java index 762fce486..40fb6c155 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/SnippetizerTester.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/SnippetizerTester.java @@ -128,7 +128,7 @@ public void assertParseStep(Step expectedStep, String script) throws Exception { @Override protected Object invokeStep(StepDescriptor d, String name, Object args) { try { - return d.newInstance(parseArgs(args, d).namedArgs); + return d.newInstance(parseArgs(args, d, null).namedArgs); } catch (Exception e) { throw new AssertionError(e); } From 75c7406744bb7484af3f1b34930110a5a8893ed3 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Fri, 24 Jul 2020 15:00:49 -0700 Subject: [PATCH 03/76] add listener to report errors to pipeline output --- .../jenkinsci/plugins/workflow/cps/DSL.java | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index b25a3eb59..b59147adc 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -265,8 +265,9 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { boolean sync; ClassLoader originalLoader = Thread.currentThread().getContextClassLoader(); try { + TaskListener listener = context.get(TaskListener.class); if (unreportedAmbiguousFunctions.remove(name)) { - reportAmbiguousStepInvocation(context, d); + reportAmbiguousStepInvocation(context, d, listener); } d.checkContextAvailability(context); Thread.currentThread().setContextClassLoader(CpsVmExecutorService.ORIGINAL_CONTEXT_CLASS_LOADER.get()); @@ -274,7 +275,12 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { s = d.newInstance(ps.namedArgs); } else { DescribableModel stepModel = DescribableModel.of(d.clazz); - s = stepModel.instantiate(ps.namedArgs, context.get(TaskListener.class)); + s = stepModel.instantiate(ps.namedArgs, listener);//context.get(TaskListener.class)); + } + if (listener != null) { + ps.msgs.forEach(listener.getLogger()::println); + } else { + ps.msgs.forEach(System.out::println); } // Persist the node - block start and end nodes do their own persistence. @@ -429,10 +435,10 @@ protected Object invokeDescribable(String symbol, Object _args) { } } - private void reportAmbiguousStepInvocation(CpsStepContext context, StepDescriptor d) { + private void reportAmbiguousStepInvocation(CpsStepContext context, StepDescriptor d, @Nullable TaskListener listener) { Exception e = null; - try { - TaskListener listener = context.get(TaskListener.class); +// try { +// TaskListener listener = context.get(TaskListener.class); if (listener != null) { List ambiguousClassNames = StepDescriptor.all().stream() .filter(sd -> sd.getFunctionName().equals(d.getFunctionName())) @@ -446,9 +452,9 @@ private void reportAmbiguousStepInvocation(CpsStepContext context, StepDescripto listener.getLogger().println(message); return; } - } catch (InterruptedException | IOException temp) { - e = temp; - } +// } catch (InterruptedException | IOException temp) { +// e = temp; +// } LOGGER.log(Level.FINE, "Unable to report ambiguous step invocation for: " + d.getFunctionName(), e); } @@ -466,14 +472,16 @@ private static int preallocatedHashmapCapacity(int elementsToHold) { static class NamedArgsAndClosure { final Map namedArgs; final Closure body; + final List msgs; private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvVars envVars) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; + this.msgs = new ArrayList<>(); for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more - Object v = flattenGString(entry.getValue(), envVars); + Object v = flattenGString(entry.getValue(), envVars, msgs); this.namedArgs.put(k, v); } } @@ -489,7 +497,7 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvVars * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v, @Nullable EnvVars envVars) { + private static Object flattenGString(Object v, @Nullable EnvVars envVars, List msgs) { if (v instanceof GString) { String flattened = v.toString(); List watchedVars = null; @@ -497,15 +505,14 @@ private static Object flattenGString(Object v, @Nullable EnvVars envVars) { watchedVars = envVars.isValueWatched(flattened); } if (watchedVars != null && !watchedVars.isEmpty()) { - LOGGER.log(Level.WARNING, "Use single quotes to prevent leaking via Groovy interpolation. The following variables are at risk:"); - LOGGER.log(Level.WARNING, watchedVars.toString()); + msgs.add("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + watchedVars.toString()); } return flattened; } else if (v instanceof List) { boolean mutated = false; List r = new ArrayList<>(); for (Object o : ((List) v)) { - Object o2 = flattenGString(o, envVars); + Object o2 = flattenGString(o, envVars, msgs); mutated |= o != o2; r.add(o2); } @@ -515,9 +522,9 @@ private static Object flattenGString(Object v, @Nullable EnvVars envVars) { Map r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) v).size())); for (Map.Entry e : ((Map) v).entrySet()) { Object k = e.getKey(); - Object k2 = flattenGString(k, envVars); + Object k2 = flattenGString(k, envVars, msgs); Object o = e.getValue(); - Object o2 = flattenGString(o, envVars); + Object o2 = flattenGString(o, envVars, msgs); mutated |= k != k2 || o != o2; r.put(k2, o2); } @@ -543,9 +550,6 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, @Nullable Env return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, envVars); } -// static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg) { -// return parseArgs(arg, expectsBlock, soleArgumentKey, singleRequiredArg, null); -// } /** * Given the Groovy style argument packing used in the sole object parameter of {@link GroovyObject#invokeMethod(String, Object)}, * compute the named argument map and an optional closure that represents the body. From 191f08eadabdd2f3764ab1c427272fb4a30666fe Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 28 Jul 2020 15:52:08 -0700 Subject: [PATCH 04/76] PoC 2 for groovy interpolation interception. Does not require core mods --- pom.xml | 7 ++ .../jenkinsci/plugins/workflow/cps/DSL.java | 66 ++++++++++--------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index 5f455aae8..7037cf10f 100644 --- a/pom.xml +++ b/pom.xml @@ -90,6 +90,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api + 2.23-SNAPSHOT org.jenkins-ci.plugins.workflow @@ -130,6 +131,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api + 2.23-SNAPSHOT tests test @@ -160,6 +162,11 @@ workflow-durable-task-step test + + org.jenkins-ci.plugins + durable-task + test + org.jenkins-ci.plugins credentials-binding diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index b59147adc..d95f4b46d 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -41,6 +41,7 @@ import hudson.model.Run; import hudson.model.TaskListener; import java.io.IOException; +import java.io.PrintStream; import java.io.Serializable; import java.lang.annotation.Annotation; import java.util.ArrayList; @@ -80,6 +81,7 @@ import org.jenkinsci.plugins.workflow.flow.StepListener; import org.jenkinsci.plugins.workflow.graph.FlowNode; import org.jenkinsci.plugins.workflow.steps.BodyExecutionCallback; +import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander; import org.jenkinsci.plugins.workflow.steps.Step; import org.jenkinsci.plugins.workflow.steps.StepContext; import org.jenkinsci.plugins.workflow.steps.StepDescriptor; @@ -233,13 +235,13 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } CpsStepContext context = new CpsStepContext(d, thread, handle, an, null); - EnvVars envVars = null; + EnvironmentExpander expander = null; try { - envVars = context.get(EnvVars.class); + expander = thread.getContextVariable(EnvironmentExpander.class, context::getExecution, context::getNode); } catch (IOException | InterruptedException e) { - LOGGER.log(Level.FINE, "No environment variables found for the step context"); + LOGGER.log(Level.FINE, "No environment expanders found for the step context"); } - NamedArgsAndClosure ps = parseArgs(args, d, envVars); + NamedArgsAndClosure ps = parseArgs(args, d, expander);//envVars); context.setBody(ps.body, thread); // Ensure ArgumentsAction is attached before we notify even synchronous listeners: try { @@ -277,10 +279,14 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { DescribableModel stepModel = DescribableModel.of(d.clazz); s = stepModel.instantiate(ps.namedArgs, listener);//context.get(TaskListener.class)); } - if (listener != null) { - ps.msgs.forEach(listener.getLogger()::println); - } else { - ps.msgs.forEach(System.out::println); + if (expander != null) { + PrintStream logger; + if (listener != null) { + logger = listener.getLogger(); + } else { + logger = System.out; + } + expander.callback(logger); } // Persist the node - block start and end nodes do their own persistence. @@ -474,14 +480,14 @@ static class NamedArgsAndClosure { final Closure body; final List msgs; - private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvVars envVars) { + private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvironmentExpander expander) {//EnvVars envVars) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; this.msgs = new ArrayList<>(); for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more - Object v = flattenGString(entry.getValue(), envVars, msgs); + Object v = flattenGString(entry.getValue(), expander, msgs);//envVars, msgs); this.namedArgs.put(k, v); } } @@ -497,22 +503,18 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvVars * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v, @Nullable EnvVars envVars, List msgs) { + private static Object flattenGString(Object v, @Nullable /*EnvVars envVars*/EnvironmentExpander expander, List msgs) { if (v instanceof GString) { String flattened = v.toString(); - List watchedVars = null; - if (envVars != null) { - watchedVars = envVars.isValueWatched(flattened); - } - if (watchedVars != null && !watchedVars.isEmpty()) { - msgs.add("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + watchedVars.toString()); + if (expander != null) { + expander.findWatchedVars(flattened); } return flattened; } else if (v instanceof List) { boolean mutated = false; List r = new ArrayList<>(); for (Object o : ((List) v)) { - Object o2 = flattenGString(o, envVars, msgs); + Object o2 = flattenGString(o, expander, msgs); mutated |= o != o2; r.add(o2); } @@ -522,9 +524,9 @@ private static Object flattenGString(Object v, @Nullable EnvVars envVars, List r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) v).size())); for (Map.Entry e : ((Map) v).entrySet()) { Object k = e.getKey(); - Object k2 = flattenGString(k, envVars, msgs); + Object k2 = flattenGString(k, expander, msgs); Object o = e.getValue(); - Object o2 = flattenGString(o, envVars, msgs); + Object o2 = flattenGString(o, expander, msgs); mutated |= k != k2 || o != o2; r.put(k2, o2); } @@ -534,7 +536,7 @@ private static Object flattenGString(Object v, @Nullable EnvVars envVars, List stepModel = DescribableModel.of(d.clazz); @@ -542,12 +544,12 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, @Nullable Env if (singleArgumentOnly) { // Can fetch the one argument we need DescribableParameter dp = stepModel.getSoleRequiredParameter(); String paramName = (dp != null) ? dp.getName() : null; - return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, envVars); + return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, expander); } } catch (NoStaplerConstructorException e) { // Ignore steps without databound constructors and treat them as normal. } - return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, envVars); + return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, expander); } /** @@ -572,21 +574,21 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, @Nullable Env * @param soleArgumentKey * If the context in which this method call happens allow implicit sole default argument, specify its name. * If null, the call must be with names arguments. - * @param envVars +// * @param envVars * The environment variables of the context */ - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable EnvVars envVars) { + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable EnvironmentExpander expander) {//EnvVars envVars) { if (arg instanceof NamedArgsAndClosure) return (NamedArgsAndClosure) arg; if (arg instanceof Map) // TODO is this clause actually used? - return new NamedArgsAndClosure((Map) arg, null, envVars); + return new NamedArgsAndClosure((Map) arg, null, expander); if (arg instanceof Closure && expectsBlock) - return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, envVars); + return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, expander); if (arg instanceof Object[]) {// this is how Groovy appears to pack argument list into one Object for invokeMethod List a = Arrays.asList((Object[])arg); if (a.size()==0) - return new NamedArgsAndClosure(Collections.emptyMap(),null, envVars); + return new NamedArgsAndClosure(Collections.emptyMap(),null, expander); Closure c=null; @@ -601,21 +603,21 @@ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String so if (!singleRequiredArg || (soleArgumentKey != null && mapArg.size() == 1 && mapArg.containsKey(soleArgumentKey))) { // this is how Groovy passes in Map - return new NamedArgsAndClosure(mapArg, c, envVars); + return new NamedArgsAndClosure(mapArg, c, expander); } } switch (a.size()) { case 0: - return new NamedArgsAndClosure(Collections.emptyMap(),c, envVars); + return new NamedArgsAndClosure(Collections.emptyMap(),c, expander); case 1: - return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, envVars); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, expander); default: throw new IllegalArgumentException("Expected named arguments but got "+a); } } - return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, envVars); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, expander); } private static Map singleParam(String soleArgumentKey, Object arg) { if (soleArgumentKey != null) { From ccf90400e6fec474c799ef2ba1a9bbf610721b09 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 30 Jul 2020 18:35:55 -0700 Subject: [PATCH 05/76] Wrap EnvironmentExpander and EnvVars together for parseArgs --- .../workflow/cps/ContextVariableSet.java | 1 + .../jenkinsci/plugins/workflow/cps/DSL.java | 95 +++++++------------ .../workflow/cps/EnvironmentWatcher.java | 58 +++++++++++ 3 files changed, 95 insertions(+), 59 deletions(-) create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/ContextVariableSet.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/ContextVariableSet.java index b35966310..1275dffb8 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/ContextVariableSet.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/ContextVariableSet.java @@ -81,6 +81,7 @@ private static final class DynamicContextQuery { } } + @FunctionalInterface interface ThrowingSupplier { T get() throws IOException; } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index d95f4b46d..5fec0ed11 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -41,7 +41,6 @@ import hudson.model.Run; import hudson.model.TaskListener; import java.io.IOException; -import java.io.PrintStream; import java.io.Serializable; import java.lang.annotation.Annotation; import java.util.ArrayList; @@ -81,7 +80,6 @@ import org.jenkinsci.plugins.workflow.flow.StepListener; import org.jenkinsci.plugins.workflow.graph.FlowNode; import org.jenkinsci.plugins.workflow.steps.BodyExecutionCallback; -import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander; import org.jenkinsci.plugins.workflow.steps.Step; import org.jenkinsci.plugins.workflow.steps.StepContext; import org.jenkinsci.plugins.workflow.steps.StepDescriptor; @@ -227,21 +225,15 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { FlowNode an = null; CpsThread thread = CpsThread.current(); - // TODO: is this okay? - if (/*ps.body == null*/!d.takesImplicitBlockArgument() && !hack) { + if (!d.takesImplicitBlockArgument() && !hack) { an = new StepAtomNode(exec, d, thread.head.get()); } else { an = new StepStartNode(exec, d, thread.head.get()); } CpsStepContext context = new CpsStepContext(d, thread, handle, an, null); - EnvironmentExpander expander = null; - try { - expander = thread.getContextVariable(EnvironmentExpander.class, context::getExecution, context::getNode); - } catch (IOException | InterruptedException e) { - LOGGER.log(Level.FINE, "No environment expanders found for the step context"); - } - NamedArgsAndClosure ps = parseArgs(args, d, expander);//envVars); + EnvironmentWatcher envWatcher = new EnvironmentWatcher(context); + NamedArgsAndClosure ps = parseArgs(args, d, envWatcher); context.setBody(ps.body, thread); // Ensure ArgumentsAction is attached before we notify even synchronous listeners: try { @@ -277,17 +269,9 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { s = d.newInstance(ps.namedArgs); } else { DescribableModel stepModel = DescribableModel.of(d.clazz); - s = stepModel.instantiate(ps.namedArgs, listener);//context.get(TaskListener.class)); - } - if (expander != null) { - PrintStream logger; - if (listener != null) { - logger = listener.getLogger(); - } else { - logger = System.out; - } - expander.callback(logger); + s = stepModel.instantiate(ps.namedArgs, listener); } + envWatcher.logResults(listener); // Persist the node - block start and end nodes do their own persistence. CpsFlowExecution.maybeAutoPersistNode(an); @@ -443,24 +427,19 @@ protected Object invokeDescribable(String symbol, Object _args) { private void reportAmbiguousStepInvocation(CpsStepContext context, StepDescriptor d, @Nullable TaskListener listener) { Exception e = null; -// try { -// TaskListener listener = context.get(TaskListener.class); - if (listener != null) { - List ambiguousClassNames = StepDescriptor.all().stream() - .filter(sd -> sd.getFunctionName().equals(d.getFunctionName())) - .map(sd -> sd.clazz.getName()) - .collect(Collectors.toList()); - String message = String.format("Warning: Invoking ambiguous Pipeline Step ‘%1$s’ (%2$s). " + - "‘%1$s’ could refer to any of the following steps: %3$s. " + - "You can invoke steps by class name instead to avoid ambiguity. " + - "For example: steps.'%2$s'(...)", - d.getFunctionName(), d.clazz.getName(), ambiguousClassNames); - listener.getLogger().println(message); - return; - } -// } catch (InterruptedException | IOException temp) { -// e = temp; -// } + if (listener != null) { + List ambiguousClassNames = StepDescriptor.all().stream() + .filter(sd -> sd.getFunctionName().equals(d.getFunctionName())) + .map(sd -> sd.clazz.getName()) + .collect(Collectors.toList()); + String message = String.format("Warning: Invoking ambiguous Pipeline Step ‘%1$s’ (%2$s). " + + "‘%1$s’ could refer to any of the following steps: %3$s. " + + "You can invoke steps by class name instead to avoid ambiguity. " + + "For example: steps.'%2$s'(...)", + d.getFunctionName(), d.clazz.getName(), ambiguousClassNames); + listener.getLogger().println(message); + return; + } LOGGER.log(Level.FINE, "Unable to report ambiguous step invocation for: " + d.getFunctionName(), e); } @@ -480,14 +459,14 @@ static class NamedArgsAndClosure { final Closure body; final List msgs; - private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvironmentExpander expander) {//EnvVars envVars) { + private NamedArgsAndClosure(Map namedArgs, Closure body, EnvironmentWatcher envWatcher) {//@Nullable EnvironmentExpander expander) {//EnvVars envVars) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; this.msgs = new ArrayList<>(); for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more - Object v = flattenGString(entry.getValue(), expander, msgs);//envVars, msgs); + Object v = flattenGString(entry.getValue(), envWatcher);//expander, msgs);//envVars, msgs); this.namedArgs.put(k, v); } } @@ -503,18 +482,16 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable Environm * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v, @Nullable /*EnvVars envVars*/EnvironmentExpander expander, List msgs) { + private static Object flattenGString(Object v, EnvironmentWatcher envWatcher) { if (v instanceof GString) { String flattened = v.toString(); - if (expander != null) { - expander.findWatchedVars(flattened); - } + envWatcher.scan(flattened); return flattened; } else if (v instanceof List) { boolean mutated = false; List r = new ArrayList<>(); for (Object o : ((List) v)) { - Object o2 = flattenGString(o, expander, msgs); + Object o2 = flattenGString(o, envWatcher); mutated |= o != o2; r.add(o2); } @@ -524,9 +501,9 @@ private static Object flattenGString(Object v, @Nullable /*EnvVars envVars*/Envi Map r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) v).size())); for (Map.Entry e : ((Map) v).entrySet()) { Object k = e.getKey(); - Object k2 = flattenGString(k, expander, msgs); + Object k2 = flattenGString(k, envWatcher); Object o = e.getValue(); - Object o2 = flattenGString(o, expander, msgs); + Object o2 = flattenGString(o, envWatcher); mutated |= k != k2 || o != o2; r.put(k2, o2); } @@ -536,7 +513,7 @@ private static Object flattenGString(Object v, @Nullable /*EnvVars envVars*/Envi } } - static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, @Nullable EnvironmentExpander expander) {//EnvVars envVars) { + static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, EnvironmentWatcher envWatcher) { boolean singleArgumentOnly = false; try { DescribableModel stepModel = DescribableModel.of(d.clazz); @@ -544,12 +521,12 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, @Nullable Env if (singleArgumentOnly) { // Can fetch the one argument we need DescribableParameter dp = stepModel.getSoleRequiredParameter(); String paramName = (dp != null) ? dp.getName() : null; - return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, expander); + return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, envWatcher); } } catch (NoStaplerConstructorException e) { // Ignore steps without databound constructors and treat them as normal. } - return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, expander); + return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, envWatcher); } /** @@ -577,18 +554,18 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, @Nullable Env // * @param envVars * The environment variables of the context */ - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable EnvironmentExpander expander) {//EnvVars envVars) { + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, EnvironmentWatcher envWatcher) { if (arg instanceof NamedArgsAndClosure) return (NamedArgsAndClosure) arg; if (arg instanceof Map) // TODO is this clause actually used? - return new NamedArgsAndClosure((Map) arg, null, expander); + return new NamedArgsAndClosure((Map) arg, null, envWatcher); if (arg instanceof Closure && expectsBlock) - return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, expander); + return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, envWatcher); if (arg instanceof Object[]) {// this is how Groovy appears to pack argument list into one Object for invokeMethod List a = Arrays.asList((Object[])arg); if (a.size()==0) - return new NamedArgsAndClosure(Collections.emptyMap(),null, expander); + return new NamedArgsAndClosure(Collections.emptyMap(),null, envWatcher); Closure c=null; @@ -603,21 +580,21 @@ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String so if (!singleRequiredArg || (soleArgumentKey != null && mapArg.size() == 1 && mapArg.containsKey(soleArgumentKey))) { // this is how Groovy passes in Map - return new NamedArgsAndClosure(mapArg, c, expander); + return new NamedArgsAndClosure(mapArg, c, envWatcher); } } switch (a.size()) { case 0: - return new NamedArgsAndClosure(Collections.emptyMap(),c, expander); + return new NamedArgsAndClosure(Collections.emptyMap(),c, envWatcher); case 1: - return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, expander); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, envWatcher); default: throw new IllegalArgumentException("Expected named arguments but got "+a); } } - return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, expander); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, envWatcher); } private static Map singleParam(String soleArgumentKey, Object arg) { if (soleArgumentKey != null) { diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java new file mode 100644 index 000000000..9014f5cf5 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java @@ -0,0 +1,58 @@ +package org.jenkinsci.plugins.workflow.cps; + +import hudson.EnvVars; +import hudson.model.TaskListener; +import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander; + +import javax.annotation.CheckForNull; +import java.io.IOException; +import java.io.PrintStream; +import java.io.Serializable; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class EnvironmentWatcher implements Serializable { + private EnvVars envVars = null; + private EnvironmentExpander expander = null; + private Set watchedVars; + private List scanResults; + + public EnvironmentWatcher(CpsStepContext context) { + try { + envVars = context.get(EnvVars.class); + expander = context.get(EnvironmentExpander.class); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } + if (expander != null) { + watchedVars = expander.getWatchedVars(); + } + } + + public void scan(String text) { + // TODO: would EnvVars ever be null? + if (watchedVars == null || envVars == null) { + scanResults = null; + } else { + scanResults = watchedVars.stream().filter(e -> text.contains(envVars.get(e))).collect(Collectors.toList()); + } + } + + public void logResults(TaskListener listener) { + if (scanResults != null && !scanResults.isEmpty()) { + PrintStream logger; + if (listener != null) { + logger = listener.getLogger(); + } else { + logger = System.out; + } + logger.println("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + scanResults.toString()); + } + } + +// @CheckForNull +// public List getScanResults() { +// return scanResults; +// } +} From 5c53b948cdbb11f48992e9d3dde428e884c3311e Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 30 Jul 2020 20:16:31 -0700 Subject: [PATCH 06/76] use incrementals, revert jenkins version --- pom.xml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 7037cf10f..cbf2add7d 100644 --- a/pom.xml +++ b/pom.xml @@ -64,8 +64,7 @@ 2.83 -SNAPSHOT - - 2.246-SNAPSHOT + 2.176.4 8 false 1.32 @@ -90,7 +89,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api - 2.23-SNAPSHOT + 2.23-rc564.92d97c88bfc7 org.jenkins-ci.plugins.workflow @@ -131,7 +130,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api - 2.23-SNAPSHOT + 2.23-rc564.92d97c88bfc7 tests test @@ -170,7 +169,7 @@ org.jenkins-ci.plugins credentials-binding - 1.24-SNAPSHOT + 1.24-rc367.5efa464be9e2 test From c5ef8ad0288e5bf9208bd7187f485e1081912124 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 30 Jul 2020 20:37:31 -0700 Subject: [PATCH 07/76] code cleanup --- pom.xml | 2 +- .../plugins/workflow/cps/CpsStepContext.java | 2 - .../jenkinsci/plugins/workflow/cps/DSL.java | 10 ++-- .../plugins/workflow/cps/DSLTest.java | 49 ++----------------- 4 files changed, 10 insertions(+), 53 deletions(-) diff --git a/pom.xml b/pom.xml index cbf2add7d..271649519 100644 --- a/pom.xml +++ b/pom.xml @@ -169,7 +169,7 @@ org.jenkins-ci.plugins credentials-binding - 1.24-rc367.5efa464be9e2 + 1.24-rc368.9646ad36e52a test diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java index 5e0ffb6ef..2ea16a236 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java @@ -162,7 +162,6 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla private @CheckForNull BodyReference body; private final int threadId; -// private CpsThread thread; /** * {@linkplain Descriptor#getId() step descriptor ID}. @@ -184,7 +183,6 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla @CpsVmThreadOnly CpsStepContext(StepDescriptor step, CpsThread thread, FlowExecutionOwner executionRef, FlowNode node, @CheckForNull Closure body) { this.threadId = thread.id; -// this.thread = thread; this.executionRef = executionRef; this.id = node.getId(); this.node = node; diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 5fec0ed11..7deb049f1 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -459,7 +459,7 @@ static class NamedArgsAndClosure { final Closure body; final List msgs; - private NamedArgsAndClosure(Map namedArgs, Closure body, EnvironmentWatcher envWatcher) {//@Nullable EnvironmentExpander expander) {//EnvVars envVars) { + private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvironmentWatcher envWatcher) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; this.msgs = new ArrayList<>(); @@ -482,10 +482,12 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, EnvironmentWatcher * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v, EnvironmentWatcher envWatcher) { + private static Object flattenGString(Object v, @Nullable EnvironmentWatcher envWatcher) { if (v instanceof GString) { String flattened = v.toString(); - envWatcher.scan(flattened); + if (envWatcher != null) { + envWatcher.scan(flattened); + } return flattened; } else if (v instanceof List) { boolean mutated = false; @@ -554,7 +556,7 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, EnvironmentWa // * @param envVars * The environment variables of the context */ - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, EnvironmentWatcher envWatcher) { + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable EnvironmentWatcher envWatcher) { if (arg instanceof NamedArgsAndClosure) return (NamedArgsAndClosure) arg; if (arg instanceof Map) // TODO is this clause actually used? diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 8e611172c..d5d00be29 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -170,50 +170,6 @@ private static class Exec extends SynchronousStepExecution { } } - @Test public void flattenGStringLeak1() throws Exception { - p.setDefinition(new CpsFlowDefinition("echo \"${env.JOB_NAME}\"", true)); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - } - - @Test public void flattenGStringLeak2() throws Exception { - p.setDefinition(new CpsFlowDefinition("def script = \"hello ${env.JOB_NAME}\"; echo(script)", true)); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - } - - @Test public void flattenGStringLeak3() throws Exception { - p.setDefinition(new CpsFlowDefinition("echo \"${'job name is ' + env.JOB_NAME}\"", true)); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - } - - @Test public void flattenGStringLeak4() throws Exception { - p.setDefinition(new CpsFlowDefinition("def s = { -> \"${env.JOB_NAME}\" }; echo(s())", true)); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - } - - @Test public void flattenGStringLeak5() throws Exception { - p.setDefinition(new CpsFlowDefinition( - "parallel(one: {withEnv(['foo=bar']){echo env.foo; sleep 5}}," + - "two: {sleep 2; echo env.foo})", true)); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - } - - // goes to DSL, but not a gstring - @Test public void flattenGStringLeak6() throws Exception { - p.setDefinition(new CpsFlowDefinition("def name = \"${env.JOB_NAME}\"; def script = 'hello' + name; echo(script)", true)); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - } - - @Test public void flattenGStringLeak7() throws Exception { - p.setDefinition(new CpsFlowDefinition("echo \"${ -> env.JOB_NAME }\"", true)); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - } - - @Test public void flattenGStringLeak8() throws Exception { - p.setDefinition(new CpsFlowDefinition( - "withEnv(['foo=bar']){echo env.foo}", true)); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - } - @Test public void withCredentials() throws Exception { final String credentialsId = "creds"; final String username = "bob"; @@ -226,7 +182,6 @@ private static class Exec extends SynchronousStepExecution { + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" + " // available as an env variable, but will be masked if you try to print it out any which way\n" + " // note: single quotes prevent Groovy interpolation; expansion is by Bourne Shell, which is what you want\n" -// + " sh 'echo $PASSWORD'\n" + " // this is bad!\n" + " sh \"echo $PASSWORD\"\n" + " // also available as a Groovy variable\n" @@ -235,7 +190,9 @@ private static class Exec extends SynchronousStepExecution { + " echo \"username is $USERNAME\"\n" + "}\n" + "}", true)); - r.assertLogNotContains("cr3t", r.assertBuildStatusSuccess(p.scheduleBuild2(0))); + WorkflowRun workflowRun = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + r.assertLogContains("Affected variables: [PASSWORD]", workflowRun); + r.assertLogContains("Affected variables: [USERNAME]", workflowRun); } /** From 1a7793d9e179c7181d944aac188e4a504ebdaf4f Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 30 Jul 2020 21:26:12 -0700 Subject: [PATCH 08/76] catch null arguments --- .../jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java index 9014f5cf5..c0b223b80 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java @@ -10,6 +10,8 @@ import java.io.Serializable; import java.util.List; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; public class EnvironmentWatcher implements Serializable { @@ -18,12 +20,16 @@ public class EnvironmentWatcher implements Serializable { private Set watchedVars; private List scanResults; + private static final Logger LOGGER = Logger.getLogger(EnvironmentWatcher.class.getName()); + public EnvironmentWatcher(CpsStepContext context) { try { envVars = context.get(EnvVars.class); expander = context.get(EnvironmentExpander.class); } catch (IOException | InterruptedException e) { e.printStackTrace(); + } catch (IllegalArgumentException e) { + LOGGER.log(Level.WARNING, "Error storing the arguments for step: " + context.getStepDescriptor().getFunctionName(), e); } if (expander != null) { watchedVars = expander.getWatchedVars(); From 0829025e4b16ea270dee18fed5c0b53b48817771 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Fri, 31 Jul 2020 10:11:53 -0700 Subject: [PATCH 09/76] Make unit test windows friendly. Remove dollar sign from password --- .../org/jenkinsci/plugins/workflow/cps/DSLTest.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index d5d00be29..cbe1d21f9 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -173,20 +173,16 @@ private static class Exec extends SynchronousStepExecution { @Test public void withCredentials() throws Exception { final String credentialsId = "creds"; final String username = "bob"; - final String password = "s$$cr3t"; + final String password = "secr3t"; UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + String shellStep = Functions.isWindows()? "bat \"echo $PASSWORD\"\n" : "sh \"echo $PASSWORD\"\n"; p.setDefinition(new CpsFlowDefinition("" + "node {\n" + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" - + " // available as an env variable, but will be masked if you try to print it out any which way\n" - + " // note: single quotes prevent Groovy interpolation; expansion is by Bourne Shell, which is what you want\n" - + " // this is bad!\n" - + " sh \"echo $PASSWORD\"\n" - + " // also available as a Groovy variable\n" + + shellStep + " echo USERNAME\n" - + " // or inside double quotes for string interpolation\n" + " echo \"username is $USERNAME\"\n" + "}\n" + "}", true)); From 18653c028503c8c033f55ed4ddd12aae37f75199 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 4 Aug 2020 09:19:35 -0700 Subject: [PATCH 10/76] update to use newer implementation of EnvironmentExpander --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 271649519..74588a92f 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api - 2.23-rc564.92d97c88bfc7 + 2.23-rc565.4e7dd9b85962 org.jenkins-ci.plugins.workflow @@ -130,7 +130,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api - 2.23-rc564.92d97c88bfc7 + 2.23-rc565.4e7dd9b85962 tests test @@ -169,7 +169,7 @@ org.jenkins-ci.plugins credentials-binding - 1.24-rc368.9646ad36e52a + 1.24-rc369.e14c2682d492 test From 5ba9b2a705da2b41e8cfcc53b14608675e1203eb Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 5 Aug 2020 09:23:25 -0700 Subject: [PATCH 11/76] Use updated api in EnvironmentExpander --- pom.xml | 6 +++--- .../jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java | 7 +------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 74588a92f..d26a4c00e 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api - 2.23-rc565.4e7dd9b85962 + 2.23-rc566.c1fa948b49be org.jenkins-ci.plugins.workflow @@ -130,7 +130,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api - 2.23-rc565.4e7dd9b85962 + 2.23-rc566.c1fa948b49be tests test @@ -169,7 +169,7 @@ org.jenkins-ci.plugins credentials-binding - 1.24-rc369.e14c2682d492 + 1.24-rc370.8c9bfdc92872 test diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java index c0b223b80..44ee5a1e1 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java @@ -32,7 +32,7 @@ public EnvironmentWatcher(CpsStepContext context) { LOGGER.log(Level.WARNING, "Error storing the arguments for step: " + context.getStepDescriptor().getFunctionName(), e); } if (expander != null) { - watchedVars = expander.getWatchedVars(); + watchedVars = expander.getSensitiveVars(); } } @@ -56,9 +56,4 @@ public void logResults(TaskListener listener) { logger.println("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + scanResults.toString()); } } - -// @CheckForNull -// public List getScanResults() { -// return scanResults; -// } } From aeebf177dccd985fbed72fe0492134b6d1a3d610 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 27 Aug 2020 11:55:37 -0600 Subject: [PATCH 12/76] address review comments --- pom.xml | 12 +----------- .../plugins/workflow/cps/CpsStepContext.java | 9 ++++----- .../java/org/jenkinsci/plugins/workflow/cps/DSL.java | 2 +- .../plugins/workflow/cps/EnvironmentWatcher.java | 8 +------- 4 files changed, 7 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index d26a4c00e..0cdc9d181 100644 --- a/pom.xml +++ b/pom.xml @@ -74,15 +74,10 @@ io.jenkins.tools.bom bom-2.176.x - 9 + 11 import pom - - org.jenkins-ci.plugins - credentials - 2.3.7 - @@ -161,11 +156,6 @@ workflow-durable-task-step test - - org.jenkins-ci.plugins - durable-task - test - org.jenkins-ci.plugins credentials-binding diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java index 2ea16a236..51059619c 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java @@ -155,7 +155,7 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla private transient List bodyInvokers = new ArrayList<>(); /** - * While {@link CpsStepContext} has not received teh response, maintains the body closure. + * While {@link CpsStepContext} has not received the response, maintains the body closure. * * This is the implicit closure block passed to the step invocation. */ @@ -181,18 +181,17 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla private transient volatile boolean loadingThreadGroup; @CpsVmThreadOnly - CpsStepContext(StepDescriptor step, CpsThread thread, FlowExecutionOwner executionRef, FlowNode node, @CheckForNull Closure body) { + CpsStepContext(StepDescriptor step, CpsThread thread, FlowExecutionOwner executionRef, FlowNode node) { this.threadId = thread.id; this.executionRef = executionRef; this.id = node.getId(); this.node = node; - this.body = body != null ? thread.group.export(body) : null; this.stepDescriptorId = step.getId(); } - public void setBody(@Nonnull Closure body, @Nonnull CpsThread thread) { + public void setBody(@Nonnull Closure bodyToSet, @Nonnull CpsThread thread) { if (this.body == null) { - this.body = thread.group.export(body); + this.body = thread.group.export(bodyToSet); } else { throw new IllegalStateException("Context already has a body"); } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 7deb049f1..ce07a3f41 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -231,7 +231,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { an = new StepStartNode(exec, d, thread.head.get()); } - CpsStepContext context = new CpsStepContext(d, thread, handle, an, null); + CpsStepContext context = new CpsStepContext(d, thread, handle, an); EnvironmentWatcher envWatcher = new EnvironmentWatcher(context); NamedArgsAndClosure ps = parseArgs(args, d, envWatcher); context.setBody(ps.body, thread); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java index 44ee5a1e1..29181cffe 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java @@ -47,13 +47,7 @@ public void scan(String text) { public void logResults(TaskListener listener) { if (scanResults != null && !scanResults.isEmpty()) { - PrintStream logger; - if (listener != null) { - logger = listener.getLogger(); - } else { - logger = System.out; - } - logger.println("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + scanResults.toString()); + listener.getLogger().println("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + scanResults.toString()); } } } From 02804b3ea40df3619abcb4682fcca5c47c1b3654 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 1 Sep 2020 14:16:11 -0600 Subject: [PATCH 13/76] add factory method, add report action, and summary page --- .../jenkinsci/plugins/workflow/cps/DSL.java | 10 ++-- .../workflow/cps/EnvironmentWatcher.java | 52 ++++++++++------- .../cps/view/EnvironmentWatcherListener.java | 31 ++++++++++ .../cps/view/EnvironmentWatcherRunReport.java | 56 +++++++++++++++++++ .../EnvironmentWatcherRunReport/summary.jelly | 37 ++++++++++++ 5 files changed, 162 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherListener.java create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport.java create mode 100644 src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index ce07a3f41..10ca4540f 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -223,7 +223,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { // TODO: generalize the notion of Step taking over the FlowNode creation. boolean hack = d instanceof ParallelStep.DescriptorImpl || d instanceof LoadStep.DescriptorImpl; - FlowNode an = null; + FlowNode an; CpsThread thread = CpsThread.current(); if (!d.takesImplicitBlockArgument() && !hack) { an = new StepAtomNode(exec, d, thread.head.get()); @@ -232,7 +232,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } CpsStepContext context = new CpsStepContext(d, thread, handle, an); - EnvironmentWatcher envWatcher = new EnvironmentWatcher(context); + EnvironmentWatcher envWatcher = EnvironmentWatcher.of(context, exec); NamedArgsAndClosure ps = parseArgs(args, d, envWatcher); context.setBody(ps.body, thread); // Ensure ArgumentsAction is attached before we notify even synchronous listeners: @@ -271,7 +271,9 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { DescribableModel stepModel = DescribableModel.of(d.clazz); s = stepModel.instantiate(ps.namedArgs, listener); } - envWatcher.logResults(listener); + if (envWatcher != null) { + envWatcher.logResults(listener); + } // Persist the node - block start and end nodes do their own persistence. CpsFlowExecution.maybeAutoPersistNode(an); @@ -466,7 +468,7 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable Environm for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more - Object v = flattenGString(entry.getValue(), envWatcher);//expander, msgs);//envVars, msgs); + Object v = flattenGString(entry.getValue(), envWatcher); this.namedArgs.put(k, v); } } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java index 29181cffe..373ab6208 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java @@ -1,12 +1,14 @@ package org.jenkinsci.plugins.workflow.cps; import hudson.EnvVars; +import hudson.model.Run; import hudson.model.TaskListener; +import org.jenkinsci.plugins.workflow.cps.view.EnvironmentWatcherRunReport; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander; -import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import java.io.IOException; -import java.io.PrintStream; import java.io.Serializable; import java.util.List; import java.util.Set; @@ -15,39 +17,49 @@ import java.util.stream.Collectors; public class EnvironmentWatcher implements Serializable { - private EnvVars envVars = null; - private EnvironmentExpander expander = null; + private EnvVars envVars; private Set watchedVars; private List scanResults; + private static EnvironmentWatcherRunReport runReport; private static final Logger LOGGER = Logger.getLogger(EnvironmentWatcher.class.getName()); - public EnvironmentWatcher(CpsStepContext context) { + public static EnvironmentWatcher of(CpsStepContext context, CpsFlowExecution exec) { try { - envVars = context.get(EnvVars.class); - expander = context.get(EnvironmentExpander.class); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } catch (IllegalArgumentException e) { - LOGGER.log(Level.WARNING, "Error storing the arguments for step: " + context.getStepDescriptor().getFunctionName(), e); - } - if (expander != null) { - watchedVars = expander.getSensitiveVars(); + EnvVars contextEnvVars = context.get(EnvVars.class); + EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); + if (contextEnvVars != null && contextExpander != null) { + if (runReport == null) { + FlowExecutionOwner owner = exec.getOwner(); + if (owner != null && owner.getExecutable() instanceof Run) { + runReport = ((Run) owner.getExecutable()).getAction(EnvironmentWatcherRunReport.class); + } + } + if (runReport != null) { + return new EnvironmentWatcher(contextEnvVars, contextExpander); + } + } + } catch (InterruptedException | IOException e) { + LOGGER.log(Level.FINE, "Unable to create EnvironmentWatcher instance.\n" + e.getMessage()); } + return null; + } + + public EnvironmentWatcher(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander) { + this.envVars = envVars; + watchedVars = expander.getSensitiveVars(); } public void scan(String text) { - // TODO: would EnvVars ever be null? - if (watchedVars == null || envVars == null) { - scanResults = null; - } else { - scanResults = watchedVars.stream().filter(e -> text.contains(envVars.get(e))).collect(Collectors.toList()); - } + scanResults = watchedVars.stream().filter(e -> text.contains(envVars.get(e))).collect(Collectors.toList()); } public void logResults(TaskListener listener) { if (scanResults != null && !scanResults.isEmpty()) { listener.getLogger().println("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + scanResults.toString()); + runReport.record(scanResults); } } + + private static final long serialVersionUID = 1L; } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherListener.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherListener.java new file mode 100644 index 000000000..c88891d7c --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherListener.java @@ -0,0 +1,31 @@ +package org.jenkinsci.plugins.workflow.cps.view; + +import hudson.Extension; +import hudson.model.Run; +import org.jenkinsci.plugins.workflow.flow.FlowExecution; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionListener; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Listener to add action for UI report for each run + */ +@Extension +public class EnvironmentWatcherListener extends FlowExecutionListener { + private static final Logger LOGGER = Logger.getLogger(EnvironmentWatcherListener.class.getName()); + + @Override + public void onRunning(@Nonnull FlowExecution execution) { + FlowExecutionOwner owner = execution.getOwner(); + try { + if (owner != null && owner.getExecutable() instanceof Run) { + ((Run) owner.getExecutable()).addAction(new EnvironmentWatcherRunReport()); + } + } catch (IOException e) { + LOGGER.warning(e.getMessage()); + } + } +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport.java new file mode 100644 index 000000000..e67ef63b0 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport.java @@ -0,0 +1,56 @@ +package org.jenkinsci.plugins.workflow.cps.view; + +import hudson.model.Run; +import jenkins.model.RunAction2; +import java.io.Serializable; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Action to generate the UI report for watched environment variables + */ +public class EnvironmentWatcherRunReport implements RunAction2, Serializable { + + private Set results; + private transient Run run; + + public String getIconFileName() { + return null; + } + + public String getDisplayName() { + return null; + } + + public String getUrlName() { + return null; + } + + public void record(List stepResults) { + if (results == null) { + results = new HashSet<>(); + } + results.addAll(stepResults); + } + + public Set getResults() { + return results; + } + + public boolean getInProgress() { + return run.isBuilding(); + } + + @Override + public void onAttached(Run run) { + this.run = run; + } + + @Override + public void onLoad(Run run) { + this.run = run; + } + + private static final long serialVersionUID = 1L; +} diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly new file mode 100644 index 000000000..efe24aa9a --- /dev/null +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly @@ -0,0 +1,37 @@ + + + + + + + Possible insecure use of sensitive variables: + + (in progress) + + + + + + + +
${variable}
+ + \ No newline at end of file From 20d781d51990f6971a07b6c1ea5bd699656fcdb3 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 2 Sep 2020 11:15:08 -0600 Subject: [PATCH 14/76] change from table to list, change icon --- .../cps/view/EnvironmentWatcherRunReport/summary.jelly | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly index efe24aa9a..17286bacb 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly @@ -21,17 +21,15 @@ THE SOFTWARE. - + Possible insecure use of sensitive variables: (in progress) - +
    -
- - +
  • ${variable}
    • -
    ${variable}
    +     \ No newline at end of file From b404458d32db852782e13c7bfc9b56458d9a80cc Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Sat, 5 Sep 2020 00:54:40 -0600 Subject: [PATCH 15/76] update jelly formatting, update unit test --- .../EnvironmentWatcherRunReport/summary.jelly | 2 +- .../plugins/workflow/cps/DSLTest.java | 51 +++++++++++-------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly index 17286bacb..0a11dc4de 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly @@ -28,7 +28,7 @@ THE SOFTWARE.
      -
    • ${variable}
      • +
    • ${variable}
      •
    diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index cbe1d21f9..1cf68d3b5 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -32,6 +32,7 @@ import hudson.model.AbstractDescribableImpl; import hudson.model.Descriptor; import hudson.model.Result; + import java.util.Collections; import java.util.List; import java.util.Map; @@ -51,7 +52,6 @@ import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import org.jenkinsci.plugins.workflow.steps.StepExecution; import org.jenkinsci.plugins.workflow.steps.SynchronousStepExecution; -import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep; import org.jenkinsci.plugins.workflow.testMetaStep.AmbiguousEchoLowerStep; import org.jenkinsci.plugins.workflow.testMetaStep.AmbiguousEchoUpperStep; @@ -170,27 +170,6 @@ private static class Exec extends SynchronousStepExecution { } } - @Test public void withCredentials() throws Exception { - final String credentialsId = "creds"; - final String username = "bob"; - final String password = "secr3t"; - UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); - CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - String shellStep = Functions.isWindows()? "bat \"echo $PASSWORD\"\n" : "sh \"echo $PASSWORD\"\n"; - p.setDefinition(new CpsFlowDefinition("" - + "node {\n" - + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" - + shellStep - + " echo USERNAME\n" - + " echo \"username is $USERNAME\"\n" - + "}\n" - + "}", true)); - WorkflowRun workflowRun = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - r.assertLogContains("Affected variables: [PASSWORD]", workflowRun); - r.assertLogContains("Affected variables: [USERNAME]", workflowRun); - } - /** * Tests the ability to execute meta-step with clean syntax */ @@ -447,6 +426,34 @@ public void namedSoleParamForStep() throws Exception { "'org.jenkinsci.plugins.workflow.steps.SleepStep': comment,units", b); } + //TODO: JENKINS-47101, remove safe list check and change $PASSWORD variable to $TEMP + @Test public void sensitiveVarsLogging() throws Exception { + final String credentialsId = "creds"; + final String username = "bob"; + final String password = "secr3t"; + UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); + CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + String shellStep = Functions.isWindows()? "bat \"echo $PASSWORD\"\n" : "sh \"echo $PASSWORD\"\n"; + p.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" + + shellStep + + "}\n" + + "}", true)); + WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + r.assertLogContains("Affected variables: [PASSWORD]", run); + LinearScanner scan = new LinearScanner(); + FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("sh")); + ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); + Assert.assertFalse(argAction.isUnmodifiedArguments()); + Assert.assertTrue(argAction.getArguments().values().iterator().next() instanceof ArgumentsAction.NotStoredReason); + } + + @Test public void argumentsSafeList() throws Exception { + + } + public static class CLStep extends Step { public final String name; @DataBoundConstructor public CLStep(String name) {this.name = name;} From 22c4ae3b978a397ec02c5c4131ac54bac95a07d8 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 8 Sep 2020 09:52:07 -0600 Subject: [PATCH 16/76] Check for empty body --- .../plugins/workflow/cps/CpsStepContext.java | 8 ++++---- .../org/jenkinsci/plugins/workflow/cps/DSL.java | 13 ++++++++++++- .../org/jenkinsci/plugins/workflow/cps/DSLTest.java | 11 ++++++++++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java index 51059619c..9f24017d1 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java @@ -189,11 +189,11 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla this.stepDescriptorId = step.getId(); } - public void setBody(@Nonnull Closure bodyToSet, @Nonnull CpsThread thread) { - if (this.body == null) { - this.body = thread.group.export(bodyToSet); - } else { + public void setBody(Closure bodyToSet, @Nonnull CpsThread thread) { + if (this.body != null) { throw new IllegalStateException("Context already has a body"); + } else if (bodyToSet != null) { + this.body = thread.group.export(bodyToSet); } } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 10ca4540f..c4f53be38 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -26,6 +26,7 @@ import com.cloudbees.groovy.cps.Continuable; import com.cloudbees.groovy.cps.Outcome; +import com.cloudbees.groovy.cps.impl.CpsClosure; import groovy.lang.Closure; import groovy.lang.GString; import groovy.lang.GroovyObject; @@ -43,6 +44,7 @@ import java.io.IOException; import java.io.Serializable; import java.lang.annotation.Annotation; +import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -225,7 +227,16 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { FlowNode an; CpsThread thread = CpsThread.current(); - if (!d.takesImplicitBlockArgument() && !hack) { + boolean hasBody = false; + if (args.getClass().isArray()) { + for (int i = 0; i < Array.getLength(args); i++) { + if (Array.get(args, i) instanceof CpsClosure) { + hasBody = true; + break; + } + } + } + if (!hasBody && !hack) { an = new StepAtomNode(exec, d, thread.head.get()); } else { an = new StepStartNode(exec, d, thread.head.get()); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 1cf68d3b5..047b2f184 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -450,8 +450,17 @@ public void namedSoleParamForStep() throws Exception { Assert.assertTrue(argAction.getArguments().values().iterator().next() instanceof ArgumentsAction.NotStoredReason); } - @Test public void argumentsSafeList() throws Exception { + @Test public void noBody() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p.setDefinition((new CpsFlowDefinition("echo('hello')", true))); + r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + @Test public void noBodyError() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p.setDefinition((new CpsFlowDefinition("node{timeout(time: 1, unit: 'SECONDS')}", true))); + WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); + r.assertLogContains("There is no body to invoke", b); } public static class CLStep extends Step { From 41984a6eba0b94768e79068dd71553c268f24149 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 10 Sep 2020 01:16:32 -0600 Subject: [PATCH 17/76] Update body check, support legacy stage behavior --- pom.xml | 8 ++++- .../jenkinsci/plugins/workflow/cps/DSL.java | 17 ++++++---- .../plugins/workflow/cps/DSLTest.java | 32 +++++++++++++++++-- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 0cdc9d181..9cbd3deb1 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ 8 false 1.32 + 2.3     @@ -256,9 +257,14 @@ org.jenkins-ci.plugins pipeline-stage-step - 2.3 + ${pipeline-stage-step.version} test + + org.jenkins-ci.plugins + pipeline-stage-step + ${pipeline-stage-step.version} + diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index c4f53be38..8c733efc6 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -86,6 +86,7 @@ import org.jenkinsci.plugins.workflow.steps.StepContext; import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import org.jenkinsci.plugins.workflow.steps.StepExecution; +import org.jenkinsci.plugins.workflow.support.steps.StageStep; import org.jvnet.hudson.annotation_indexer.Index; import org.kohsuke.stapler.ClassDescriptor; import org.kohsuke.stapler.NoStaplerConstructorException; @@ -228,16 +229,20 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { FlowNode an; CpsThread thread = CpsThread.current(); boolean hasBody = false; - if (args.getClass().isArray()) { - for (int i = 0; i < Array.getLength(args); i++) { - if (Array.get(args, i) instanceof CpsClosure) { + if (!hack) { + if (args.getClass().isArray()) { + if (Array.get(args, Array.getLength(args) - 1) instanceof CpsClosure) { hasBody = true; - break; } } } - if (!hasBody && !hack) { - an = new StepAtomNode(exec, d, thread.head.get()); + if (!hack && !hasBody) { + // Legacy Stage Step support means the step has no body but still takesImplicitBlockArgument + if (!(d instanceof StageStep.DescriptorImpl) && d.takesImplicitBlockArgument()) { + throw new IllegalStateException(String.format("%s step must be called with a body", name)); + } else { + an = new StepAtomNode(exec, d, thread.head.get()); + } } else { an = new StepStartNode(exec, d, thread.head.get()); } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 047b2f184..850af26c9 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -426,7 +426,7 @@ public void namedSoleParamForStep() throws Exception { "'org.jenkinsci.plugins.workflow.steps.SleepStep': comment,units", b); } - //TODO: JENKINS-47101, remove safe list check and change $PASSWORD variable to $TEMP + //TODO: JENKINS-47101, remove safe list check and change $PASSWORD variable to an old safe list variable @Test public void sensitiveVarsLogging() throws Exception { final String credentialsId = "creds"; final String username = "bob"; @@ -460,7 +460,35 @@ public void namedSoleParamForStep() throws Exception { WorkflowJob p = r.createProject(WorkflowJob.class, "p"); p.setDefinition((new CpsFlowDefinition("node{timeout(time: 1, unit: 'SECONDS')}", true))); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains("There is no body to invoke", b); + r.assertLogContains("timeout step must be called with a body", b); + } + + @Test public void legacyStage() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition( + "stage(name: 'A');\n" + + "echo('in A');\n" + + "stage(name: 'B');\n" + + "echo('in B');\n" + + "echo('done')", true)); + WorkflowRun b = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + } + + @Test public void standardStage() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition( + "node{\n" + + " stage ('Build') {\n" + + " sh \"echo 'Building'\"\n" + + " }\n" + + " stage ('Test') {\n" + + " sh \"echo 'testing'\"\n" + + " }\n" + + " stage ('Deploy') {\n" + + " sh \"echo 'deploy'\"\n" + + " }\n" + + "}\n", true)); + WorkflowRun b = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); } public static class CLStep extends Step { From d4759b12fc6d797cc785bda0af14e309164259d8 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 10 Sep 2020 08:43:40 -0600 Subject: [PATCH 18/76] add check for empty args --- src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 8c733efc6..1ada09e70 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -230,8 +230,9 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { CpsThread thread = CpsThread.current(); boolean hasBody = false; if (!hack) { - if (args.getClass().isArray()) { - if (Array.get(args, Array.getLength(args) - 1) instanceof CpsClosure) { + if (args != null && args.getClass().isArray()) { + int size = Array.getLength(args); + if (size > 0 && Array.get(args, size - 1) instanceof CpsClosure) { hasBody = true; } } From 4a9807282e290a06d459575f24c9b959a4dc5113 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 10 Sep 2020 10:36:27 -0600 Subject: [PATCH 19/76] fix variable clashing --- .../org/jenkinsci/plugins/workflow/cps/DSLTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 850af26c9..ce6b5823a 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -433,7 +433,7 @@ public void namedSoleParamForStep() throws Exception { final String password = "secr3t"; UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p = r.createProject(WorkflowJob.class, "p"); String shellStep = Functions.isWindows()? "bat \"echo $PASSWORD\"\n" : "sh \"echo $PASSWORD\"\n"; p.setDefinition(new CpsFlowDefinition("" + "node {\n" @@ -451,20 +451,20 @@ public void namedSoleParamForStep() throws Exception { } @Test public void noBody() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p = r.createProject(WorkflowJob.class, "p"); p.setDefinition((new CpsFlowDefinition("echo('hello')", true))); r.assertBuildStatusSuccess(p.scheduleBuild2(0)); } @Test public void noBodyError() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p = r.createProject(WorkflowJob.class, "p"); p.setDefinition((new CpsFlowDefinition("node{timeout(time: 1, unit: 'SECONDS')}", true))); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); r.assertLogContains("timeout step must be called with a body", b); } @Test public void legacyStage() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p = r.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( "stage(name: 'A');\n" + "echo('in A');\n" + @@ -475,7 +475,7 @@ public void namedSoleParamForStep() throws Exception { } @Test public void standardStage() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p = r.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( "node{\n" + " stage ('Build') {\n" + From 8bdc019304535bead1f991374898b23c12ba5513 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 10 Sep 2020 13:31:41 -0600 Subject: [PATCH 20/76] address review comments refactor EnvironmentWatcher --- pom.xml | 8 +- .../jenkinsci/plugins/workflow/cps/DSL.java | 28 ++++--- ....java => InterpolatedSecretsDetector.java} | 19 ++--- .../cps/view/EnvironmentWatcherListener.java | 31 ------- .../cps/view/EnvironmentWatcherRunReport.java | 56 ------------- ...terpolatedSecretsDetectorReportAction.java | 83 +++++++++++++++++++ .../plugins/workflow/cps/DSLTest.java | 10 +-- 7 files changed, 115 insertions(+), 120 deletions(-) rename src/main/java/org/jenkinsci/plugins/workflow/cps/{EnvironmentWatcher.java => InterpolatedSecretsDetector.java} (74%) delete mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherListener.java delete mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport.java create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsDetectorReportAction.java diff --git a/pom.xml b/pom.xml index 9cbd3deb1..0cdc9d181 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,6 @@ 8 false 1.32 - 2.3 @@ -257,14 +256,9 @@ org.jenkins-ci.plugins pipeline-stage-step - ${pipeline-stage-step.version} + 2.3 test - - org.jenkins-ci.plugins - pipeline-stage-step - ${pipeline-stage-step.version} - diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 1ada09e70..f4e308653 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -86,7 +86,6 @@ import org.jenkinsci.plugins.workflow.steps.StepContext; import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import org.jenkinsci.plugins.workflow.steps.StepExecution; -import org.jenkinsci.plugins.workflow.support.steps.StageStep; import org.jvnet.hudson.annotation_indexer.Index; import org.kohsuke.stapler.ClassDescriptor; import org.kohsuke.stapler.NoStaplerConstructorException; @@ -230,16 +229,23 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { CpsThread thread = CpsThread.current(); boolean hasBody = false; if (!hack) { - if (args != null && args.getClass().isArray()) { - int size = Array.getLength(args); - if (size > 0 && Array.get(args, size - 1) instanceof CpsClosure) { - hasBody = true; + if (args != null) { + if (args instanceof NamedArgsAndClosure) { + if (((NamedArgsAndClosure) args).body != null) { + hasBody = true; + } + } else if (args.getClass().isArray()) { + int size = Array.getLength(args); + if (size > 0 && Array.get(args, size - 1) instanceof CpsClosure) { + hasBody = true; + } } } } if (!hack && !hasBody) { // Legacy Stage Step support means the step has no body but still takesImplicitBlockArgument - if (!(d instanceof StageStep.DescriptorImpl) && d.takesImplicitBlockArgument()) { + if (!(d.getClass().getName().equals("org.jenkinsci.plugins.workflow.support.steps.StageStep$DescriptorImpl")) + && d.takesImplicitBlockArgument()) { throw new IllegalStateException(String.format("%s step must be called with a body", name)); } else { an = new StepAtomNode(exec, d, thread.head.get()); @@ -249,7 +255,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } CpsStepContext context = new CpsStepContext(d, thread, handle, an); - EnvironmentWatcher envWatcher = EnvironmentWatcher.of(context, exec); + InterpolatedSecretsDetector envWatcher = InterpolatedSecretsDetector.of(context, exec); NamedArgsAndClosure ps = parseArgs(args, d, envWatcher); context.setBody(ps.body, thread); // Ensure ArgumentsAction is attached before we notify even synchronous listeners: @@ -478,7 +484,7 @@ static class NamedArgsAndClosure { final Closure body; final List msgs; - private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable EnvironmentWatcher envWatcher) { + private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable InterpolatedSecretsDetector envWatcher) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; this.msgs = new ArrayList<>(); @@ -501,7 +507,7 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable Environm * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v, @Nullable EnvironmentWatcher envWatcher) { + private static Object flattenGString(Object v, @Nullable InterpolatedSecretsDetector envWatcher) { if (v instanceof GString) { String flattened = v.toString(); if (envWatcher != null) { @@ -534,7 +540,7 @@ private static Object flattenGString(Object v, @Nullable EnvironmentWatcher envW } } - static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, EnvironmentWatcher envWatcher) { + static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, InterpolatedSecretsDetector envWatcher) { boolean singleArgumentOnly = false; try { DescribableModel stepModel = DescribableModel.of(d.clazz); @@ -575,7 +581,7 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, EnvironmentWa // * @param envVars * The environment variables of the context */ - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable EnvironmentWatcher envWatcher) { + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable InterpolatedSecretsDetector envWatcher) { if (arg instanceof NamedArgsAndClosure) return (NamedArgsAndClosure) arg; if (arg instanceof Map) // TODO is this clause actually used? diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java similarity index 74% rename from src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java rename to src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java index 373ab6208..6ab12d942 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/EnvironmentWatcher.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java @@ -3,28 +3,27 @@ import hudson.EnvVars; import hudson.model.Run; import hudson.model.TaskListener; -import org.jenkinsci.plugins.workflow.cps.view.EnvironmentWatcherRunReport; +import org.jenkinsci.plugins.workflow.cps.view.InterpolatedSecretsDetectorReportAction; import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander; import javax.annotation.Nonnull; import java.io.IOException; -import java.io.Serializable; import java.util.List; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -public class EnvironmentWatcher implements Serializable { +public class InterpolatedSecretsDetector { private EnvVars envVars; private Set watchedVars; private List scanResults; - private static EnvironmentWatcherRunReport runReport; + private static InterpolatedSecretsDetectorReportAction runReport; - private static final Logger LOGGER = Logger.getLogger(EnvironmentWatcher.class.getName()); + private static final Logger LOGGER = Logger.getLogger(InterpolatedSecretsDetector.class.getName()); - public static EnvironmentWatcher of(CpsStepContext context, CpsFlowExecution exec) { + public static InterpolatedSecretsDetector of(CpsStepContext context, CpsFlowExecution exec) { try { EnvVars contextEnvVars = context.get(EnvVars.class); EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); @@ -32,20 +31,20 @@ public static EnvironmentWatcher of(CpsStepContext context, CpsFlowExecution exe if (runReport == null) { FlowExecutionOwner owner = exec.getOwner(); if (owner != null && owner.getExecutable() instanceof Run) { - runReport = ((Run) owner.getExecutable()).getAction(EnvironmentWatcherRunReport.class); + runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsDetectorReportAction.class); } } if (runReport != null) { - return new EnvironmentWatcher(contextEnvVars, contextExpander); + return new InterpolatedSecretsDetector(contextEnvVars, contextExpander); } } } catch (InterruptedException | IOException e) { - LOGGER.log(Level.FINE, "Unable to create EnvironmentWatcher instance.\n" + e.getMessage()); + LOGGER.log(Level.FINE, "Unable to create EnvironmentWatcher instance."); } return null; } - public EnvironmentWatcher(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander) { + public InterpolatedSecretsDetector(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander) { this.envVars = envVars; watchedVars = expander.getSensitiveVars(); } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherListener.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherListener.java deleted file mode 100644 index c88891d7c..000000000 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherListener.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.jenkinsci.plugins.workflow.cps.view; - -import hudson.Extension; -import hudson.model.Run; -import org.jenkinsci.plugins.workflow.flow.FlowExecution; -import org.jenkinsci.plugins.workflow.flow.FlowExecutionListener; -import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; - -import javax.annotation.Nonnull; -import java.io.IOException; -import java.util.logging.Logger; - -/** - * Listener to add action for UI report for each run - */ -@Extension -public class EnvironmentWatcherListener extends FlowExecutionListener { - private static final Logger LOGGER = Logger.getLogger(EnvironmentWatcherListener.class.getName()); - - @Override - public void onRunning(@Nonnull FlowExecution execution) { - FlowExecutionOwner owner = execution.getOwner(); - try { - if (owner != null && owner.getExecutable() instanceof Run) { - ((Run) owner.getExecutable()).addAction(new EnvironmentWatcherRunReport()); - } - } catch (IOException e) { - LOGGER.warning(e.getMessage()); - } - } -} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport.java deleted file mode 100644 index e67ef63b0..000000000 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.jenkinsci.plugins.workflow.cps.view; - -import hudson.model.Run; -import jenkins.model.RunAction2; -import java.io.Serializable; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -/** - * Action to generate the UI report for watched environment variables - */ -public class EnvironmentWatcherRunReport implements RunAction2, Serializable { - - private Set results; - private transient Run run; - - public String getIconFileName() { - return null; - } - - public String getDisplayName() { - return null; - } - - public String getUrlName() { - return null; - } - - public void record(List stepResults) { - if (results == null) { - results = new HashSet<>(); - } - results.addAll(stepResults); - } - - public Set getResults() { - return results; - } - - public boolean getInProgress() { - return run.isBuilding(); - } - - @Override - public void onAttached(Run run) { - this.run = run; - } - - @Override - public void onLoad(Run run) { - this.run = run; - } - - private static final long serialVersionUID = 1L; -} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsDetectorReportAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsDetectorReportAction.java new file mode 100644 index 000000000..54da9eb9d --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsDetectorReportAction.java @@ -0,0 +1,83 @@ +package org.jenkinsci.plugins.workflow.cps.view; + +import hudson.Extension; +import hudson.model.Run; +import jenkins.model.RunAction2; +import org.jenkinsci.plugins.workflow.flow.FlowExecution; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionListener; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; + +/** + * Action to generate the UI report for watched environment variables + */ +public class InterpolatedSecretsDetectorReportAction implements RunAction2 { + + private Set results; + private transient Run run; + + public String getIconFileName() { + return null; + } + + public String getDisplayName() { + return null; + } + + public String getUrlName() { + return null; + } + + public void record(List stepResults) { + if (results == null) { + results = new HashSet<>(); + } + results.addAll(stepResults); + } + + public Set getResults() { + return results; + } + + public boolean getInProgress() { + return run.isBuilding(); + } + + @Override + public void onAttached(Run run) { + this.run = run; + } + + @Override + public void onLoad(Run run) { + this.run = run; + } + + /** + * Listener to add action for UI report for each run + */ + @Extension + public static class EnvironmentWatcherListener extends FlowExecutionListener { + private static final Logger LOGGER = Logger.getLogger(InterpolatedSecretsDetectorReportAction.EnvironmentWatcherListener.class.getName()); + + @Override + public void onRunning(@Nonnull FlowExecution execution) { + FlowExecutionOwner owner = execution.getOwner(); + try { + if (owner != null && owner.getExecutable() instanceof Run) { + ((Run) owner.getExecutable()).addAction(new InterpolatedSecretsDetectorReportAction()); + } + } catch (IOException e) { + LOGGER.warning(e.getMessage()); + } + } + } + + private static final long serialVersionUID = 1L; +} diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index ce6b5823a..902c40f56 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -433,7 +433,7 @@ public void namedSoleParamForStep() throws Exception { final String password = "secr3t"; UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); - p = r.createProject(WorkflowJob.class, "p"); +// p = r.createProject(WorkflowJob.class, "p"); String shellStep = Functions.isWindows()? "bat \"echo $PASSWORD\"\n" : "sh \"echo $PASSWORD\"\n"; p.setDefinition(new CpsFlowDefinition("" + "node {\n" @@ -451,20 +451,20 @@ public void namedSoleParamForStep() throws Exception { } @Test public void noBody() throws Exception { - p = r.createProject(WorkflowJob.class, "p"); +// p = r.createProject(WorkflowJob.class, "p"); p.setDefinition((new CpsFlowDefinition("echo('hello')", true))); r.assertBuildStatusSuccess(p.scheduleBuild2(0)); } @Test public void noBodyError() throws Exception { - p = r.createProject(WorkflowJob.class, "p"); +// p = r.createProject(WorkflowJob.class, "p"); p.setDefinition((new CpsFlowDefinition("node{timeout(time: 1, unit: 'SECONDS')}", true))); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); r.assertLogContains("timeout step must be called with a body", b); } @Test public void legacyStage() throws Exception { - p = r.createProject(WorkflowJob.class, "p"); +// p = r.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( "stage(name: 'A');\n" + "echo('in A');\n" + @@ -475,7 +475,7 @@ public void namedSoleParamForStep() throws Exception { } @Test public void standardStage() throws Exception { - p = r.createProject(WorkflowJob.class, "p"); +// p = r.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( "node{\n" + " stage ('Build') {\n" + From e975453e2e6bf38f1cd7f76d369027b4ffb2a8a3 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 10 Sep 2020 14:35:32 -0600 Subject: [PATCH 21/76] Refactor Action name, generate action only when there are secrets exposed --- .../cps/InterpolatedSecretsDetector.java | 31 ++++++++------- ...on.java => InterpolatedSecretsAction.java} | 23 +---------- .../plugins/workflow/cps/DSLTest.java | 38 +++++++------------ 3 files changed, 31 insertions(+), 61 deletions(-) rename src/main/java/org/jenkinsci/plugins/workflow/cps/view/{InterpolatedSecretsDetectorReportAction.java => InterpolatedSecretsAction.java} (58%) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java index 6ab12d942..6334bbfa9 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java @@ -3,7 +3,7 @@ import hudson.EnvVars; import hudson.model.Run; import hudson.model.TaskListener; -import org.jenkinsci.plugins.workflow.cps.view.InterpolatedSecretsDetectorReportAction; +import org.jenkinsci.plugins.workflow.cps.view.InterpolatedSecretsAction; import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander; @@ -19,7 +19,7 @@ public class InterpolatedSecretsDetector { private EnvVars envVars; private Set watchedVars; private List scanResults; - private static InterpolatedSecretsDetectorReportAction runReport; + private CpsFlowExecution exec; private static final Logger LOGGER = Logger.getLogger(InterpolatedSecretsDetector.class.getName()); @@ -28,15 +28,7 @@ public static InterpolatedSecretsDetector of(CpsStepContext context, CpsFlowExec EnvVars contextEnvVars = context.get(EnvVars.class); EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); if (contextEnvVars != null && contextExpander != null) { - if (runReport == null) { - FlowExecutionOwner owner = exec.getOwner(); - if (owner != null && owner.getExecutable() instanceof Run) { - runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsDetectorReportAction.class); - } - } - if (runReport != null) { - return new InterpolatedSecretsDetector(contextEnvVars, contextExpander); - } + return new InterpolatedSecretsDetector(contextEnvVars, contextExpander, exec); } } catch (InterruptedException | IOException e) { LOGGER.log(Level.FINE, "Unable to create EnvironmentWatcher instance."); @@ -44,19 +36,30 @@ public static InterpolatedSecretsDetector of(CpsStepContext context, CpsFlowExec return null; } - public InterpolatedSecretsDetector(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander) { + public InterpolatedSecretsDetector(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander, @Nonnull CpsFlowExecution exec) { this.envVars = envVars; watchedVars = expander.getSensitiveVars(); + this.exec = exec; } public void scan(String text) { scanResults = watchedVars.stream().filter(e -> text.contains(envVars.get(e))).collect(Collectors.toList()); } - public void logResults(TaskListener listener) { + public void logResults(TaskListener listener) throws IOException { if (scanResults != null && !scanResults.isEmpty()) { listener.getLogger().println("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + scanResults.toString()); - runReport.record(scanResults); + FlowExecutionOwner owner = exec.getOwner(); + if (owner != null && owner.getExecutable() instanceof Run) { + InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); + if (runReport == null) { + runReport = new InterpolatedSecretsAction(); + ((Run) owner.getExecutable()).addAction(runReport); + } + runReport.record(scanResults); + } else { + LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); + } } } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsDetectorReportAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java similarity index 58% rename from src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsDetectorReportAction.java rename to src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 54da9eb9d..323c399f0 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsDetectorReportAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -17,7 +17,7 @@ /** * Action to generate the UI report for watched environment variables */ -public class InterpolatedSecretsDetectorReportAction implements RunAction2 { +public class InterpolatedSecretsAction implements RunAction2 { private Set results; private transient Run run; @@ -59,25 +59,4 @@ public void onLoad(Run run) { this.run = run; } - /** - * Listener to add action for UI report for each run - */ - @Extension - public static class EnvironmentWatcherListener extends FlowExecutionListener { - private static final Logger LOGGER = Logger.getLogger(InterpolatedSecretsDetectorReportAction.EnvironmentWatcherListener.class.getName()); - - @Override - public void onRunning(@Nonnull FlowExecution execution) { - FlowExecutionOwner owner = execution.getOwner(); - try { - if (owner != null && owner.getExecutable() instanceof Run) { - ((Run) owner.getExecutable()).addAction(new InterpolatedSecretsDetectorReportAction()); - } - } catch (IOException e) { - LOGGER.warning(e.getMessage()); - } - } - } - - private static final long serialVersionUID = 1L; } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 902c40f56..6158f8850 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -39,7 +39,9 @@ import java.util.Set; import static org.hamcrest.Matchers.containsString; +import org.hamcrest.MatcherAssert; import org.jenkinsci.plugins.workflow.actions.ArgumentsAction; +import org.jenkinsci.plugins.workflow.cps.view.InterpolatedSecretsAction; import org.jenkinsci.plugins.workflow.graph.FlowNode; import org.jenkinsci.plugins.workflow.graphanalysis.LinearScanner; import org.jenkinsci.plugins.workflow.graphanalysis.NodeStepTypePredicate; @@ -55,6 +57,8 @@ import org.jenkinsci.plugins.workflow.testMetaStep.AmbiguousEchoLowerStep; import org.jenkinsci.plugins.workflow.testMetaStep.AmbiguousEchoUpperStep; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.*; import org.junit.Assert; @@ -433,7 +437,6 @@ public void namedSoleParamForStep() throws Exception { final String password = "secr3t"; UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); -// p = r.createProject(WorkflowJob.class, "p"); String shellStep = Functions.isWindows()? "bat \"echo $PASSWORD\"\n" : "sh \"echo $PASSWORD\"\n"; p.setDefinition(new CpsFlowDefinition("" + "node {\n" @@ -443,50 +446,35 @@ public void namedSoleParamForStep() throws Exception { + "}", true)); WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); r.assertLogContains("Affected variables: [PASSWORD]", run); + InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); + Assert.assertNotNull(reportAction); + Set reportResults = reportAction.getResults(); + MatcherAssert.assertThat(reportResults.size(), is(1)); + MatcherAssert.assertThat(reportResults.iterator().next(), is("PASSWORD")); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("sh")); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); Assert.assertFalse(argAction.isUnmodifiedArguments()); - Assert.assertTrue(argAction.getArguments().values().iterator().next() instanceof ArgumentsAction.NotStoredReason); - } - - @Test public void noBody() throws Exception { -// p = r.createProject(WorkflowJob.class, "p"); - p.setDefinition((new CpsFlowDefinition("echo('hello')", true))); - r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); } @Test public void noBodyError() throws Exception { -// p = r.createProject(WorkflowJob.class, "p"); - p.setDefinition((new CpsFlowDefinition("node{timeout(time: 1, unit: 'SECONDS')}", true))); + p.setDefinition((new CpsFlowDefinition("timeout(time: 1, unit: 'SECONDS')", true))); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); r.assertLogContains("timeout step must be called with a body", b); } @Test public void legacyStage() throws Exception { -// p = r.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( "stage(name: 'A');\n" + - "echo('in A');\n" + - "stage(name: 'B');\n" + - "echo('in B');\n" + "echo('done')", true)); WorkflowRun b = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); } @Test public void standardStage() throws Exception { -// p = r.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition( - "node{\n" + - " stage ('Build') {\n" + - " sh \"echo 'Building'\"\n" + - " }\n" + - " stage ('Test') {\n" + - " sh \"echo 'testing'\"\n" + - " }\n" + - " stage ('Deploy') {\n" + - " sh \"echo 'deploy'\"\n" + - " }\n" + + "stage('Build'){\n" + + " echo('building')\n" + "}\n", true)); WorkflowRun b = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); } From d4765a4748651138741760682fa3cc7c3e2bcdca Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 10 Sep 2020 15:17:20 -0600 Subject: [PATCH 22/76] update null environment variable test --- .../jenkinsci/plugins/workflow/cps/ParamsVariableTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/ParamsVariableTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/ParamsVariableTest.java index 200e8583c..51ee3c7f7 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/ParamsVariableTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/ParamsVariableTest.java @@ -30,6 +30,7 @@ import hudson.model.ParametersDefinitionProperty; import hudson.model.PasswordParameterDefinition; import hudson.model.PasswordParameterValue; +import hudson.model.Result; import hudson.model.StringParameterDefinition; import hudson.model.StringParameterValue; import org.jenkinsci.plugins.workflow.job.WorkflowJob; @@ -65,7 +66,8 @@ public class ParamsVariableTest { WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p"); p.setDefinition(new CpsFlowDefinition("echo(/TEXT=${params.TEXT}/)",true)); p.addProperty(new ParametersDefinitionProperty(new StringParameterDefinition("TEXT", ""))); - r.assertLogContains("TEXT=null", r.assertBuildStatusSuccess(p.scheduleBuild2(0, new ParametersAction(new StringParameterValue("TEXT", /* not possible via UI, but to simulate other ParameterValue impls */null))))); + WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0, new ParametersAction(new StringParameterValue("TEXT", /* not possible via UI, but to simulate other ParameterValue impls */null)))); + r.assertLogContains("Null value not allowed as an environment variable: TEXT", b); } } From 341972a29d42db8501af34a154a1a62de0f9eb02 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 10 Sep 2020 23:51:50 -0600 Subject: [PATCH 23/76] check for "bat" args on windows --- src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 6158f8850..6ed650024 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -452,7 +452,7 @@ public void namedSoleParamForStep() throws Exception { MatcherAssert.assertThat(reportResults.size(), is(1)); MatcherAssert.assertThat(reportResults.iterator().next(), is("PASSWORD")); LinearScanner scan = new LinearScanner(); - FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("sh")); + FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(Functions.isWindows()? "bat" : "sh")); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); Assert.assertFalse(argAction.isUnmodifiedArguments()); MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); From ade619f5aa4f3d22811164614e0460a95da6df49 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Fri, 11 Sep 2020 00:09:41 -0600 Subject: [PATCH 24/76] avoid reflective API --- src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index f4e308653..2d429c63c 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -234,9 +234,9 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { if (((NamedArgsAndClosure) args).body != null) { hasBody = true; } - } else if (args.getClass().isArray()) { - int size = Array.getLength(args); - if (size > 0 && Array.get(args, size - 1) instanceof CpsClosure) { + } else if (args instanceof Object[]) { + Object[] array = (Object[]) args; + if (array.length > 0 && array[array.length - 1] instanceof CpsClosure) { hasBody = true; } } From 79d255a8427bc41a575bb6f808b355dfaadecb3e Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Fri, 11 Sep 2020 15:40:07 -0600 Subject: [PATCH 25/76] address review comments --- .../jenkinsci/plugins/workflow/cps/DSL.java | 38 ++++++++++--------- .../cps/InterpolatedSecretsDetector.java | 6 +-- .../cps/view/InterpolatedSecretsAction.java | 8 ---- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 2d429c63c..cf91504b4 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -44,7 +44,6 @@ import java.io.IOException; import java.io.Serializable; import java.lang.annotation.Annotation; -import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -227,21 +226,8 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { FlowNode an; CpsThread thread = CpsThread.current(); - boolean hasBody = false; - if (!hack) { - if (args != null) { - if (args instanceof NamedArgsAndClosure) { - if (((NamedArgsAndClosure) args).body != null) { - hasBody = true; - } - } else if (args instanceof Object[]) { - Object[] array = (Object[]) args; - if (array.length > 0 && array[array.length - 1] instanceof CpsClosure) { - hasBody = true; - } - } - } - } + boolean hasBody = hack? false : argsHasBody(args); + if (!hack && !hasBody) { // Legacy Stage Step support means the step has no body but still takesImplicitBlockArgument if (!(d.getClass().getName().equals("org.jenkinsci.plugins.workflow.support.steps.StageStep$DescriptorImpl")) @@ -356,6 +342,23 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } } + // Check if step arguments contain a step body + private boolean argsHasBody(Object args) { + if (args != null) { + if (args instanceof NamedArgsAndClosure) { + if (((NamedArgsAndClosure) args).body != null) { + return true; + } + } else if (args instanceof Object[]) { + Object[] array = (Object[]) args; + if (array.length > 0 && array[array.length - 1] instanceof CpsClosure) { + return true; + } + } + } + return false; + } + private static String loadSoleArgumentKey(StepDescriptor d) { try { String[] names = new ClassDescriptor(d.clazz).loadConstructorParamNames(); @@ -451,7 +454,6 @@ protected Object invokeDescribable(String symbol, Object _args) { } private void reportAmbiguousStepInvocation(CpsStepContext context, StepDescriptor d, @Nullable TaskListener listener) { - Exception e = null; if (listener != null) { List ambiguousClassNames = StepDescriptor.all().stream() .filter(sd -> sd.getFunctionName().equals(d.getFunctionName())) @@ -465,7 +467,7 @@ private void reportAmbiguousStepInvocation(CpsStepContext context, StepDescripto listener.getLogger().println(message); return; } - LOGGER.log(Level.FINE, "Unable to report ambiguous step invocation for: " + d.getFunctionName(), e); + LOGGER.log(Level.FINE, "Unable to report ambiguous step invocation for: " + d.getFunctionName()); } /** Returns the capacity we need to allocate for a HashMap so it will hold all elements without needing to resize. */ diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java index 6334bbfa9..2e5eff4e8 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java @@ -16,10 +16,10 @@ import java.util.stream.Collectors; public class InterpolatedSecretsDetector { - private EnvVars envVars; - private Set watchedVars; + private final EnvVars envVars; + private final Set watchedVars; private List scanResults; - private CpsFlowExecution exec; + private final CpsFlowExecution exec; private static final Logger LOGGER = Logger.getLogger(InterpolatedSecretsDetector.class.getName()); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 323c399f0..a95295ecc 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -1,18 +1,10 @@ package org.jenkinsci.plugins.workflow.cps.view; -import hudson.Extension; import hudson.model.Run; import jenkins.model.RunAction2; -import org.jenkinsci.plugins.workflow.flow.FlowExecution; -import org.jenkinsci.plugins.workflow.flow.FlowExecutionListener; -import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; - -import javax.annotation.Nonnull; -import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.logging.Logger; /** * Action to generate the UI report for watched environment variables From 0abb6652ac68abbc95e504018f912561f2cbf709 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 14 Sep 2020 23:29:56 -0600 Subject: [PATCH 26/76] address review comments --- .../org/jenkinsci/plugins/workflow/cps/DSL.java | 15 +++++++-------- .../workflow/cps/InterpolatedSecretsDetector.java | 4 +--- .../EnvironmentWatcherRunReport/summary.jelly | 4 ++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index cf91504b4..9861e872c 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -26,7 +26,6 @@ import com.cloudbees.groovy.cps.Continuable; import com.cloudbees.groovy.cps.Outcome; -import com.cloudbees.groovy.cps.impl.CpsClosure; import groovy.lang.Closure; import groovy.lang.GString; import groovy.lang.GroovyObject; @@ -351,7 +350,7 @@ private boolean argsHasBody(Object args) { } } else if (args instanceof Object[]) { Object[] array = (Object[]) args; - if (array.length > 0 && array[array.length - 1] instanceof CpsClosure) { + if (array.length > 0 && array[array.length - 1] instanceof Closure) { return true; } } @@ -509,18 +508,18 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable Interpol * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v, @Nullable InterpolatedSecretsDetector envWatcher) { + private static Object flattenGString(Object v, @Nullable InterpolatedSecretsDetector secretsDetector) { if (v instanceof GString) { String flattened = v.toString(); - if (envWatcher != null) { - envWatcher.scan(flattened); + if (secretsDetector != null) { + secretsDetector.scan(flattened); } return flattened; } else if (v instanceof List) { boolean mutated = false; List r = new ArrayList<>(); for (Object o : ((List) v)) { - Object o2 = flattenGString(o, envWatcher); + Object o2 = flattenGString(o, secretsDetector); mutated |= o != o2; r.add(o2); } @@ -530,9 +529,9 @@ private static Object flattenGString(Object v, @Nullable InterpolatedSecretsDete Map r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) v).size())); for (Map.Entry e : ((Map) v).entrySet()) { Object k = e.getKey(); - Object k2 = flattenGString(k, envWatcher); + Object k2 = flattenGString(k, secretsDetector); Object o = e.getValue(); - Object o2 = flattenGString(o, envWatcher); + Object o2 = flattenGString(o, secretsDetector); mutated |= k != k2 || o != o2; r.put(k2, o2); } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java index 2e5eff4e8..34f6c02d2 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java @@ -36,7 +36,7 @@ public static InterpolatedSecretsDetector of(CpsStepContext context, CpsFlowExec return null; } - public InterpolatedSecretsDetector(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander, @Nonnull CpsFlowExecution exec) { + private InterpolatedSecretsDetector(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander, @Nonnull CpsFlowExecution exec) { this.envVars = envVars; watchedVars = expander.getSensitiveVars(); this.exec = exec; @@ -62,6 +62,4 @@ public void logResults(TaskListener listener) throws IOException { } } } - - private static final long serialVersionUID = 1L; } diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly index 0a11dc4de..9777e78bb 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly @@ -22,9 +22,9 @@ THE SOFTWARE. - Possible insecure use of sensitive variables: + ${%Possible insecure use of sensitive variables:} - (in progress) + ${%(in progress)}
      From 7a5d0c7536dfdb424e3ea30a04af57932c89ead4 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 15 Sep 2020 00:19:51 -0600 Subject: [PATCH 27/76] update jelly file path, fix localization error with parenthesis --- .../summary.jelly | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/main/resources/org/jenkinsci/plugins/workflow/cps/view/{EnvironmentWatcherRunReport => InterpolatedSecretsAction}/summary.jelly (98%) diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly similarity index 98% rename from src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly rename to src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index 9777e78bb..a05333449 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/EnvironmentWatcherRunReport/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -24,7 +24,7 @@ THE SOFTWARE. ${%Possible insecure use of sensitive variables:} - ${%(in progress)} + (${%in progress})
        From 506d5b3920e47cc738d58c377ebd08b53946f6e3 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 15 Sep 2020 00:30:12 -0600 Subject: [PATCH 28/76] add placeholder explanation page for jelly --- .../workflow/cps/view/InterpolatedSecretsAction/summary.jelly | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index a05333449..c9ff5f71e 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -22,7 +22,9 @@ THE SOFTWARE. - ${%Possible insecure use of sensitive variables:} + ${%Possible insecure use of sensitive variables} + + (${%click here for an explanation}): (${%in progress}) From 93b018bef6f565724748bd84dc5288e090230170 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 15 Sep 2020 08:28:37 -0600 Subject: [PATCH 29/76] more refactoring of envwatcher --- .../jenkinsci/plugins/workflow/cps/DSL.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 9861e872c..920d6e7cd 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -240,8 +240,8 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } CpsStepContext context = new CpsStepContext(d, thread, handle, an); - InterpolatedSecretsDetector envWatcher = InterpolatedSecretsDetector.of(context, exec); - NamedArgsAndClosure ps = parseArgs(args, d, envWatcher); + InterpolatedSecretsDetector secretsDetector = InterpolatedSecretsDetector.of(context, exec); + NamedArgsAndClosure ps = parseArgs(args, d, secretsDetector); context.setBody(ps.body, thread); // Ensure ArgumentsAction is attached before we notify even synchronous listeners: try { @@ -279,8 +279,8 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { DescribableModel stepModel = DescribableModel.of(d.clazz); s = stepModel.instantiate(ps.namedArgs, listener); } - if (envWatcher != null) { - envWatcher.logResults(listener); + if (secretsDetector != null) { + secretsDetector.logResults(listener); } // Persist the node - block start and end nodes do their own persistence. @@ -485,14 +485,14 @@ static class NamedArgsAndClosure { final Closure body; final List msgs; - private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable InterpolatedSecretsDetector envWatcher) { + private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable InterpolatedSecretsDetector secretsDetector) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; this.msgs = new ArrayList<>(); for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more - Object v = flattenGString(entry.getValue(), envWatcher); + Object v = flattenGString(entry.getValue(), secretsDetector); this.namedArgs.put(k, v); } } @@ -541,7 +541,7 @@ private static Object flattenGString(Object v, @Nullable InterpolatedSecretsDete } } - static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, InterpolatedSecretsDetector envWatcher) { + static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, InterpolatedSecretsDetector secretsDetector) { boolean singleArgumentOnly = false; try { DescribableModel stepModel = DescribableModel.of(d.clazz); @@ -549,12 +549,12 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, InterpolatedS if (singleArgumentOnly) { // Can fetch the one argument we need DescribableParameter dp = stepModel.getSoleRequiredParameter(); String paramName = (dp != null) ? dp.getName() : null; - return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, envWatcher); + return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, secretsDetector); } } catch (NoStaplerConstructorException e) { // Ignore steps without databound constructors and treat them as normal. } - return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, envWatcher); + return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, secretsDetector); } /** @@ -582,18 +582,18 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, InterpolatedS // * @param envVars * The environment variables of the context */ - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable InterpolatedSecretsDetector envWatcher) { + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable InterpolatedSecretsDetector secretsDetector) { if (arg instanceof NamedArgsAndClosure) return (NamedArgsAndClosure) arg; if (arg instanceof Map) // TODO is this clause actually used? - return new NamedArgsAndClosure((Map) arg, null, envWatcher); + return new NamedArgsAndClosure((Map) arg, null, secretsDetector); if (arg instanceof Closure && expectsBlock) - return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, envWatcher); + return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, secretsDetector); if (arg instanceof Object[]) {// this is how Groovy appears to pack argument list into one Object for invokeMethod List a = Arrays.asList((Object[])arg); if (a.size()==0) - return new NamedArgsAndClosure(Collections.emptyMap(),null, envWatcher); + return new NamedArgsAndClosure(Collections.emptyMap(),null, secretsDetector); Closure c=null; @@ -608,21 +608,21 @@ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String so if (!singleRequiredArg || (soleArgumentKey != null && mapArg.size() == 1 && mapArg.containsKey(soleArgumentKey))) { // this is how Groovy passes in Map - return new NamedArgsAndClosure(mapArg, c, envWatcher); + return new NamedArgsAndClosure(mapArg, c, secretsDetector); } } switch (a.size()) { case 0: - return new NamedArgsAndClosure(Collections.emptyMap(),c, envWatcher); + return new NamedArgsAndClosure(Collections.emptyMap(),c, secretsDetector); case 1: - return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, envWatcher); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, secretsDetector); default: throw new IllegalArgumentException("Expected named arguments but got "+a); } } - return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, envWatcher); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, secretsDetector); } private static Map singleParam(String soleArgumentKey, Object arg) { if (soleArgumentKey != null) { From 065b129c412e198843c6a38df374abdb9504afa7 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 15 Sep 2020 10:54:03 -0600 Subject: [PATCH 30/76] update step-api dependency --- pom.xml | 5 +++-- .../plugins/workflow/cps/InterpolatedSecretsDetector.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 0cdc9d181..d1a524668 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ 8 false 1.32 + 2.23-rc567.0fe52fbdf6b5 @@ -84,7 +85,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api - 2.23-rc566.c1fa948b49be + ${workflow-step-api.version} org.jenkins-ci.plugins.workflow @@ -125,7 +126,7 @@ org.jenkins-ci.plugins.workflow workflow-step-api - 2.23-rc566.c1fa948b49be + ${workflow-step-api.version} tests test diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java index 34f6c02d2..f89962a1b 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java @@ -38,7 +38,7 @@ public static InterpolatedSecretsDetector of(CpsStepContext context, CpsFlowExec private InterpolatedSecretsDetector(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander, @Nonnull CpsFlowExecution exec) { this.envVars = envVars; - watchedVars = expander.getSensitiveVars(); + watchedVars = expander.getSensitiveVariables(); this.exec = exec; } From 4903d4e2b9cb0affec679b5c56f7e8dc1f573a83 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 16 Sep 2020 01:10:16 -0600 Subject: [PATCH 31/76] support detecting interpolation in describables --- pom.xml | 2 +- .../jenkinsci/plugins/workflow/cps/DSL.java | 16 +++++++++--- .../plugins/workflow/cps/DSLTest.java | 26 +++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index d1a524668..261fff33c 100644 --- a/pom.xml +++ b/pom.xml @@ -160,7 +160,7 @@ org.jenkins-ci.plugins credentials-binding - 1.24-rc370.8c9bfdc92872 + 1.24-rc374.2fd9f7804b0e test diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 920d6e7cd..96cc0248e 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -212,13 +212,18 @@ protected Object invokeStep(StepDescriptor d, Object args) { return invokeStep(d, d.getFunctionName(), args); } + protected Object invokeStep(StepDescriptor d, String name, Object args) { + return invokeStep(d, name, args, null); + } + /** * When {@link #invokeMethod(String, Object)} is calling a {@link StepDescriptor} * @param d The {@link StepDescriptor} being invoked. * @param name The name used to invoke the step. May be {@link StepDescriptor#getFunctionName}, a symbol as in {@link StepDescriptor#metaStepsOf}, or {@code d.clazz.getName()}. * @param args The arguments passed to the step. + * @param describableArgs The raw arguments to the describable (when called from {@link #invokeDescribable(String, Object)} */ - protected Object invokeStep(StepDescriptor d, String name, Object args) { + protected Object invokeStep(StepDescriptor d, String name, Object args, @Nullable Object describableArgs) { // TODO: generalize the notion of Step taking over the FlowNode creation. boolean hack = d instanceof ParallelStep.DescriptorImpl || d instanceof LoadStep.DescriptorImpl; @@ -241,6 +246,10 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { CpsStepContext context = new CpsStepContext(d, thread, handle, an); InterpolatedSecretsDetector secretsDetector = InterpolatedSecretsDetector.of(context, exec); + if (describableArgs != null) { + // parse the raw describable arguments here to check for interpolated secrets + parseArgs(describableArgs, d, secretsDetector); + } NamedArgsAndClosure ps = parseArgs(args, d, secretsDetector); context.setBody(ps.body, thread); // Ensure ArgumentsAction is attached before we notify even synchronous listeners: @@ -376,6 +385,7 @@ protected Object invokeDescribable(String symbol, Object _args) { StepDescriptor metaStep = metaSteps.size()==1 ? metaSteps.get(0) : null; boolean singleArgumentOnly = false; + InterpolatedSecretsDetector secretsDetector = null; if (metaStep != null) { Descriptor symbolDescriptor = SymbolLookup.get().findDescriptor((Class)(metaStep.getMetaStepArgumentType()), symbol); DescribableModel symbolModel = DescribableModel.of(symbolDescriptor.clazz); @@ -385,7 +395,7 @@ protected Object invokeDescribable(String symbol, Object _args) { // The only time a closure is valid is when the resulting Describable is immediately executed via a meta-step NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), - UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, null); + UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, secretsDetector); UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); if (metaStep==null) { @@ -445,7 +455,7 @@ protected Object invokeDescribable(String symbol, Object _args) { ud = new UninstantiatedDescribable(symbol, null, dargs); margs.put(p.getName(),ud); - return invokeStep(metaStep, symbol, new NamedArgsAndClosure(margs, args.body, null)); + return invokeStep(metaStep, symbol, new NamedArgsAndClosure(margs, args.body, null), _args); } catch (Exception e) { throw new IllegalArgumentException("Failed to prepare "+symbol+" step",e); } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 6ed650024..091b463f9 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -458,6 +458,32 @@ public void namedSoleParamForStep() throws Exception { MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); } + @Test public void describableInterpolation() throws Exception { + final String credentialsId = "creds"; + final String username = "bob"; + final String password = "secr3t"; + UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); + CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); + p.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" + + "archiveArtifacts(\"${PASSWORD}\")" + + "}\n" + + "}", true)); + WorkflowRun run = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); + r.assertLogContains("Affected variables: [PASSWORD]", run); + InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); + Assert.assertNotNull(reportAction); + Set reportResults = reportAction.getResults(); + MatcherAssert.assertThat(reportResults.size(), is(1)); + MatcherAssert.assertThat(reportResults.iterator().next(), is("PASSWORD")); + // TODO: Code below currently fails +// LinearScanner scan = new LinearScanner(); +// FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("archiveArtifacts")); +// ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); +// Assert.assertFalse(argAction.isUnmodifiedArguments()); +// MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); + } @Test public void noBodyError() throws Exception { p.setDefinition((new CpsFlowDefinition("timeout(time: 1, unit: 'SECONDS')", true))); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); From cc5b3b0678b9607e6cfd197b56b9ed0c4fff8c18 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 17 Sep 2020 23:49:24 -0600 Subject: [PATCH 32/76] Track groovy strings instead of using InterpolatedSecretsDetector. --- pom.xml | 2 +- .../plugins/workflow/cps/CpsStepContext.java | 11 +- .../jenkinsci/plugins/workflow/cps/DSL.java | 128 ++++++++++-------- .../cps/InterpolatedSecretsDetector.java | 65 --------- .../plugins/workflow/cps/DSLTest.java | 16 ++- 5 files changed, 89 insertions(+), 133 deletions(-) delete mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java diff --git a/pom.xml b/pom.xml index 261fff33c..65c91ce6e 100644 --- a/pom.xml +++ b/pom.xml @@ -160,7 +160,7 @@ org.jenkins-ci.plugins credentials-binding - 1.24-rc374.2fd9f7804b0e + 1.24-rc375.06428a051632 test diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java index 9f24017d1..51a64e770 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/CpsStepContext.java @@ -181,22 +181,15 @@ public class CpsStepContext extends DefaultStepContext { // TODO add XStream cla private transient volatile boolean loadingThreadGroup; @CpsVmThreadOnly - CpsStepContext(StepDescriptor step, CpsThread thread, FlowExecutionOwner executionRef, FlowNode node) { + CpsStepContext(StepDescriptor step, CpsThread thread, FlowExecutionOwner executionRef, FlowNode node, @CheckForNull Closure body) { this.threadId = thread.id; this.executionRef = executionRef; this.id = node.getId(); this.node = node; + this.body = body != null ? thread.group.export(body) : null; this.stepDescriptorId = step.getId(); } - public void setBody(Closure bodyToSet, @Nonnull CpsThread thread) { - if (this.body != null) { - throw new IllegalStateException("Context already has a body"); - } else if (bodyToSet != null) { - this.body = thread.group.export(bodyToSet); - } - } - /** * Obtains {@link StepDescriptor} that represents the step this context is invoking. * diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 96cc0248e..25e31d7ae 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -76,10 +76,12 @@ import static org.jenkinsci.plugins.workflow.cps.persistence.PersistenceContext.*; import org.jenkinsci.plugins.workflow.cps.steps.LoadStep; import org.jenkinsci.plugins.workflow.cps.steps.ParallelStep; +import org.jenkinsci.plugins.workflow.cps.view.InterpolatedSecretsAction; import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; import org.jenkinsci.plugins.workflow.flow.StepListener; import org.jenkinsci.plugins.workflow.graph.FlowNode; import org.jenkinsci.plugins.workflow.steps.BodyExecutionCallback; +import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander; import org.jenkinsci.plugins.workflow.steps.Step; import org.jenkinsci.plugins.workflow.steps.StepContext; import org.jenkinsci.plugins.workflow.steps.StepDescriptor; @@ -88,6 +90,7 @@ import org.kohsuke.stapler.ClassDescriptor; import org.kohsuke.stapler.NoStaplerConstructorException; +import javax.annotation.CheckForNull; import javax.annotation.Nullable; /** @@ -212,28 +215,29 @@ protected Object invokeStep(StepDescriptor d, Object args) { return invokeStep(d, d.getFunctionName(), args); } - protected Object invokeStep(StepDescriptor d, String name, Object args) { - return invokeStep(d, name, args, null); - } - /** * When {@link #invokeMethod(String, Object)} is calling a {@link StepDescriptor} * @param d The {@link StepDescriptor} being invoked. * @param name The name used to invoke the step. May be {@link StepDescriptor#getFunctionName}, a symbol as in {@link StepDescriptor#metaStepsOf}, or {@code d.clazz.getName()}. * @param args The arguments passed to the step. - * @param describableArgs The raw arguments to the describable (when called from {@link #invokeDescribable(String, Object)} */ - protected Object invokeStep(StepDescriptor d, String name, Object args, @Nullable Object describableArgs) { + protected Object invokeStep(StepDescriptor d, String name, Object args) { + Set interpolatedStrings; + if (args instanceof NamedArgsAndClosure) { + interpolatedStrings = ((NamedArgsAndClosure) args).getInterpolatedStrings(); + } else { + interpolatedStrings = new HashSet<>(); + } + final NamedArgsAndClosure ps = parseArgs(args, d, interpolatedStrings); - // TODO: generalize the notion of Step taking over the FlowNode creation. - boolean hack = d instanceof ParallelStep.DescriptorImpl || d instanceof LoadStep.DescriptorImpl; + CpsThread thread = CpsThread.current(); FlowNode an; - CpsThread thread = CpsThread.current(); - boolean hasBody = hack? false : argsHasBody(args); - if (!hack && !hasBody) { - // Legacy Stage Step support means the step has no body but still takesImplicitBlockArgument + // TODO: generalize the notion of Step taking over the FlowNode creation. + boolean hack = d instanceof ParallelStep.DescriptorImpl || d instanceof LoadStep.DescriptorImpl; + + if (ps.body == null && !hack) { if (!(d.getClass().getName().equals("org.jenkinsci.plugins.workflow.support.steps.StageStep$DescriptorImpl")) && d.takesImplicitBlockArgument()) { throw new IllegalStateException(String.format("%s step must be called with a body", name)); @@ -244,14 +248,13 @@ protected Object invokeStep(StepDescriptor d, String name, Object args, @Nullabl an = new StepStartNode(exec, d, thread.head.get()); } - CpsStepContext context = new CpsStepContext(d, thread, handle, an); - InterpolatedSecretsDetector secretsDetector = InterpolatedSecretsDetector.of(context, exec); - if (describableArgs != null) { - // parse the raw describable arguments here to check for interpolated secrets - parseArgs(describableArgs, d, secretsDetector); + final CpsStepContext context = new CpsStepContext(d, thread, handle, an, ps.body); + try { + logInterpolationWarnings(interpolatedStrings, context); + } catch (IOException | InterruptedException e) { + LOGGER.log(Level.WARNING, "Unable to log interpolated string warnings"); } - NamedArgsAndClosure ps = parseArgs(args, d, secretsDetector); - context.setBody(ps.body, thread); + // Ensure ArgumentsAction is attached before we notify even synchronous listeners: try { // No point storing empty arguments, and ParallelStep is a special case where we can't store its closure arguments @@ -288,9 +291,6 @@ protected Object invokeStep(StepDescriptor d, String name, Object args, @Nullabl DescribableModel stepModel = DescribableModel.of(d.clazz); s = stepModel.instantiate(ps.namedArgs, listener); } - if (secretsDetector != null) { - secretsDetector.logResults(listener); - } // Persist the node - block start and end nodes do their own persistence. CpsFlowExecution.maybeAutoPersistNode(an); @@ -350,21 +350,30 @@ protected Object invokeStep(StepDescriptor d, String name, Object args, @Nullabl } } - // Check if step arguments contain a step body - private boolean argsHasBody(Object args) { - if (args != null) { - if (args instanceof NamedArgsAndClosure) { - if (((NamedArgsAndClosure) args).body != null) { - return true; - } - } else if (args instanceof Object[]) { - Object[] array = (Object[]) args; - if (array.length > 0 && array[array.length - 1] instanceof Closure) { - return true; + private void logInterpolationWarnings(Set interpolatedStrings, CpsStepContext context) throws IOException, InterruptedException { + if (!interpolatedStrings.isEmpty()) { + EnvVars contextEnvVars = context.get(EnvVars.class); + EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); + + List scanResults = contextExpander.getSensitiveVariables().stream() + .filter(e -> interpolatedStrings.stream().anyMatch(g -> g.contains(contextEnvVars.get(e)))) + .collect(Collectors.toList()); + + if (scanResults != null && !scanResults.isEmpty()) { + context.get(TaskListener.class).getLogger().println("The following Groovy string(s) may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variable(s): " + scanResults.toString()); + FlowExecutionOwner owner = exec.getOwner(); + if (owner != null && owner.getExecutable() instanceof Run) { + InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); + if (runReport == null) { + runReport = new InterpolatedSecretsAction(); + ((Run) owner.getExecutable()).addAction(runReport); + } + runReport.record(scanResults); + } else { + LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); } } } - return false; } private static String loadSoleArgumentKey(StepDescriptor d) { @@ -381,11 +390,11 @@ private static String loadSoleArgumentKey(StepDescriptor d) { */ @SuppressWarnings({"unchecked", "rawtypes"}) protected Object invokeDescribable(String symbol, Object _args) { + Set interpolatedStrings = new HashSet<>(); List metaSteps = StepDescriptor.metaStepsOf(symbol); StepDescriptor metaStep = metaSteps.size()==1 ? metaSteps.get(0) : null; boolean singleArgumentOnly = false; - InterpolatedSecretsDetector secretsDetector = null; if (metaStep != null) { Descriptor symbolDescriptor = SymbolLookup.get().findDescriptor((Class)(metaStep.getMetaStepArgumentType()), symbol); DescribableModel symbolModel = DescribableModel.of(symbolDescriptor.clazz); @@ -395,7 +404,7 @@ protected Object invokeDescribable(String symbol, Object _args) { // The only time a closure is valid is when the resulting Describable is immediately executed via a meta-step NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), - UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, secretsDetector); + UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, interpolatedStrings); UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); if (metaStep==null) { @@ -455,7 +464,7 @@ protected Object invokeDescribable(String symbol, Object _args) { ud = new UninstantiatedDescribable(symbol, null, dargs); margs.put(p.getName(),ud); - return invokeStep(metaStep, symbol, new NamedArgsAndClosure(margs, args.body, null), _args); + return invokeStep(metaStep, symbol, new NamedArgsAndClosure(margs, args.body, interpolatedStrings)); } catch (Exception e) { throw new IllegalArgumentException("Failed to prepare "+symbol+" step",e); } @@ -490,22 +499,29 @@ private static int preallocatedHashmapCapacity(int elementsToHold) { } } + // TODO: add java doc static class NamedArgsAndClosure { final Map namedArgs; final Closure body; final List msgs; + final Set interpolatedStrings; - private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable InterpolatedSecretsDetector secretsDetector) { + private NamedArgsAndClosure(Map namedArgs, Closure body, Set interpolatedStrings) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; this.msgs = new ArrayList<>(); + this.interpolatedStrings = interpolatedStrings; for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more - Object v = flattenGString(entry.getValue(), secretsDetector); + Object v = flattenGString(entry.getValue(), interpolatedStrings); this.namedArgs.put(k, v); } } + + private Set getInterpolatedStrings() { + return Collections.unmodifiableSet(interpolatedStrings); + } } /** @@ -518,18 +534,18 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nullable Interpol * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v, @Nullable InterpolatedSecretsDetector secretsDetector) { + private static Object flattenGString(Object v, @CheckForNull Set interpolatedStrings) { if (v instanceof GString) { String flattened = v.toString(); - if (secretsDetector != null) { - secretsDetector.scan(flattened); + if (interpolatedStrings != null) { + interpolatedStrings.add(flattened); } return flattened; } else if (v instanceof List) { boolean mutated = false; List r = new ArrayList<>(); for (Object o : ((List) v)) { - Object o2 = flattenGString(o, secretsDetector); + Object o2 = flattenGString(o, interpolatedStrings); mutated |= o != o2; r.add(o2); } @@ -539,9 +555,9 @@ private static Object flattenGString(Object v, @Nullable InterpolatedSecretsDete Map r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) v).size())); for (Map.Entry e : ((Map) v).entrySet()) { Object k = e.getKey(); - Object k2 = flattenGString(k, secretsDetector); + Object k2 = flattenGString(k, interpolatedStrings); Object o = e.getValue(); - Object o2 = flattenGString(o, secretsDetector); + Object o2 = flattenGString(o, interpolatedStrings); mutated |= k != k2 || o != o2; r.put(k2, o2); } @@ -551,7 +567,7 @@ private static Object flattenGString(Object v, @Nullable InterpolatedSecretsDete } } - static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, InterpolatedSecretsDetector secretsDetector) { + static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, Set interpolatedStrings) { boolean singleArgumentOnly = false; try { DescribableModel stepModel = DescribableModel.of(d.clazz); @@ -559,12 +575,12 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, InterpolatedS if (singleArgumentOnly) { // Can fetch the one argument we need DescribableParameter dp = stepModel.getSoleRequiredParameter(); String paramName = (dp != null) ? dp.getName() : null; - return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, secretsDetector); + return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, interpolatedStrings); } } catch (NoStaplerConstructorException e) { // Ignore steps without databound constructors and treat them as normal. } - return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, secretsDetector); + return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, interpolatedStrings); } /** @@ -592,18 +608,18 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, InterpolatedS // * @param envVars * The environment variables of the context */ - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nullable InterpolatedSecretsDetector secretsDetector) { + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, Set interpolatedStrings) { if (arg instanceof NamedArgsAndClosure) return (NamedArgsAndClosure) arg; if (arg instanceof Map) // TODO is this clause actually used? - return new NamedArgsAndClosure((Map) arg, null, secretsDetector); + return new NamedArgsAndClosure((Map) arg, null, interpolatedStrings); if (arg instanceof Closure && expectsBlock) - return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, secretsDetector); + return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, interpolatedStrings); if (arg instanceof Object[]) {// this is how Groovy appears to pack argument list into one Object for invokeMethod List a = Arrays.asList((Object[])arg); if (a.size()==0) - return new NamedArgsAndClosure(Collections.emptyMap(),null, secretsDetector); + return new NamedArgsAndClosure(Collections.emptyMap(),null, interpolatedStrings); Closure c=null; @@ -618,21 +634,21 @@ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String so if (!singleRequiredArg || (soleArgumentKey != null && mapArg.size() == 1 && mapArg.containsKey(soleArgumentKey))) { // this is how Groovy passes in Map - return new NamedArgsAndClosure(mapArg, c, secretsDetector); + return new NamedArgsAndClosure(mapArg, c, interpolatedStrings); } } switch (a.size()) { case 0: - return new NamedArgsAndClosure(Collections.emptyMap(),c, secretsDetector); + return new NamedArgsAndClosure(Collections.emptyMap(),c, interpolatedStrings); case 1: - return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, secretsDetector); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, a.get(0)), c, interpolatedStrings); default: throw new IllegalArgumentException("Expected named arguments but got "+a); } } - return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, secretsDetector); + return new NamedArgsAndClosure(singleParam(soleArgumentKey, arg), null, interpolatedStrings); } private static Map singleParam(String soleArgumentKey, Object arg) { if (soleArgumentKey != null) { diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java deleted file mode 100644 index f89962a1b..000000000 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedSecretsDetector.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.jenkinsci.plugins.workflow.cps; - -import hudson.EnvVars; -import hudson.model.Run; -import hudson.model.TaskListener; -import org.jenkinsci.plugins.workflow.cps.view.InterpolatedSecretsAction; -import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; -import org.jenkinsci.plugins.workflow.steps.EnvironmentExpander; - -import javax.annotation.Nonnull; -import java.io.IOException; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -public class InterpolatedSecretsDetector { - private final EnvVars envVars; - private final Set watchedVars; - private List scanResults; - private final CpsFlowExecution exec; - - private static final Logger LOGGER = Logger.getLogger(InterpolatedSecretsDetector.class.getName()); - - public static InterpolatedSecretsDetector of(CpsStepContext context, CpsFlowExecution exec) { - try { - EnvVars contextEnvVars = context.get(EnvVars.class); - EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); - if (contextEnvVars != null && contextExpander != null) { - return new InterpolatedSecretsDetector(contextEnvVars, contextExpander, exec); - } - } catch (InterruptedException | IOException e) { - LOGGER.log(Level.FINE, "Unable to create EnvironmentWatcher instance."); - } - return null; - } - - private InterpolatedSecretsDetector(@Nonnull EnvVars envVars, @Nonnull EnvironmentExpander expander, @Nonnull CpsFlowExecution exec) { - this.envVars = envVars; - watchedVars = expander.getSensitiveVariables(); - this.exec = exec; - } - - public void scan(String text) { - scanResults = watchedVars.stream().filter(e -> text.contains(envVars.get(e))).collect(Collectors.toList()); - } - - public void logResults(TaskListener listener) throws IOException { - if (scanResults != null && !scanResults.isEmpty()) { - listener.getLogger().println("The following Groovy string may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variables: " + scanResults.toString()); - FlowExecutionOwner owner = exec.getOwner(); - if (owner != null && owner.getExecutable() instanceof Run) { - InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); - if (runReport == null) { - runReport = new InterpolatedSecretsAction(); - ((Run) owner.getExecutable()).addAction(runReport); - } - runReport.record(scanResults); - } else { - LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); - } - } - } -} diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 091b463f9..c61f17534 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -445,7 +445,7 @@ public void namedSoleParamForStep() throws Exception { + "}\n" + "}", true)); WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - r.assertLogContains("Affected variables: [PASSWORD]", run); + r.assertLogContains("Affected variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); Set reportResults = reportAction.getResults(); @@ -471,7 +471,7 @@ public void namedSoleParamForStep() throws Exception { + "}\n" + "}", true)); WorkflowRun run = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains("Affected variables: [PASSWORD]", run); + r.assertLogContains("Affected variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); Set reportResults = reportAction.getResults(); @@ -484,6 +484,18 @@ public void namedSoleParamForStep() throws Exception { // Assert.assertFalse(argAction.isUnmodifiedArguments()); // MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); } + + @Test public void noMetaStep() throws Exception { + p.setDefinition(new CpsFlowDefinition("monomorphStep([firstArg:'one', secondArg:'two'])", true)); + r.assertLogContains("First arg: one, second arg: two", r.assertBuildStatusSuccess(p.scheduleBuild2(0))); + WorkflowRun run = p.getLastBuild(); + LinearScanner scanner = new LinearScanner(); + FlowNode node = scanner.findFirstMatch(run.getExecution().getCurrentHeads(), new NodeStepTypePredicate("monomorphStep")); + ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); + Assert.assertNotNull(argumentsAction); + Assert.assertEquals("one,two", ArgumentsAction.getStepArgumentsAsString(node)); + } + @Test public void noBodyError() throws Exception { p.setDefinition((new CpsFlowDefinition("timeout(time: 1, unit: 'SECONDS')", true))); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); From 8517f122b8923f7708d80fc56549faf678387431 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 21 Sep 2020 08:17:26 -0600 Subject: [PATCH 33/76] added InterpolatedUninstantiatedDescribable --- .../jenkinsci/plugins/workflow/cps/DSL.java | 44 +++++++++++-------- ...InterpolatedUninstantiatedDescribable.java | 29 ++++++++++++ .../plugins/workflow/cps/DSLTest.java | 29 ++++++++---- 3 files changed, 75 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedUninstantiatedDescribable.java diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 25e31d7ae..a103fbd09 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -222,12 +222,19 @@ protected Object invokeStep(StepDescriptor d, Object args) { * @param args The arguments passed to the step. */ protected Object invokeStep(StepDescriptor d, String name, Object args) { - Set interpolatedStrings; + Set interpolatedStrings = null; if (args instanceof NamedArgsAndClosure) { - interpolatedStrings = ((NamedArgsAndClosure) args).getInterpolatedStrings(); - } else { + interpolatedStrings = ((NamedArgsAndClosure) args).interpolatedStrings; + } else if (args instanceof Object[]) { + Object[] array = (Object[]) args; + if (array.length > 0 && array[array.length - 1] instanceof InterpolatedUninstantiatedDescribable) { + interpolatedStrings = ((InterpolatedUninstantiatedDescribable) array[array.length - 1]).getInterpolatedStrings(); + } + } + if (interpolatedStrings == null) { interpolatedStrings = new HashSet<>(); } + final NamedArgsAndClosure ps = parseArgs(args, d, interpolatedStrings); CpsThread thread = CpsThread.current(); @@ -248,14 +255,8 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { an = new StepStartNode(exec, d, thread.head.get()); } - final CpsStepContext context = new CpsStepContext(d, thread, handle, an, ps.body); - try { - logInterpolationWarnings(interpolatedStrings, context); - } catch (IOException | InterruptedException e) { - LOGGER.log(Level.WARNING, "Unable to log interpolated string warnings"); - } - // Ensure ArgumentsAction is attached before we notify even synchronous listeners: + final CpsStepContext context = new CpsStepContext(d, thread, handle, an, ps.body); try { // No point storing empty arguments, and ParallelStep is a special case where we can't store its closure arguments if (ps.namedArgs != null && !(ps.namedArgs.isEmpty()) && isKeepStepArguments() && !(d instanceof ParallelStep.DescriptorImpl)) { @@ -280,6 +281,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { ClassLoader originalLoader = Thread.currentThread().getContextClassLoader(); try { TaskListener listener = context.get(TaskListener.class); + logInterpolationWarnings(interpolatedStrings, context, listener); if (unreportedAmbiguousFunctions.remove(name)) { reportAmbiguousStepInvocation(context, d, listener); } @@ -350,7 +352,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } } - private void logInterpolationWarnings(Set interpolatedStrings, CpsStepContext context) throws IOException, InterruptedException { + private void logInterpolationWarnings(Set interpolatedStrings, CpsStepContext context, TaskListener listener) throws IOException, InterruptedException { if (!interpolatedStrings.isEmpty()) { EnvVars contextEnvVars = context.get(EnvVars.class); EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); @@ -360,7 +362,7 @@ private void logInterpolationWarnings(Set interpolatedStrings, CpsStepCo .collect(Collectors.toList()); if (scanResults != null && !scanResults.isEmpty()) { - context.get(TaskListener.class).getLogger().println("The following Groovy string(s) may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variable(s): " + scanResults.toString()); + listener.getLogger().println("The following Groovy string(s) may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variable(s): " + scanResults.toString()); FlowExecutionOwner owner = exec.getOwner(); if (owner != null && owner.getExecutable() instanceof Run) { InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); @@ -405,7 +407,6 @@ protected Object invokeDescribable(String symbol, Object _args) { // The only time a closure is valid is when the resulting Describable is immediately executed via a meta-step NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, interpolatedStrings); - UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); if (metaStep==null) { // there's no meta-step associated with it, so this symbol is not executable. @@ -417,8 +418,9 @@ protected Object invokeDescribable(String symbol, Object _args) { // also note that in this case 'd' is not trustworthy, as depending on // where this UninstantiatedDescribable is ultimately used, the symbol // might be resolved with a specific type. - return ud; + return new InterpolatedUninstantiatedDescribable(symbol, null, args.namedArgs, interpolatedStrings); } else { + UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); Descriptor d = SymbolLookup.get().findDescriptor((Class)(metaStep.getMetaStepArgumentType()), symbol); try { // execute this Describable through a meta-step @@ -518,12 +520,18 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, Set interp this.namedArgs.put(k, v); } } - - private Set getInterpolatedStrings() { - return Collections.unmodifiableSet(interpolatedStrings); - } } +// static class InterpolatedUninstantiatedDescribable { +// final UninstantiatedDescribable ud; +// final Set interpolatedStrings; +// +// private InterpolatedUninstantiatedDescribable(UninstantiatedDescribable ud, Set interpolatedStrings) { +// this.ud = ud; +// this.interpolatedStrings = interpolatedStrings; +// } +// } + /** * Coerce {@link GString}, to save {@link StepDescriptor#newInstance(Map)} from being made aware of that. * This is not the only type coercion that Groovy does, so this is not very kosher, but diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedUninstantiatedDescribable.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedUninstantiatedDescribable.java new file mode 100644 index 000000000..980f463e6 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedUninstantiatedDescribable.java @@ -0,0 +1,29 @@ +package org.jenkinsci.plugins.workflow.cps; + +import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable; + +import java.util.Map; +import java.util.Set; + +public class InterpolatedUninstantiatedDescribable extends UninstantiatedDescribable { + private final Set interpolatedStrings; + + public InterpolatedUninstantiatedDescribable(String symbol, String klass, Map arguments, Set interpolatedStrings) { + super(symbol, klass, arguments); + this.interpolatedStrings = interpolatedStrings; + } + + public Set getInterpolatedStrings() { + return interpolatedStrings; + } + + @Override + public boolean equals(Object o) { + return super.equals(o); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index c61f17534..5dfc01ff4 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -485,15 +485,26 @@ public void namedSoleParamForStep() throws Exception { // MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); } - @Test public void noMetaStep() throws Exception { - p.setDefinition(new CpsFlowDefinition("monomorphStep([firstArg:'one', secondArg:'two'])", true)); - r.assertLogContains("First arg: one, second arg: two", r.assertBuildStatusSuccess(p.scheduleBuild2(0))); - WorkflowRun run = p.getLastBuild(); - LinearScanner scanner = new LinearScanner(); - FlowNode node = scanner.findFirstMatch(run.getExecution().getCurrentHeads(), new NodeStepTypePredicate("monomorphStep")); - ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); - Assert.assertNotNull(argumentsAction); - Assert.assertEquals("one,two", ArgumentsAction.getStepArgumentsAsString(node)); + @Test public void describableNoMetaStep() throws Exception { + final String credentialsId = "creds"; + final String username = "bob"; + final String password = "secr3t"; + UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); + CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); + p.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" + + "monomorphWithSymbolStep(monomorphSymbol([firstArg:\"${PASSWORD}\", secondArg:'two']))" + + "}\n" + + "}", true)); + r.assertLogContains("First arg: ****, second arg: two", r.assertBuildStatusSuccess(p.scheduleBuild2(0))); + // TODO: Code below currently fails +// WorkflowRun run = p.getLastBuild(); +// LinearScanner scanner = new LinearScanner(); +// FlowNode node = scanner.findFirstMatch(run.getExecution().getCurrentHeads(), new NodeStepTypePredicate("monomorphStep")); +// ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); +// Assert.assertNotNull(argumentsAction); +// Assert.assertEquals("one,two", ArgumentsAction.getStepArgumentsAsString(node)); } @Test public void noBodyError() throws Exception { From f12f226a509bd36f6b661b9cf5f49003afda470e Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 21 Sep 2020 14:56:37 -0600 Subject: [PATCH 34/76] add null checks for environmentexpander and envars --- .../jenkinsci/plugins/workflow/cps/DSL.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index a103fbd09..de87c01ba 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -353,27 +353,31 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } private void logInterpolationWarnings(Set interpolatedStrings, CpsStepContext context, TaskListener listener) throws IOException, InterruptedException { - if (!interpolatedStrings.isEmpty()) { - EnvVars contextEnvVars = context.get(EnvVars.class); - EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); - - List scanResults = contextExpander.getSensitiveVariables().stream() - .filter(e -> interpolatedStrings.stream().anyMatch(g -> g.contains(contextEnvVars.get(e)))) - .collect(Collectors.toList()); + if (interpolatedStrings.isEmpty()) { + return; + } + EnvVars contextEnvVars = context.get(EnvVars.class); + EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); + if (contextEnvVars == null || contextExpander == null) { + return; + } - if (scanResults != null && !scanResults.isEmpty()) { - listener.getLogger().println("The following Groovy string(s) may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variable(s): " + scanResults.toString()); - FlowExecutionOwner owner = exec.getOwner(); - if (owner != null && owner.getExecutable() instanceof Run) { - InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); - if (runReport == null) { - runReport = new InterpolatedSecretsAction(); - ((Run) owner.getExecutable()).addAction(runReport); - } - runReport.record(scanResults); - } else { - LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); + List scanResults = contextExpander.getSensitiveVariables().stream() + .filter(e -> interpolatedStrings.stream().anyMatch(g -> g.contains(contextEnvVars.get(e)))) + .collect(Collectors.toList()); + + if (scanResults != null && !scanResults.isEmpty()) { + listener.getLogger().println("The following Groovy string(s) may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variable(s): " + scanResults.toString()); + FlowExecutionOwner owner = exec.getOwner(); + if (owner != null && owner.getExecutable() instanceof Run) { + InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); + if (runReport == null) { + runReport = new InterpolatedSecretsAction(); + ((Run) owner.getExecutable()).addAction(runReport); } + runReport.record(scanResults); + } else { + LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); } } } From 6e8b5eb99200853905b7eedb6c91ec7678600d42 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 23 Sep 2020 01:36:56 -0600 Subject: [PATCH 35/76] Refactor ArgumentsActionImpl using EnvironmentExpander and removing safe list --- .../jenkinsci/plugins/workflow/cps/DSL.java | 31 ++-- .../cps/actions/ArgumentsActionImpl.java | 148 +++--------------- .../cps/actions/ArgumentsActionImplTest.java | 57 ++++--- 3 files changed, 72 insertions(+), 164 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index de87c01ba..733404e52 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -255,18 +255,28 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { an = new StepStartNode(exec, d, thread.head.get()); } - // Ensure ArgumentsAction is attached before we notify even synchronous listeners: final CpsStepContext context = new CpsStepContext(d, thread, handle, an, ps.body); + EnvVars allEnv = null; + Set sensitiveVariables = null; + try { + allEnv = context.get(EnvVars.class); + EnvironmentExpander envExpander = context.get(EnvironmentExpander.class); + if (envExpander != null) { + sensitiveVariables = envExpander.getSensitiveVariables(); + } + } catch (IOException | InterruptedException e) { + LOGGER.log(Level.WARNING, "Unable to retrieve environment variables", e); + } + // Ensure ArgumentsAction is attached before we notify even synchronous listeners: try { // No point storing empty arguments, and ParallelStep is a special case where we can't store its closure arguments if (ps.namedArgs != null && !(ps.namedArgs.isEmpty()) && isKeepStepArguments() && !(d instanceof ParallelStep.DescriptorImpl)) { // Get the environment variables to find ones that might be credentials bindings Computer comp = context.get(Computer.class); - EnvVars allEnv = new EnvVars(context.get(EnvVars.class)); if (comp != null && allEnv != null) { allEnv.entrySet().removeAll(comp.getEnvironment().entrySet()); } - an.addAction(new ArgumentsActionImpl(ps.namedArgs, allEnv)); + an.addAction(new ArgumentsActionImpl(ps.namedArgs, allEnv, sensitiveVariables)); } } catch (Exception e) { // Avoid breaking execution because we can't store some sort of crazy Step argument @@ -281,7 +291,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { ClassLoader originalLoader = Thread.currentThread().getContextClassLoader(); try { TaskListener listener = context.get(TaskListener.class); - logInterpolationWarnings(interpolatedStrings, context, listener); + logInterpolationWarnings(interpolatedStrings, allEnv, sensitiveVariables, listener); if (unreportedAmbiguousFunctions.remove(name)) { reportAmbiguousStepInvocation(context, d, listener); } @@ -352,18 +362,13 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } } - private void logInterpolationWarnings(Set interpolatedStrings, CpsStepContext context, TaskListener listener) throws IOException, InterruptedException { - if (interpolatedStrings.isEmpty()) { - return; - } - EnvVars contextEnvVars = context.get(EnvVars.class); - EnvironmentExpander contextExpander = context.get(EnvironmentExpander.class); - if (contextEnvVars == null || contextExpander == null) { + private void logInterpolationWarnings(Set interpolatedStrings, @CheckForNull EnvVars envVars, Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { + if (interpolatedStrings.isEmpty() || envVars == null || envVars.isEmpty() || sensitiveVariables == null || sensitiveVariables.isEmpty()) { return; } - List scanResults = contextExpander.getSensitiveVariables().stream() - .filter(e -> interpolatedStrings.stream().anyMatch(g -> g.contains(contextEnvVars.get(e)))) + List scanResults = sensitiveVariables.stream() + .filter(e -> interpolatedStrings.stream().anyMatch(g -> g.contains(envVars.get(e)))) .collect(Collectors.toList()); if (scanResults != null && !scanResults.isEmpty()) { diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java index 20ba11216..595ef927e 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java @@ -54,6 +54,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** @@ -71,13 +72,13 @@ public class ArgumentsActionImpl extends ArgumentsAction { private static final Logger LOGGER = Logger.getLogger(ArgumentsActionImpl.class.getName()); - public ArgumentsActionImpl(@Nonnull Map stepArguments, @CheckForNull EnvVars env) { - arguments = serializationCheck(sanitizeMapAndRecordMutation(stepArguments, env)); + public ArgumentsActionImpl(@Nonnull Map stepArguments, @CheckForNull EnvVars env, @CheckForNull Set sensitiveVariables) { + arguments = serializationCheck(sanitizeMapAndRecordMutation(stepArguments, env, sensitiveVariables)); } /** Create a step, sanitizing strings for secured content */ public ArgumentsActionImpl(@Nonnull Map stepArguments) { - this(stepArguments, new EnvVars()); + this(stepArguments, new EnvVars(), null); } /** For testing use only */ @@ -87,26 +88,11 @@ public ArgumentsActionImpl(@Nonnull Map stepArguments) { } /** See if sensitive environment variable content is in a string */ - public static boolean isStringSafe(@CheckForNull String input, @CheckForNull EnvVars variables, @Nonnull Set safeEnvVariables) { - if (input == null || variables == null || variables.size() == 0) { - return true; + public static boolean isStringSensitive(@CheckForNull String input, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { + if (input == null || variables == null || variables.size() == 0 || sensitiveVariables == null || sensitiveVariables.size() ==0) { + return false; } - StringBuilder pattern = new StringBuilder(); - int count = 0; - for (Map.Entry ent : variables.entrySet()) { - String val = ent.getValue(); - if (val == null || val.isEmpty() || safeEnvVariables.contains(ent.getKey())) { // Skip values that are safe - continue; - } - if (count > 0) { - pattern.append('|'); - } - pattern.append(Pattern.quote(val)); - count++; - } - return (count > 0) - ? !Pattern.compile(pattern.toString()).matcher(input).find() - : true; + return sensitiveVariables.stream().map(variables::get).anyMatch(input::contains); } /** Restrict stored arguments to a reasonable subset of types so we don't retain totally arbitrary objects @@ -138,103 +124,11 @@ boolean isStorableType(Object ob) { return c.isPrimitive() || (c.isArray() && !(c.getComponentType().isPrimitive())); // Primitive arrays are not legal here } - /** Normal environment variables, as opposed to ones that might come from credentials bindings */ - private static final HashSet SAFE_ENVIRONMENT_VARIABLES = new HashSet<>(Arrays.asList( - // Pipeline/Jenkins variables in normal builds - "BRANCH_NAME", - "BUILD_DISPLAY_NAME", - "BUILD_ID", - "BUILD_NUMBER", - "BUILD_TAG", - "BUILD_URL", - "CHANGE_AUTHOR", - "CHANGE_AUTHOR_DISPLAY_NAME", - "CHANGE_AUTHOR_EMAIL", - "CHANGE_ID", - "CHANGE_TARGET", - "CHANGE_TITLE", - "CHANGE_URL", - "EXECUTOR_NUMBER", - "HUDSON_COOKIE", - "HUDSON_HOME", - "HUDSON_SERVER_COOKIE", - "HUDSON_URL", - "JENKINS_HOME", - "JENKINS_SERVER_COOKIE", - "JENKINS_URL", - "JOB_BASE_NAME", - "JOB_NAME", - "JOB_URL", - "NODE_LABELS", - "NODE_NAME", - "STAGE_NAME", - "WORKSPACE", - - // Normal system variables for posix environments - "HOME", - "LANG", - "LOGNAME", - "MAIL", - "NLSPATH", - "PATH", - "PWD", - "SHELL", - "SHLVL", - "TERM", - "USER", - "XFILESEARCHPATH", - - // Windows system variables - "ALLUSERSPROFILE", - "APPDATA", - "CD", - "ClientName", - "CMDEXTVERSION", - "CMDCMDLINE", - "CommonProgramFiles", - "COMPUTERNAME", - "COMSPEC", - "DATE", - "ERRORLEVEL", - "HighestNumaNodeNumber", - "HOMEDRIVE", - "HOMEPATH", - "LOCALAPPDATA", - "LOGONSERVER", - "NUMBER_OF_PROCESSORS", - "OS", - "PATHEXT", - "PROCESSOR_ARCHITECTURE", - "PROCESSOR_ARCHITEW6432", - "PROCESSOR_IDENTIFIER", - "PROCESSOR_LEVEL", - "PROCESSOR_REVISION", - "ProgramW6432", - "ProgramData", - "ProgramFiles", - "ProgramFiles (x86)", - "PROMPT", - "PSModulePath", - "Public", - "RANDOM", - "%SessionName%", - "SYSTEMDRIVE", - "SYSTEMROOT", - "TEMP", "TMP", - "TIME", - "UserDnsDomain", - "USERDOMAIN", - "USERDOMAIN_roamingprofile", -// "USERNAME", // Not whitelisted because this is a likely variable name for credentials binding - "USERPROFILE", - "WINDIR" - )); - /** * Sanitize a list recursively */ @CheckForNull - Object sanitizeListAndRecordMutation(@Nonnull List objects, @CheckForNull EnvVars variables) { + Object sanitizeListAndRecordMutation(@Nonnull List objects, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { // Package scoped so we can test it directly if (isOversized(objects)) { @@ -245,7 +139,7 @@ Object sanitizeListAndRecordMutation(@Nonnull List objects, @CheckForNull EnvVar boolean isMutated = false; List output = new ArrayList(objects.size()); for (Object o : objects) { - Object modded = sanitizeObjectAndRecordMutation(o, variables); + Object modded = sanitizeObjectAndRecordMutation(o, variables, sensitiveVariables); if (modded != o) { // Sanitization stripped out some values, so we need to store the mutated object @@ -261,13 +155,13 @@ Object sanitizeListAndRecordMutation(@Nonnull List objects, @CheckForNull EnvVar /** For object arrays, we sanitize recursively, as with Lists */ @CheckForNull - Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull EnvVars variables) { + Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { if (isOversized(objects)) { this.isUnmodifiedBySanitization = false; return NotStoredReason.OVERSIZE_VALUE; } List inputList = Arrays.asList(objects); - Object sanitized = sanitizeListAndRecordMutation(inputList, variables); + Object sanitized = sanitizeListAndRecordMutation(inputList, variables, sensitiveVariables); if (sanitized == inputList) { // Works because if not mutated, we return original input instance return objects; } else if (sanitized instanceof List) { @@ -279,14 +173,14 @@ Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull E /** Recursively sanitize a single object by: * - Exploding {@link Step}s and {@link UninstantiatedDescribable}s into their Maps to sanitize - * - Removing unsafe strings using {@link #isStringSafe(String, EnvVars, Set)} and replace with {@link NotStoredReason#MASKED_VALUE} + * - Removing unsafe strings using {@link #isStringSensitive(String, EnvVars, Set)} and replace with {@link NotStoredReason#MASKED_VALUE} * - Removing oversized objects using {@link #isOversized(Object)} and replacing with {@link NotStoredReason#OVERSIZE_VALUE} * While making an effort not to retain needless copies of objects and to re-use originals where possible * (including the Step or UninstantiatedDescribable) */ @CheckForNull @SuppressWarnings("unchecked") - Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull EnvVars vars) { + Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull EnvVars vars, @CheckForNull Set sensitiveVariables) { // Package scoped so we can test it directly Object tempVal = o; DescribableModel m = null; @@ -320,18 +214,18 @@ Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull Env Object modded = tempVal; if (modded instanceof Map) { // Recursive sanitization, oh my! - modded = sanitizeMapAndRecordMutation((Map)modded, vars); + modded = sanitizeMapAndRecordMutation((Map)modded, vars, sensitiveVariables); } else if (modded instanceof List) { - modded = sanitizeListAndRecordMutation((List) modded, vars); + modded = sanitizeListAndRecordMutation((List) modded, vars, sensitiveVariables); } else if (modded != null && modded.getClass().isArray()) { Class componentType = modded.getClass().getComponentType(); if (!componentType.isPrimitive()) { // Object arrays get recursively sanitized - modded = sanitizeArrayAndRecordMutation((Object[])modded, vars); + modded = sanitizeArrayAndRecordMutation((Object[])modded, vars, sensitiveVariables); } else { // Primitive arrays aren't a valid type here this.isUnmodifiedBySanitization = true; return NotStoredReason.UNSERIALIZABLE; } - } else if (modded instanceof String && vars != null && !vars.isEmpty() && !isStringSafe((String)modded, vars, SAFE_ENVIRONMENT_VARIABLES)) { + } else if (modded instanceof String && vars != null && !vars.isEmpty() && isStringSensitive((String)modded, vars, sensitiveVariables)) { this.isUnmodifiedBySanitization = false; return NotStoredReason.MASKED_VALUE; } @@ -386,16 +280,16 @@ Map serializationCheck(@Nonnull Map arguments) { } /** - * Goes through {@link #sanitizeObjectAndRecordMutation(Object, EnvVars)} for each value in a map input. + * Goes through {@link #sanitizeObjectAndRecordMutation(Object, EnvVars, Set)} for each value in a map input. */ @Nonnull - Map sanitizeMapAndRecordMutation(@Nonnull Map mapContents, @CheckForNull EnvVars variables) { + Map sanitizeMapAndRecordMutation(@Nonnull Map mapContents, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { // Package scoped so we can test it directly HashMap output = Maps.newHashMapWithExpectedSize(mapContents.size()); boolean isMutated = false; for (Map.Entry param : mapContents.entrySet()) { - Object modded = sanitizeObjectAndRecordMutation(param.getValue(), variables); + Object modded = sanitizeObjectAndRecordMutation(param.getValue(), variables, sensitiveVariables); if (modded != param.getValue()) { // Sanitization stripped out some values, so we need to store the mutated object diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java index 3b38d4113..b3e67a0c8 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java @@ -156,18 +156,20 @@ public void testStringSafetyTest() throws Exception { String input = "I have a secret p4ssw0rd"; HashMap passwordBinding = new HashMap<>(); passwordBinding.put("mypass", "p4ssw0rd"); - Assert.assertTrue("Input with no variables is safe", ArgumentsActionImpl.isStringSafe(input, new EnvVars(), Collections.EMPTY_SET)); - Assert.assertFalse("Input containing bound value is unsafe", ArgumentsActionImpl.isStringSafe(input, new EnvVars(passwordBinding), Collections.EMPTY_SET)); - - Assert.assertTrue("EnvVars that do not occur are safe", ArgumentsActionImpl.isStringSafe("I have no passwords", new EnvVars(passwordBinding), Collections.EMPTY_SET)); - - HashMap safeBinding = new HashMap<>(); - safeBinding.put("harmless", "secret"); - HashSet safeVars = new HashSet<>(); - safeVars.add("harmless"); - passwordBinding.put("harmless", "secret"); - Assert.assertTrue("Input containing whitelisted bound value is safe", ArgumentsActionImpl.isStringSafe(input, new EnvVars(safeBinding), safeVars)); - Assert.assertFalse("Input containing one safe and one unsafe bound value is unsafe", ArgumentsActionImpl.isStringSafe(input, new EnvVars(passwordBinding), safeVars)); + Set sensitiveVariables = new HashSet<>(); + sensitiveVariables.add("mypass"); + Assert.assertFalse("Input with no variables is safe", ArgumentsActionImpl.isStringSensitive(input, new EnvVars(), sensitiveVariables)); + Assert.assertTrue("Input containing bound value is unsafe", ArgumentsActionImpl.isStringSensitive(input, new EnvVars(passwordBinding), sensitiveVariables)); + + Assert.assertFalse("EnvVars that do not occur are safe", ArgumentsActionImpl.isStringSensitive("I have no passwords", new EnvVars(passwordBinding), sensitiveVariables)); + +// HashMap safeBinding = new HashMap<>(); +// safeBinding.put("harmless", "secret"); +// HashSet safeVars = new HashSet<>(); +// safeVars.add("harmless"); +// passwordBinding.put("harmless", "secret"); +// Assert.assertTrue("Input containing whitelisted bound value is safe", ArgumentsActionImpl.isStringSafe(input, new EnvVars(safeBinding), safeVars)); +// Assert.assertFalse("Input containing one safe and one unsafe bound value is unsafe", ArgumentsActionImpl.isStringSafe(input, new EnvVars(passwordBinding), safeVars)); } @Test @@ -179,27 +181,30 @@ public void testRecursiveSanitizationOfContent() { String secretUsername = "secretuser"; env.put("USERVARIABLE", secretUsername); // assume secretuser is a bound credential + Set sensitiveVariables = new HashSet<>(); + sensitiveVariables.add("USERVARIABLE"); + char[] oversized = new char[maxLen+10]; Arrays.fill(oversized, 'a'); String oversizedString = new String (oversized); // Simplest masking of secret and oversized value - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, impl.sanitizeObjectAndRecordMutation(secretUsername, env)); + Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, impl.sanitizeObjectAndRecordMutation(secretUsername, env, sensitiveVariables)); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; - Assert.assertEquals(ArgumentsAction.NotStoredReason.OVERSIZE_VALUE, impl.sanitizeObjectAndRecordMutation(oversizedString, env)); + Assert.assertEquals(ArgumentsAction.NotStoredReason.OVERSIZE_VALUE, impl.sanitizeObjectAndRecordMutation(oversizedString, env, sensitiveVariables)); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; // Test explosion of Step & UninstantiatedDescribable objects Step mystep = new EchoStep("I have a "+secretUsername); - Map singleSanitization = (Map)(impl.sanitizeObjectAndRecordMutation(mystep, env)); + Map singleSanitization = (Map)(impl.sanitizeObjectAndRecordMutation(mystep, env, sensitiveVariables)); Assert.assertEquals(1, singleSanitization.size()); Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, singleSanitization.get("message")); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; - singleSanitization = ((UninstantiatedDescribable) (impl.sanitizeObjectAndRecordMutation(mystep.getDescriptor().uninstantiate(mystep), env))).getArguments(); + singleSanitization = ((UninstantiatedDescribable) (impl.sanitizeObjectAndRecordMutation(mystep.getDescriptor().uninstantiate(mystep), env, sensitiveVariables))).getArguments(); Assert.assertEquals(1, singleSanitization.size()); Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, singleSanitization.get("message")); Assert.assertFalse(impl.isUnmodifiedArguments()); @@ -208,26 +213,26 @@ public void testRecursiveSanitizationOfContent() { // Maps HashMap dangerous = new HashMap<>(); dangerous.put("name", secretUsername); - Map sanitizedMap = impl.sanitizeMapAndRecordMutation(dangerous, env); + Map sanitizedMap = impl.sanitizeMapAndRecordMutation(dangerous, env, sensitiveVariables); Assert.assertNotEquals(sanitizedMap, dangerous); Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, sanitizedMap.get("name")); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; - Map identicalMap = impl.sanitizeMapAndRecordMutation(dangerous, new EnvVars()); // String is no longer dangerous + Map identicalMap = impl.sanitizeMapAndRecordMutation(dangerous, new EnvVars(), sensitiveVariables); // String is no longer dangerous Assert.assertEquals(identicalMap, dangerous); Assert.assertTrue(impl.isUnmodifiedArguments()); // Lists List unsanitizedList = Arrays.asList("cheese", null, secretUsername); - List sanitized = (List)impl.sanitizeListAndRecordMutation(unsanitizedList, env); + List sanitized = (List)impl.sanitizeListAndRecordMutation(unsanitizedList, env, sensitiveVariables); Assert.assertEquals(3, sanitized.size()); Assert.assertFalse(impl.isUnmodifiedArguments()); Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, sanitized.get(2)); impl.isUnmodifiedBySanitization = true; - Assert.assertEquals(unsanitizedList, impl.sanitizeObjectAndRecordMutation(unsanitizedList, new EnvVars())); - Assert.assertEquals(unsanitizedList, impl.sanitizeListAndRecordMutation(unsanitizedList, new EnvVars())); + Assert.assertEquals(unsanitizedList, impl.sanitizeObjectAndRecordMutation(unsanitizedList, new EnvVars(), sensitiveVariables)); + Assert.assertEquals(unsanitizedList, impl.sanitizeListAndRecordMutation(unsanitizedList, new EnvVars(), sensitiveVariables)); } @Test @@ -235,11 +240,13 @@ public void testArraySanitization() { EnvVars env = new EnvVars(); String secretUsername = "IAmA"; env.put("USERVARIABLE", secretUsername); // assume secretuser is a bound credential + Set sensitiveVariables = new HashSet<>(); + sensitiveVariables.add("USERVARIABLE"); HashMap args = new HashMap<>(); args.put("ints", new int[]{1,2,3}); args.put("strings", new String[]{"heh",secretUsername,"lumberjack"}); - ArgumentsActionImpl filtered = new ArgumentsActionImpl(args, env); + ArgumentsActionImpl filtered = new ArgumentsActionImpl(args, env, sensitiveVariables); Map filteredArgs = filtered.getArguments(); Assert.assertEquals(2, filteredArgs.size()); @@ -280,20 +287,22 @@ public void testBasicCreateAndMask() throws Exception { passwordBinding.put("mypass", "p4ssw0rd"); Map arguments = new HashMap<>(); arguments.put("message", "I have a secret p4ssw0rd"); + Set sensitiveVariables = new HashSet<>(); + sensitiveVariables.add("mypass"); Field maxSizeF = ArgumentsAction.class.getDeclaredField("MAX_RETAINED_LENGTH"); maxSizeF.setAccessible(true); int maxSize = maxSizeF.getInt(null); // Same string, unsanitized - ArgumentsActionImpl argumentsActionImpl = new ArgumentsActionImpl(arguments, new EnvVars()); + ArgumentsActionImpl argumentsActionImpl = new ArgumentsActionImpl(arguments, new EnvVars(), sensitiveVariables); Assert.assertTrue(argumentsActionImpl.isUnmodifiedArguments()); Assert.assertEquals(arguments.get("message"), argumentsActionImpl.getArgumentValueOrReason("message")); Assert.assertEquals(1, argumentsActionImpl.getArguments().size()); Assert.assertEquals("I have a secret p4ssw0rd", argumentsActionImpl.getArguments().get("message")); // Test sanitizing arguments now - argumentsActionImpl = new ArgumentsActionImpl(arguments, new EnvVars(passwordBinding)); + argumentsActionImpl = new ArgumentsActionImpl(arguments, new EnvVars(passwordBinding), sensitiveVariables); Assert.assertFalse(argumentsActionImpl.isUnmodifiedArguments()); Assert.assertEquals(ArgumentsActionImpl.NotStoredReason.MASKED_VALUE, argumentsActionImpl.getArgumentValueOrReason("message")); Assert.assertEquals(1, argumentsActionImpl.getArguments().size()); From eaa4c901891a6c62e1f99bf78b7d9b55ab18d9a7 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 23 Sep 2020 13:56:20 -0600 Subject: [PATCH 36/76] set sensitiveVariables as field instead of recursively passing through --- .../jenkinsci/plugins/workflow/cps/DSL.java | 17 +++------ .../cps/actions/ArgumentsActionImpl.java | 36 +++++++++---------- .../cps/actions/ArgumentsActionImplTest.java | 32 +++++++---------- 3 files changed, 34 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 733404e52..7f695cbc1 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -91,6 +91,7 @@ import org.kohsuke.stapler.NoStaplerConstructorException; import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import javax.annotation.Nullable; /** @@ -257,7 +258,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { final CpsStepContext context = new CpsStepContext(d, thread, handle, an, ps.body); EnvVars allEnv = null; - Set sensitiveVariables = null; + Set sensitiveVariables = Collections.emptySet(); try { allEnv = context.get(EnvVars.class); EnvironmentExpander envExpander = context.get(EnvironmentExpander.class); @@ -362,8 +363,8 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } } - private void logInterpolationWarnings(Set interpolatedStrings, @CheckForNull EnvVars envVars, Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { - if (interpolatedStrings.isEmpty() || envVars == null || envVars.isEmpty() || sensitiveVariables == null || sensitiveVariables.isEmpty()) { + private void logInterpolationWarnings(Set interpolatedStrings, @CheckForNull EnvVars envVars, @Nonnull Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { + if (interpolatedStrings.isEmpty() || envVars == null || envVars.isEmpty() || sensitiveVariables.isEmpty()) { return; } @@ -531,16 +532,6 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, Set interp } } -// static class InterpolatedUninstantiatedDescribable { -// final UninstantiatedDescribable ud; -// final Set interpolatedStrings; -// -// private InterpolatedUninstantiatedDescribable(UninstantiatedDescribable ud, Set interpolatedStrings) { -// this.ud = ud; -// this.interpolatedStrings = interpolatedStrings; -// } -// } - /** * Coerce {@link GString}, to save {@link StepDescriptor#newInstance(Map)} from being made aware of that. * This is not the only type coercion that Groovy does, so this is not very kosher, but diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java index 595ef927e..52c9a137c 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java @@ -46,15 +46,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; /** @@ -67,24 +64,27 @@ public class ArgumentsActionImpl extends ArgumentsAction { @CheckForNull private Map arguments; + private Set sensitiveVariables; boolean isUnmodifiedBySanitization = true; private static final Logger LOGGER = Logger.getLogger(ArgumentsActionImpl.class.getName()); - public ArgumentsActionImpl(@Nonnull Map stepArguments, @CheckForNull EnvVars env, @CheckForNull Set sensitiveVariables) { - arguments = serializationCheck(sanitizeMapAndRecordMutation(stepArguments, env, sensitiveVariables)); + public ArgumentsActionImpl(@Nonnull Map stepArguments, @CheckForNull EnvVars env, @Nonnull Set sensitiveVariables) { + this.sensitiveVariables = sensitiveVariables; + arguments = serializationCheck(sanitizeMapAndRecordMutation(stepArguments, env)); } /** Create a step, sanitizing strings for secured content */ public ArgumentsActionImpl(@Nonnull Map stepArguments) { - this(stepArguments, new EnvVars(), null); + this(stepArguments, new EnvVars(), Collections.emptySet()); } /** For testing use only */ - ArgumentsActionImpl(){ + ArgumentsActionImpl(@Nonnull Set sensitiveVariables){ this.isUnmodifiedBySanitization = false; this.arguments = Collections.emptyMap(); + this.sensitiveVariables = sensitiveVariables; } /** See if sensitive environment variable content is in a string */ @@ -128,7 +128,7 @@ boolean isStorableType(Object ob) { * Sanitize a list recursively */ @CheckForNull - Object sanitizeListAndRecordMutation(@Nonnull List objects, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { + Object sanitizeListAndRecordMutation(@Nonnull List objects, @CheckForNull EnvVars variables) { // Package scoped so we can test it directly if (isOversized(objects)) { @@ -139,7 +139,7 @@ Object sanitizeListAndRecordMutation(@Nonnull List objects, @CheckForNull EnvVar boolean isMutated = false; List output = new ArrayList(objects.size()); for (Object o : objects) { - Object modded = sanitizeObjectAndRecordMutation(o, variables, sensitiveVariables); + Object modded = sanitizeObjectAndRecordMutation(o, variables); if (modded != o) { // Sanitization stripped out some values, so we need to store the mutated object @@ -155,13 +155,13 @@ Object sanitizeListAndRecordMutation(@Nonnull List objects, @CheckForNull EnvVar /** For object arrays, we sanitize recursively, as with Lists */ @CheckForNull - Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { + Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull EnvVars variables) { if (isOversized(objects)) { this.isUnmodifiedBySanitization = false; return NotStoredReason.OVERSIZE_VALUE; } List inputList = Arrays.asList(objects); - Object sanitized = sanitizeListAndRecordMutation(inputList, variables, sensitiveVariables); + Object sanitized = sanitizeListAndRecordMutation(inputList, variables); if (sanitized == inputList) { // Works because if not mutated, we return original input instance return objects; } else if (sanitized instanceof List) { @@ -180,7 +180,7 @@ Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull E */ @CheckForNull @SuppressWarnings("unchecked") - Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull EnvVars vars, @CheckForNull Set sensitiveVariables) { + Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull EnvVars vars) { // Package scoped so we can test it directly Object tempVal = o; DescribableModel m = null; @@ -214,13 +214,13 @@ Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull Env Object modded = tempVal; if (modded instanceof Map) { // Recursive sanitization, oh my! - modded = sanitizeMapAndRecordMutation((Map)modded, vars, sensitiveVariables); + modded = sanitizeMapAndRecordMutation((Map)modded, vars); } else if (modded instanceof List) { - modded = sanitizeListAndRecordMutation((List) modded, vars, sensitiveVariables); + modded = sanitizeListAndRecordMutation((List) modded, vars); } else if (modded != null && modded.getClass().isArray()) { Class componentType = modded.getClass().getComponentType(); if (!componentType.isPrimitive()) { // Object arrays get recursively sanitized - modded = sanitizeArrayAndRecordMutation((Object[])modded, vars, sensitiveVariables); + modded = sanitizeArrayAndRecordMutation((Object[])modded, vars); } else { // Primitive arrays aren't a valid type here this.isUnmodifiedBySanitization = true; return NotStoredReason.UNSERIALIZABLE; @@ -280,16 +280,16 @@ Map serializationCheck(@Nonnull Map arguments) { } /** - * Goes through {@link #sanitizeObjectAndRecordMutation(Object, EnvVars, Set)} for each value in a map input. + * Goes through {@link #sanitizeObjectAndRecordMutation(Object, EnvVars)} for each value in a map input. */ @Nonnull - Map sanitizeMapAndRecordMutation(@Nonnull Map mapContents, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { + Map sanitizeMapAndRecordMutation(@Nonnull Map mapContents, @CheckForNull EnvVars variables) { // Package scoped so we can test it directly HashMap output = Maps.newHashMapWithExpectedSize(mapContents.size()); boolean isMutated = false; for (Map.Entry param : mapContents.entrySet()) { - Object modded = sanitizeObjectAndRecordMutation(param.getValue(), variables, sensitiveVariables); + Object modded = sanitizeObjectAndRecordMutation(param.getValue(), variables); if (modded != param.getValue()) { // Sanitization stripped out some values, so we need to store the mutated object diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java index b3e67a0c8..067ebf538 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java @@ -162,21 +162,10 @@ public void testStringSafetyTest() throws Exception { Assert.assertTrue("Input containing bound value is unsafe", ArgumentsActionImpl.isStringSensitive(input, new EnvVars(passwordBinding), sensitiveVariables)); Assert.assertFalse("EnvVars that do not occur are safe", ArgumentsActionImpl.isStringSensitive("I have no passwords", new EnvVars(passwordBinding), sensitiveVariables)); - -// HashMap safeBinding = new HashMap<>(); -// safeBinding.put("harmless", "secret"); -// HashSet safeVars = new HashSet<>(); -// safeVars.add("harmless"); -// passwordBinding.put("harmless", "secret"); -// Assert.assertTrue("Input containing whitelisted bound value is safe", ArgumentsActionImpl.isStringSafe(input, new EnvVars(safeBinding), safeVars)); -// Assert.assertFalse("Input containing one safe and one unsafe bound value is unsafe", ArgumentsActionImpl.isStringSafe(input, new EnvVars(passwordBinding), safeVars)); } @Test public void testRecursiveSanitizationOfContent() { - int maxLen = ArgumentsActionImpl.getMaxRetainedLength(); - ArgumentsActionImpl impl = new ArgumentsActionImpl(); - EnvVars env = new EnvVars(); String secretUsername = "secretuser"; env.put("USERVARIABLE", secretUsername); // assume secretuser is a bound credential @@ -184,27 +173,30 @@ public void testRecursiveSanitizationOfContent() { Set sensitiveVariables = new HashSet<>(); sensitiveVariables.add("USERVARIABLE"); + int maxLen = ArgumentsActionImpl.getMaxRetainedLength(); + ArgumentsActionImpl impl = new ArgumentsActionImpl(sensitiveVariables); + char[] oversized = new char[maxLen+10]; Arrays.fill(oversized, 'a'); String oversizedString = new String (oversized); // Simplest masking of secret and oversized value - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, impl.sanitizeObjectAndRecordMutation(secretUsername, env, sensitiveVariables)); + Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, impl.sanitizeObjectAndRecordMutation(secretUsername, env)); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; - Assert.assertEquals(ArgumentsAction.NotStoredReason.OVERSIZE_VALUE, impl.sanitizeObjectAndRecordMutation(oversizedString, env, sensitiveVariables)); + Assert.assertEquals(ArgumentsAction.NotStoredReason.OVERSIZE_VALUE, impl.sanitizeObjectAndRecordMutation(oversizedString, env)); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; // Test explosion of Step & UninstantiatedDescribable objects Step mystep = new EchoStep("I have a "+secretUsername); - Map singleSanitization = (Map)(impl.sanitizeObjectAndRecordMutation(mystep, env, sensitiveVariables)); + Map singleSanitization = (Map)(impl.sanitizeObjectAndRecordMutation(mystep, env)); Assert.assertEquals(1, singleSanitization.size()); Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, singleSanitization.get("message")); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; - singleSanitization = ((UninstantiatedDescribable) (impl.sanitizeObjectAndRecordMutation(mystep.getDescriptor().uninstantiate(mystep), env, sensitiveVariables))).getArguments(); + singleSanitization = ((UninstantiatedDescribable) (impl.sanitizeObjectAndRecordMutation(mystep.getDescriptor().uninstantiate(mystep), env))).getArguments(); Assert.assertEquals(1, singleSanitization.size()); Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, singleSanitization.get("message")); Assert.assertFalse(impl.isUnmodifiedArguments()); @@ -213,26 +205,26 @@ public void testRecursiveSanitizationOfContent() { // Maps HashMap dangerous = new HashMap<>(); dangerous.put("name", secretUsername); - Map sanitizedMap = impl.sanitizeMapAndRecordMutation(dangerous, env, sensitiveVariables); + Map sanitizedMap = impl.sanitizeMapAndRecordMutation(dangerous, env); Assert.assertNotEquals(sanitizedMap, dangerous); Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, sanitizedMap.get("name")); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; - Map identicalMap = impl.sanitizeMapAndRecordMutation(dangerous, new EnvVars(), sensitiveVariables); // String is no longer dangerous + Map identicalMap = impl.sanitizeMapAndRecordMutation(dangerous, new EnvVars()); // String is no longer dangerous Assert.assertEquals(identicalMap, dangerous); Assert.assertTrue(impl.isUnmodifiedArguments()); // Lists List unsanitizedList = Arrays.asList("cheese", null, secretUsername); - List sanitized = (List)impl.sanitizeListAndRecordMutation(unsanitizedList, env, sensitiveVariables); + List sanitized = (List)impl.sanitizeListAndRecordMutation(unsanitizedList, env); Assert.assertEquals(3, sanitized.size()); Assert.assertFalse(impl.isUnmodifiedArguments()); Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, sanitized.get(2)); impl.isUnmodifiedBySanitization = true; - Assert.assertEquals(unsanitizedList, impl.sanitizeObjectAndRecordMutation(unsanitizedList, new EnvVars(), sensitiveVariables)); - Assert.assertEquals(unsanitizedList, impl.sanitizeListAndRecordMutation(unsanitizedList, new EnvVars(), sensitiveVariables)); + Assert.assertEquals(unsanitizedList, impl.sanitizeObjectAndRecordMutation(unsanitizedList, new EnvVars())); + Assert.assertEquals(unsanitizedList, impl.sanitizeListAndRecordMutation(unsanitizedList, new EnvVars())); } @Test From 70c4c223dd4d08ca065fb9f8b0e41bb3ccc07c6d Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 23 Sep 2020 15:13:55 -0600 Subject: [PATCH 37/76] Move interpolatedStrings into parseArgs --- .../jenkinsci/plugins/workflow/cps/DSL.java | 44 +++++++++---------- .../plugins/workflow/cps/DSLTest.java | 13 ------ .../workflow/cps/SnippetizerTester.java | 2 +- 3 files changed, 23 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 7f695cbc1..f2f23ff9d 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -223,20 +223,7 @@ protected Object invokeStep(StepDescriptor d, Object args) { * @param args The arguments passed to the step. */ protected Object invokeStep(StepDescriptor d, String name, Object args) { - Set interpolatedStrings = null; - if (args instanceof NamedArgsAndClosure) { - interpolatedStrings = ((NamedArgsAndClosure) args).interpolatedStrings; - } else if (args instanceof Object[]) { - Object[] array = (Object[]) args; - if (array.length > 0 && array[array.length - 1] instanceof InterpolatedUninstantiatedDescribable) { - interpolatedStrings = ((InterpolatedUninstantiatedDescribable) array[array.length - 1]).getInterpolatedStrings(); - } - } - if (interpolatedStrings == null) { - interpolatedStrings = new HashSet<>(); - } - - final NamedArgsAndClosure ps = parseArgs(args, d, interpolatedStrings); + final NamedArgsAndClosure ps = parseArgs(args, d); CpsThread thread = CpsThread.current(); @@ -292,7 +279,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { ClassLoader originalLoader = Thread.currentThread().getContextClassLoader(); try { TaskListener listener = context.get(TaskListener.class); - logInterpolationWarnings(interpolatedStrings, allEnv, sensitiveVariables, listener); + logInterpolationWarnings(ps.interpolatedStrings, allEnv, sensitiveVariables, listener); if (unreportedAmbiguousFunctions.remove(name)) { reportAmbiguousStepInvocation(context, d, listener); } @@ -518,7 +505,7 @@ static class NamedArgsAndClosure { final List msgs; final Set interpolatedStrings; - private NamedArgsAndClosure(Map namedArgs, Closure body, Set interpolatedStrings) { + private NamedArgsAndClosure(Map namedArgs, Closure body, @Nonnull Set interpolatedStrings) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; this.msgs = new ArrayList<>(); @@ -542,12 +529,10 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, Set interp * but better to do it here in the Groovy-specific code so we do not need to rely on that. * @return {@code v} or an equivalent with all {@link GString}s flattened, including in nested {@link List}s or {@link Map}s */ - private static Object flattenGString(Object v, @CheckForNull Set interpolatedStrings) { + private static Object flattenGString(Object v, @Nonnull Set interpolatedStrings) { if (v instanceof GString) { String flattened = v.toString(); - if (interpolatedStrings != null) { - interpolatedStrings.add(flattened); - } + interpolatedStrings.add(flattened); return flattened; } else if (v instanceof List) { boolean mutated = false; @@ -575,7 +560,22 @@ private static Object flattenGString(Object v, @CheckForNull Set interpo } } - static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, Set interpolatedStrings) { + static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { + Set interpolatedStrings = new HashSet<>(); + if (arg instanceof NamedArgsAndClosure) { + interpolatedStrings = ((NamedArgsAndClosure) arg).interpolatedStrings; + } else if (arg instanceof Object[]) { + Object[] array = (Object[]) arg; + for (Object o : array) { + if (o instanceof InterpolatedUninstantiatedDescribable) { + interpolatedStrings.addAll(((InterpolatedUninstantiatedDescribable) o).getInterpolatedStrings()); + } + } + if (array.length > 0 && array[array.length - 1] instanceof InterpolatedUninstantiatedDescribable) { + interpolatedStrings = ((InterpolatedUninstantiatedDescribable) array[array.length - 1]).getInterpolatedStrings(); + } + } + boolean singleArgumentOnly = false; try { DescribableModel stepModel = DescribableModel.of(d.clazz); @@ -616,7 +616,7 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d, Set i // * @param envVars * The environment variables of the context */ - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, Set interpolatedStrings) { + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nonnull Set interpolatedStrings) { if (arg instanceof NamedArgsAndClosure) return (NamedArgsAndClosure) arg; if (arg instanceof Map) // TODO is this clause actually used? diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 5dfc01ff4..5a40153f9 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -477,12 +477,6 @@ public void namedSoleParamForStep() throws Exception { Set reportResults = reportAction.getResults(); MatcherAssert.assertThat(reportResults.size(), is(1)); MatcherAssert.assertThat(reportResults.iterator().next(), is("PASSWORD")); - // TODO: Code below currently fails -// LinearScanner scan = new LinearScanner(); -// FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("archiveArtifacts")); -// ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); -// Assert.assertFalse(argAction.isUnmodifiedArguments()); -// MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); } @Test public void describableNoMetaStep() throws Exception { @@ -498,13 +492,6 @@ public void namedSoleParamForStep() throws Exception { + "}\n" + "}", true)); r.assertLogContains("First arg: ****, second arg: two", r.assertBuildStatusSuccess(p.scheduleBuild2(0))); - // TODO: Code below currently fails -// WorkflowRun run = p.getLastBuild(); -// LinearScanner scanner = new LinearScanner(); -// FlowNode node = scanner.findFirstMatch(run.getExecution().getCurrentHeads(), new NodeStepTypePredicate("monomorphStep")); -// ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); -// Assert.assertNotNull(argumentsAction); -// Assert.assertEquals("one,two", ArgumentsAction.getStepArgumentsAsString(node)); } @Test public void noBodyError() throws Exception { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/SnippetizerTester.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/SnippetizerTester.java index 40fb6c155..762fce486 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/SnippetizerTester.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/SnippetizerTester.java @@ -128,7 +128,7 @@ public void assertParseStep(Step expectedStep, String script) throws Exception { @Override protected Object invokeStep(StepDescriptor d, String name, Object args) { try { - return d.newInstance(parseArgs(args, d, null).namedArgs); + return d.newInstance(parseArgs(args, d).namedArgs); } catch (Exception e) { throw new AssertionError(e); } From 8a799dc66456542e984aaf88daf1e2ca991b6142 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 23 Sep 2020 15:47:28 -0600 Subject: [PATCH 38/76] Remove duplicate code --- src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index f2f23ff9d..c17687232 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -571,9 +571,6 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { interpolatedStrings.addAll(((InterpolatedUninstantiatedDescribable) o).getInterpolatedStrings()); } } - if (array.length > 0 && array[array.length - 1] instanceof InterpolatedUninstantiatedDescribable) { - interpolatedStrings = ((InterpolatedUninstantiatedDescribable) array[array.length - 1]).getInterpolatedStrings(); - } } boolean singleArgumentOnly = false; From 1afab594b3f0b0364a7bbed5aa6bbaf597bc19a0 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 24 Sep 2020 23:32:50 -0600 Subject: [PATCH 39/76] no metaStep returns NamedArgsAndClosure --- .../jenkinsci/plugins/workflow/cps/DSL.java | 18 +++++++----- ...InterpolatedUninstantiatedDescribable.java | 29 ------------------- 2 files changed, 11 insertions(+), 36 deletions(-) delete mode 100644 src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedUninstantiatedDescribable.java diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index c17687232..64e2235d8 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -404,6 +404,7 @@ protected Object invokeDescribable(String symbol, Object _args) { // The only time a closure is valid is when the resulting Describable is immediately executed via a meta-step NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, interpolatedStrings); + UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); if (metaStep==null) { // there's no meta-step associated with it, so this symbol is not executable. @@ -415,13 +416,12 @@ protected Object invokeDescribable(String symbol, Object _args) { // also note that in this case 'd' is not trustworthy, as depending on // where this UninstantiatedDescribable is ultimately used, the symbol // might be resolved with a specific type. - return new InterpolatedUninstantiatedDescribable(symbol, null, args.namedArgs, interpolatedStrings); + args.uninstantiatedDescribable = ud; + return args; } else { - UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); Descriptor d = SymbolLookup.get().findDescriptor((Class)(metaStep.getMetaStepArgumentType()), symbol); try { // execute this Describable through a meta-step - // split args between MetaStep (represented by mm) and Describable (represented by dm) DescribableModel mm = DescribableModel.of(metaStep.clazz); DescribableModel dm = DescribableModel.of(d.clazz); @@ -504,6 +504,7 @@ static class NamedArgsAndClosure { final Closure body; final List msgs; final Set interpolatedStrings; + UninstantiatedDescribable uninstantiatedDescribable = null; private NamedArgsAndClosure(Map namedArgs, Closure body, @Nonnull Set interpolatedStrings) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); @@ -545,8 +546,8 @@ private static Object flattenGString(Object v, @Nonnull Set interpolated return mutated ? r : v; } else if (v instanceof Map) { boolean mutated = false; - Map r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) v).size())); - for (Map.Entry e : ((Map) v).entrySet()) { + Map r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) v).size())); + for (Map.Entry e : ((Map) v).entrySet()) { Object k = e.getKey(); Object k2 = flattenGString(k, interpolatedStrings); Object o = e.getValue(); @@ -555,6 +556,9 @@ private static Object flattenGString(Object v, @Nonnull Set interpolated r.put(k2, o2); } return mutated ? r : v; + } else if (v instanceof NamedArgsAndClosure) { + UninstantiatedDescribable ud = ((NamedArgsAndClosure) v).uninstantiatedDescribable; + return ud != null? ud : v; } else { return v; } @@ -567,8 +571,8 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { } else if (arg instanceof Object[]) { Object[] array = (Object[]) arg; for (Object o : array) { - if (o instanceof InterpolatedUninstantiatedDescribable) { - interpolatedStrings.addAll(((InterpolatedUninstantiatedDescribable) o).getInterpolatedStrings()); + if (o instanceof NamedArgsAndClosure) { + interpolatedStrings.addAll(((NamedArgsAndClosure) o).interpolatedStrings); } } } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedUninstantiatedDescribable.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedUninstantiatedDescribable.java deleted file mode 100644 index 980f463e6..000000000 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/InterpolatedUninstantiatedDescribable.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.jenkinsci.plugins.workflow.cps; - -import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable; - -import java.util.Map; -import java.util.Set; - -public class InterpolatedUninstantiatedDescribable extends UninstantiatedDescribable { - private final Set interpolatedStrings; - - public InterpolatedUninstantiatedDescribable(String symbol, String klass, Map arguments, Set interpolatedStrings) { - super(symbol, klass, arguments); - this.interpolatedStrings = interpolatedStrings; - } - - public Set getInterpolatedStrings() { - return interpolatedStrings; - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); - } -} From dbe7d8fb6c831b29efde6c67acc5410412a2f158 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Fri, 25 Sep 2020 00:59:16 -0600 Subject: [PATCH 40/76] Update error logging --- src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java | 8 +++++--- .../java/org/jenkinsci/plugins/workflow/cps/DSLTest.java | 5 ++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 64e2235d8..1c0fdc23d 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -279,7 +279,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { ClassLoader originalLoader = Thread.currentThread().getContextClassLoader(); try { TaskListener listener = context.get(TaskListener.class); - logInterpolationWarnings(ps.interpolatedStrings, allEnv, sensitiveVariables, listener); + logInterpolationWarnings(name, ps.interpolatedStrings, allEnv, sensitiveVariables, listener); if (unreportedAmbiguousFunctions.remove(name)) { reportAmbiguousStepInvocation(context, d, listener); } @@ -350,7 +350,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } } - private void logInterpolationWarnings(Set interpolatedStrings, @CheckForNull EnvVars envVars, @Nonnull Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { + private void logInterpolationWarnings(String stepName, Set interpolatedStrings, @CheckForNull EnvVars envVars, @Nonnull Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { if (interpolatedStrings.isEmpty() || envVars == null || envVars.isEmpty() || sensitiveVariables.isEmpty()) { return; } @@ -360,7 +360,9 @@ private void logInterpolationWarnings(Set interpolatedStrings, @CheckFor .collect(Collectors.toList()); if (scanResults != null && !scanResults.isEmpty()) { - listener.getLogger().println("The following Groovy string(s) may be insecure. Use single quotes to prevent leaking secrets via Groovy interpolation. Affected variable(s): " + scanResults.toString()); + String warning = String.format("Warning: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): %s%nSee for details.", + stepName, scanResults.toString()); + listener.getLogger().println(warning); FlowExecutionOwner owner = exec.getOwner(); if (owner != null && owner.getExecutable() instanceof Run) { InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 5a40153f9..2c5277a86 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -430,7 +430,6 @@ public void namedSoleParamForStep() throws Exception { "'org.jenkinsci.plugins.workflow.steps.SleepStep': comment,units", b); } - //TODO: JENKINS-47101, remove safe list check and change $PASSWORD variable to an old safe list variable @Test public void sensitiveVarsLogging() throws Exception { final String credentialsId = "creds"; final String username = "bob"; @@ -445,7 +444,7 @@ public void namedSoleParamForStep() throws Exception { + "}\n" + "}", true)); WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - r.assertLogContains("Affected variable(s): [PASSWORD]", run); + r.assertLogContains("Warning: A secret was passed to \"sh\" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); Set reportResults = reportAction.getResults(); @@ -471,7 +470,7 @@ public void namedSoleParamForStep() throws Exception { + "}\n" + "}", true)); WorkflowRun run = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains("Affected variable(s): [PASSWORD]", run); + r.assertLogContains("Warning: A secret was passed to \"archiveArtifacts\" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); Set reportResults = reportAction.getResults(); From 902ab29ca7b641eff80929443714faa582fbb2b6 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Fri, 25 Sep 2020 09:17:28 -0600 Subject: [PATCH 41/76] add windows support for unit test --- .../java/org/jenkinsci/plugins/workflow/cps/DSLTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 2c5277a86..3260e18ef 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -436,15 +436,15 @@ public void namedSoleParamForStep() throws Exception { final String password = "secr3t"; UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); - String shellStep = Functions.isWindows()? "bat \"echo $PASSWORD\"\n" : "sh \"echo $PASSWORD\"\n"; + String shellStep = Functions.isWindows()? "bat" : "sh"; p.setDefinition(new CpsFlowDefinition("" + "node {\n" + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" - + shellStep + + shellStep + " \"echo $PASSWORD\"\n" + "}\n" + "}", true)); WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - r.assertLogContains("Warning: A secret was passed to \"sh\" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [PASSWORD]", run); + r.assertLogContains("Warning: A secret was passed to \""+ shellStep + "\" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); Set reportResults = reportAction.getResults(); From 6a39c5c49ec80a658dbc9f0956d6c1c20c68f7fc Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 28 Sep 2020 00:52:35 -0600 Subject: [PATCH 42/76] report step name and arguments that log a warning --- .../jenkinsci/plugins/workflow/cps/DSL.java | 28 +++++++++---- .../cps/actions/ArgumentsActionImpl.java | 39 ++++++++++++++----- .../cps/view/InterpolatedSecretsAction.java | 22 ++++++----- .../InterpolatedSecretsAction/summary.jelly | 8 +++- .../plugins/workflow/cps/DSLTest.java | 28 +++++++++---- .../cps/actions/ArgumentsActionImplTest.java | 8 ++-- 6 files changed, 94 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 1c0fdc23d..e3c5765c7 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -46,6 +46,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -256,6 +257,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { LOGGER.log(Level.WARNING, "Unable to retrieve environment variables", e); } // Ensure ArgumentsAction is attached before we notify even synchronous listeners: + ArgumentsActionImpl argumentsAction = null; try { // No point storing empty arguments, and ParallelStep is a special case where we can't store its closure arguments if (ps.namedArgs != null && !(ps.namedArgs.isEmpty()) && isKeepStepArguments() && !(d instanceof ParallelStep.DescriptorImpl)) { @@ -264,7 +266,8 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { if (comp != null && allEnv != null) { allEnv.entrySet().removeAll(comp.getEnvironment().entrySet()); } - an.addAction(new ArgumentsActionImpl(ps.namedArgs, allEnv, sensitiveVariables)); + argumentsAction = new ArgumentsActionImpl(ps.namedArgs, allEnv, sensitiveVariables); + an.addAction(argumentsAction); } } catch (Exception e) { // Avoid breaking execution because we can't store some sort of crazy Step argument @@ -279,7 +282,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { ClassLoader originalLoader = Thread.currentThread().getContextClassLoader(); try { TaskListener listener = context.get(TaskListener.class); - logInterpolationWarnings(name, ps.interpolatedStrings, allEnv, sensitiveVariables, listener); + logInterpolationWarnings(name, argumentsAction, ps.interpolatedStrings, allEnv, sensitiveVariables, listener); if (unreportedAmbiguousFunctions.remove(name)) { reportAmbiguousStepInvocation(context, d, listener); } @@ -350,8 +353,8 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } } - private void logInterpolationWarnings(String stepName, Set interpolatedStrings, @CheckForNull EnvVars envVars, @Nonnull Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { - if (interpolatedStrings.isEmpty() || envVars == null || envVars.isEmpty() || sensitiveVariables.isEmpty()) { + private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsActionImpl argumentsAction, Set interpolatedStrings, @CheckForNull EnvVars envVars, @Nonnull Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { + if (argumentsAction == null || interpolatedStrings.isEmpty() || envVars == null || envVars.isEmpty() || sensitiveVariables.isEmpty()) { return; } @@ -360,8 +363,19 @@ private void logInterpolationWarnings(String stepName, Set interpolatedS .collect(Collectors.toList()); if (scanResults != null && !scanResults.isEmpty()) { - String warning = String.format("Warning: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): %s%nSee for details.", - stepName, scanResults.toString()); + Map> interpolatedArguments = new HashMap<>(); + // inspect the sanitized arguments to see if any interpolated strings were used + for (Entry> sanitizedEntries : argumentsAction.getSanitizedArguments().entrySet()) { + List sanitizedVariables = sanitizedEntries.getValue(); + for (String interpolatedVariable : scanResults) { + if (sanitizedVariables.contains(interpolatedVariable)) { + interpolatedArguments.computeIfAbsent(sanitizedEntries.getKey(), k -> new ArrayList<>()).add(interpolatedVariable); + } + } + } + + String warning = String.format("Warning: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure.%n\t\t Affected argument(s) used the following variable(s): %s%n\t\t See for details.", + stepName, interpolatedArguments.toString()); listener.getLogger().println(warning); FlowExecutionOwner owner = exec.getOwner(); if (owner != null && owner.getExecutable() instanceof Run) { @@ -370,7 +384,7 @@ private void logInterpolationWarnings(String stepName, Set interpolatedS runReport = new InterpolatedSecretsAction(); ((Run) owner.getExecutable()).addAction(runReport); } - runReport.record(scanResults); + runReport.record(stepName, interpolatedArguments); } else { LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java index 52c9a137c..45834c2f0 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java @@ -48,6 +48,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -64,7 +65,10 @@ public class ArgumentsActionImpl extends ArgumentsAction { @CheckForNull private Map arguments; - private Set sensitiveVariables; + private transient Set sensitiveVariables; + + private List sanitizedArgumentVariables = new ArrayList<>(); + private Map> sanitizedArguments = new HashMap<>(); boolean isUnmodifiedBySanitization = true; @@ -87,12 +91,19 @@ public ArgumentsActionImpl(@Nonnull Map stepArguments) { this.sensitiveVariables = sensitiveVariables; } - /** See if sensitive environment variable content is in a string */ - public static boolean isStringSensitive(@CheckForNull String input, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { + /** See if sensitive environment variable content is in a string and return the variable or null, if there is none*/ + public static String getAffectedVariable(@CheckForNull String input, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { if (input == null || variables == null || variables.size() == 0 || sensitiveVariables == null || sensitiveVariables.size() ==0) { - return false; + return null; + } + + try { + return sensitiveVariables.stream() + .filter(e -> input.contains(variables.get(e))) + .findFirst().get(); + } catch (NoSuchElementException e) { + return null; } - return sensitiveVariables.stream().map(variables::get).anyMatch(input::contains); } /** Restrict stored arguments to a reasonable subset of types so we don't retain totally arbitrary objects @@ -173,7 +184,7 @@ Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull E /** Recursively sanitize a single object by: * - Exploding {@link Step}s and {@link UninstantiatedDescribable}s into their Maps to sanitize - * - Removing unsafe strings using {@link #isStringSensitive(String, EnvVars, Set)} and replace with {@link NotStoredReason#MASKED_VALUE} + * - Removing unsafe strings using {@link #getAffectedVariable(String, EnvVars, Set)} (String, EnvVars, Set)} and replace with {@link NotStoredReason#MASKED_VALUE} * - Removing oversized objects using {@link #isOversized(Object)} and replacing with {@link NotStoredReason#OVERSIZE_VALUE} * While making an effort not to retain needless copies of objects and to re-use originals where possible * (including the Step or UninstantiatedDescribable) @@ -225,9 +236,13 @@ Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull Env this.isUnmodifiedBySanitization = true; return NotStoredReason.UNSERIALIZABLE; } - } else if (modded instanceof String && vars != null && !vars.isEmpty() && isStringSensitive((String)modded, vars, sensitiveVariables)) { - this.isUnmodifiedBySanitization = false; - return NotStoredReason.MASKED_VALUE; + } else if (modded instanceof String && vars != null && !vars.isEmpty()) { + String affectedVariable = getAffectedVariable((String)modded, vars, sensitiveVariables); + if (affectedVariable != null) { + sanitizedArgumentVariables.add(affectedVariable); + this.isUnmodifiedBySanitization = false; + return NotStoredReason.MASKED_VALUE; + } } if (modded != tempVal) { @@ -294,6 +309,8 @@ Map sanitizeMapAndRecordMutation(@Nonnull Map map if (modded != param.getValue()) { // Sanitization stripped out some values, so we need to store the mutated object output.put(param.getKey(), modded); + sanitizedArguments.put(param.getKey(), new ArrayList<>(sanitizedArgumentVariables)); + sanitizedArgumentVariables.clear(); isMutated = true; //isUnmodifiedBySanitization was already set } else { // Any mutation was just from exploding step/uninstantiated describable, and we can just use the original output.put(param.getKey(), param.getValue()); @@ -308,6 +325,10 @@ static int getMaxRetainedLength() { return MAX_RETAINED_LENGTH; } + public Map> getSanitizedArguments() { + return sanitizedArguments.isEmpty() ? Collections.EMPTY_MAP : sanitizedArguments; + } + @Nonnull @Override protected Map getArgumentsInternal() { diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index a95295ecc..c2f12d75a 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -2,16 +2,19 @@ import hudson.model.Run; import jenkins.model.RunAction2; -import java.util.HashSet; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; -import java.util.Set; +import java.util.Map; /** * Action to generate the UI report for watched environment variables */ public class InterpolatedSecretsAction implements RunAction2 { - private Set results; + private List> interpolatedWarnings; private transient Run run; public String getIconFileName() { @@ -26,15 +29,15 @@ public String getUrlName() { return null; } - public void record(List stepResults) { - if (results == null) { - results = new HashSet<>(); + public void record(@Nonnull String stepName, @Nonnull Map> stepArguments) { + if (interpolatedWarnings == null) { + interpolatedWarnings = new ArrayList<>(); } - results.addAll(stepResults); + interpolatedWarnings.add(Arrays.asList(stepName, stepArguments)); } - public Set getResults() { - return results; + public List> getWarnings() { + return interpolatedWarnings; } public boolean getInProgress() { @@ -50,5 +53,4 @@ public void onAttached(Run run) { public void onLoad(Run run) { this.run = run; } - } diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index c9ff5f71e..ddbda443d 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -29,8 +29,12 @@ THE SOFTWARE. (${%in progress})
          - -
        • ${variable}
          • + +
        • ${warning.get(0)}( + + ${arguments} + + )
          •
        diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 3260e18ef..22b65ac49 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -33,6 +33,7 @@ import hudson.model.Descriptor; import hudson.model.Result; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -444,12 +445,17 @@ public void namedSoleParamForStep() throws Exception { + "}\n" + "}", true)); WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - r.assertLogContains("Warning: A secret was passed to \""+ shellStep + "\" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [PASSWORD]", run); + r.assertLogContains("Warning: A secret was passed to \""+ shellStep + "\"", run); + r.assertLogContains("Affected argument(s) used the following variable(s): {script=[PASSWORD]}", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - Set reportResults = reportAction.getResults(); + List> reportResults = reportAction.getWarnings(); MatcherAssert.assertThat(reportResults.size(), is(1)); - MatcherAssert.assertThat(reportResults.iterator().next(), is("PASSWORD")); + List warning = reportResults.get(0); + MatcherAssert.assertThat(warning.get(0), is("sh")); + Map args = (Map)warning.get(1); + MatcherAssert.assertThat(args.size(), is(1)); + MatcherAssert.assertThat(args.get("script"), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(Functions.isWindows()? "bat" : "sh")); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); @@ -470,12 +476,17 @@ public void namedSoleParamForStep() throws Exception { + "}\n" + "}", true)); WorkflowRun run = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains("Warning: A secret was passed to \"archiveArtifacts\" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [PASSWORD]", run); + r.assertLogContains("Warning: A secret was passed to \"archiveArtifacts\"", run); + r.assertLogContains("Affected argument(s) used the following variable(s): {=[PASSWORD]}", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - Set reportResults = reportAction.getResults(); + List> reportResults = reportAction.getWarnings(); MatcherAssert.assertThat(reportResults.size(), is(1)); - MatcherAssert.assertThat(reportResults.iterator().next(), is("PASSWORD")); + List warning = reportResults.get(0); + MatcherAssert.assertThat(warning.get(0), is("archiveArtifacts")); + Map args = (Map)warning.get(1); + MatcherAssert.assertThat(args.size(), is(1)); + MatcherAssert.assertThat(args.get("anonymous"), is(Arrays.asList("PASSWORD"))); } @Test public void describableNoMetaStep() throws Exception { @@ -490,7 +501,10 @@ public void namedSoleParamForStep() throws Exception { + "monomorphWithSymbolStep(monomorphSymbol([firstArg:\"${PASSWORD}\", secondArg:'two']))" + "}\n" + "}", true)); - r.assertLogContains("First arg: ****, second arg: two", r.assertBuildStatusSuccess(p.scheduleBuild2(0))); + WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + r.assertLogContains("First arg: ****, second arg: two", run); + r.assertLogContains("Warning: A secret was passed to \"monomorphWithSymbolStep\"", run); + r.assertLogContains("Affected argument(s) used the following variable(s): {firstArg=[PASSWORD]}", run); } @Test public void noBodyError() throws Exception { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java index 067ebf538..00e92a5ae 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java @@ -15,6 +15,7 @@ import hudson.model.Action; import hudson.tasks.ArtifactArchiver; import org.apache.commons.lang.RandomStringUtils; +import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; import org.hamcrest.collection.IsMapContaining; import org.jenkinsci.plugins.credentialsbinding.impl.BindingStep; @@ -158,10 +159,9 @@ public void testStringSafetyTest() throws Exception { passwordBinding.put("mypass", "p4ssw0rd"); Set sensitiveVariables = new HashSet<>(); sensitiveVariables.add("mypass"); - Assert.assertFalse("Input with no variables is safe", ArgumentsActionImpl.isStringSensitive(input, new EnvVars(), sensitiveVariables)); - Assert.assertTrue("Input containing bound value is unsafe", ArgumentsActionImpl.isStringSensitive(input, new EnvVars(passwordBinding), sensitiveVariables)); - - Assert.assertFalse("EnvVars that do not occur are safe", ArgumentsActionImpl.isStringSensitive("I have no passwords", new EnvVars(passwordBinding), sensitiveVariables)); + Assert.assertNull("Input with no variables is safe", ArgumentsActionImpl.getAffectedVariable(input, new EnvVars(), sensitiveVariables)); + MatcherAssert.assertThat("Input containing bound value is unsafe", "mypass".equals(ArgumentsActionImpl.getAffectedVariable(input, new EnvVars(passwordBinding), sensitiveVariables))); + Assert.assertNull("EnvVars that do not occur are safe", ArgumentsActionImpl.getAffectedVariable("I have no passwords", new EnvVars(passwordBinding), sensitiveVariables)); } @Test From ec29d4ba5f4b39fba167c1dace2ab46aa8b25a0d Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 28 Sep 2020 12:21:43 -0600 Subject: [PATCH 43/76] Handle multiple sensitive variables in one argument --- .../cps/actions/ArgumentsActionImpl.java | 26 ++++++-------- .../InterpolatedSecretsAction/summary.jelly | 8 +++-- .../plugins/workflow/cps/DSLTest.java | 34 ++++++++++++++++++- .../cps/actions/ArgumentsActionImplTest.java | 6 ++-- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java index 45834c2f0..8af1ea817 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java @@ -48,12 +48,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; - +import java.util.stream.Collectors; /** * Implements {@link ArgumentsAction} by storing step arguments, with sanitization. @@ -92,18 +91,15 @@ public ArgumentsActionImpl(@Nonnull Map stepArguments) { } /** See if sensitive environment variable content is in a string and return the variable or null, if there is none*/ - public static String getAffectedVariable(@CheckForNull String input, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { + @Nonnull + public static List getAffectedVariables(@CheckForNull String input, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { if (input == null || variables == null || variables.size() == 0 || sensitiveVariables == null || sensitiveVariables.size() ==0) { - return null; + return Collections.emptyList(); } - try { - return sensitiveVariables.stream() - .filter(e -> input.contains(variables.get(e))) - .findFirst().get(); - } catch (NoSuchElementException e) { - return null; - } + return sensitiveVariables.stream() + .filter(e -> input.contains(variables.get(e))) + .collect(Collectors.toList()); } /** Restrict stored arguments to a reasonable subset of types so we don't retain totally arbitrary objects @@ -184,7 +180,7 @@ Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull E /** Recursively sanitize a single object by: * - Exploding {@link Step}s and {@link UninstantiatedDescribable}s into their Maps to sanitize - * - Removing unsafe strings using {@link #getAffectedVariable(String, EnvVars, Set)} (String, EnvVars, Set)} and replace with {@link NotStoredReason#MASKED_VALUE} + * - Removing unsafe strings using {@link #getAffectedVariables(String, EnvVars, Set)} (String, EnvVars, Set)} and replace with {@link NotStoredReason#MASKED_VALUE} * - Removing oversized objects using {@link #isOversized(Object)} and replacing with {@link NotStoredReason#OVERSIZE_VALUE} * While making an effort not to retain needless copies of objects and to re-use originals where possible * (including the Step or UninstantiatedDescribable) @@ -237,9 +233,9 @@ Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull Env return NotStoredReason.UNSERIALIZABLE; } } else if (modded instanceof String && vars != null && !vars.isEmpty()) { - String affectedVariable = getAffectedVariable((String)modded, vars, sensitiveVariables); - if (affectedVariable != null) { - sanitizedArgumentVariables.add(affectedVariable); + List affectedVariables = getAffectedVariables((String)modded, vars, sensitiveVariables); + if (affectedVariables != null && !affectedVariables.isEmpty()) { + sanitizedArgumentVariables.addAll(affectedVariables); this.isUnmodifiedBySanitization = false; return NotStoredReason.MASKED_VALUE; } diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index ddbda443d..4839cb697 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -20,7 +20,7 @@ THE SOFTWARE. --> - + ${%Possible insecure use of sensitive variables} @@ -28,14 +28,16 @@ THE SOFTWARE. (${%in progress}) +
          -
        • ${warning.get(0)}( +
        • ${warning.get(0)}( - ${arguments} + ${arguments} )
          •
        +         \ No newline at end of file diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 22b65ac49..041e6d05d 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -486,7 +486,39 @@ public void namedSoleParamForStep() throws Exception { MatcherAssert.assertThat(warning.get(0), is("archiveArtifacts")); Map args = (Map)warning.get(1); MatcherAssert.assertThat(args.size(), is(1)); - MatcherAssert.assertThat(args.get("anonymous"), is(Arrays.asList("PASSWORD"))); + MatcherAssert.assertThat(args.get(""), is(Arrays.asList("PASSWORD"))); + } + + @Test public void multipleSensitiveVariables() throws Exception { + final String credentialsId = "creds"; + final String username = "bob"; + final String password = "secr3t"; + UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); + CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); + String shellStep = Functions.isWindows()? "bat" : "sh"; + p.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" + + shellStep + " \"echo $PASSWORD $USERNAME $PASSWORD\"\n" + + "}\n" + + "}", true)); + WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + r.assertLogContains("Warning: A secret was passed to \""+ shellStep + "\"", run); + r.assertLogContains("Affected argument(s) used the following variable(s): {script=[PASSWORD, USERNAME]}", run); + InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); + Assert.assertNotNull(reportAction); + List> reportResults = reportAction.getWarnings(); + MatcherAssert.assertThat(reportResults.size(), is(1)); + List warning = reportResults.get(0); + MatcherAssert.assertThat(warning.get(0), is("sh")); + Map args = (Map)warning.get(1); + MatcherAssert.assertThat(args.size(), is(1)); + MatcherAssert.assertThat(args.get("script"), is(Arrays.asList("PASSWORD", "USERNAME"))); + LinearScanner scan = new LinearScanner(); + FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(Functions.isWindows()? "bat" : "sh")); + ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); + Assert.assertFalse(argAction.isUnmodifiedArguments()); + MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); } @Test public void describableNoMetaStep() throws Exception { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java index 00e92a5ae..47aa482fc 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java @@ -159,9 +159,9 @@ public void testStringSafetyTest() throws Exception { passwordBinding.put("mypass", "p4ssw0rd"); Set sensitiveVariables = new HashSet<>(); sensitiveVariables.add("mypass"); - Assert.assertNull("Input with no variables is safe", ArgumentsActionImpl.getAffectedVariable(input, new EnvVars(), sensitiveVariables)); - MatcherAssert.assertThat("Input containing bound value is unsafe", "mypass".equals(ArgumentsActionImpl.getAffectedVariable(input, new EnvVars(passwordBinding), sensitiveVariables))); - Assert.assertNull("EnvVars that do not occur are safe", ArgumentsActionImpl.getAffectedVariable("I have no passwords", new EnvVars(passwordBinding), sensitiveVariables)); + MatcherAssert.assertThat("Input with no variables is safe", ArgumentsActionImpl.getAffectedVariables(input, new EnvVars(), sensitiveVariables).isEmpty()); + MatcherAssert.assertThat("Input containing bound value is unsafe", Arrays.asList("mypass").equals(ArgumentsActionImpl.getAffectedVariables(input, new EnvVars(passwordBinding), sensitiveVariables))); + MatcherAssert.assertThat("EnvVars that do not occur are safe", ArgumentsActionImpl.getAffectedVariables("I have no passwords", new EnvVars(passwordBinding), sensitiveVariables).isEmpty()); } @Test From 97b153547ba9d179dc1e38df7a0a295559e9fcf8 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 28 Sep 2020 14:04:50 -0600 Subject: [PATCH 44/76] fix windows tests --- .../java/org/jenkinsci/plugins/workflow/cps/DSLTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 041e6d05d..eba152a96 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -452,12 +452,12 @@ public void namedSoleParamForStep() throws Exception { List> reportResults = reportAction.getWarnings(); MatcherAssert.assertThat(reportResults.size(), is(1)); List warning = reportResults.get(0); - MatcherAssert.assertThat(warning.get(0), is("sh")); + MatcherAssert.assertThat(warning.get(0), is(shellStep)); Map args = (Map)warning.get(1); MatcherAssert.assertThat(args.size(), is(1)); MatcherAssert.assertThat(args.get("script"), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); - FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(Functions.isWindows()? "bat" : "sh")); + FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(shellStep)); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); Assert.assertFalse(argAction.isUnmodifiedArguments()); MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); @@ -510,12 +510,12 @@ public void namedSoleParamForStep() throws Exception { List> reportResults = reportAction.getWarnings(); MatcherAssert.assertThat(reportResults.size(), is(1)); List warning = reportResults.get(0); - MatcherAssert.assertThat(warning.get(0), is("sh")); + MatcherAssert.assertThat(warning.get(0), is(shellStep)); Map args = (Map)warning.get(1); MatcherAssert.assertThat(args.size(), is(1)); MatcherAssert.assertThat(args.get("script"), is(Arrays.asList("PASSWORD", "USERNAME"))); LinearScanner scan = new LinearScanner(); - FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(Functions.isWindows()? "bat" : "sh")); + FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(shellStep)); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); Assert.assertFalse(argAction.isUnmodifiedArguments()); MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); From b0d183b9196b0e6d425109cbcac24cdf56789719 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 29 Sep 2020 13:36:17 -0600 Subject: [PATCH 45/76] update documentation --- .../java/org/jenkinsci/plugins/workflow/cps/DSL.java | 10 +++++++--- .../cps/view/InterpolatedSecretsAction/summary.jelly | 3 +-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index e3c5765c7..795aefba6 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -514,7 +514,10 @@ private static int preallocatedHashmapCapacity(int elementsToHold) { } } - // TODO: add java doc + /** + * This class holds the argument map and optional body of the step that is to be invoked. + * The UninstantiatedDescribable field is set when the associated symbol is being built as the parameter of a step to be invoked. + */ static class NamedArgsAndClosure { final Map namedArgs; final Closure body; @@ -544,6 +547,7 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nonnull Set interpolatedStrings) { @@ -630,8 +634,8 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { * @param soleArgumentKey * If the context in which this method call happens allow implicit sole default argument, specify its name. * If null, the call must be with names arguments. -// * @param envVars - * The environment variables of the context + * @param interpolatedStrings + * The collection of interpolated Groovy strings. */ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nonnull Set interpolatedStrings) { if (arg instanceof NamedArgsAndClosure) diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index 4839cb697..a1a3d58ce 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -23,8 +23,7 @@ THE SOFTWARE. ${%Possible insecure use of sensitive variables} - - (${%click here for an explanation}): + (${%click here for an explanation}): (${%in progress}) From b846c4443b1665cc9eabb7b9163a7543319eafe9 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 1 Oct 2020 10:39:21 -0600 Subject: [PATCH 46/76] refactoring --- .../jenkinsci/plugins/workflow/cps/DSL.java | 33 ++++----- .../cps/actions/ArgumentsActionImpl.java | 41 ++++------- .../cps/view/InterpolatedSecretsAction.java | 70 +++++++++++++++++-- .../InterpolatedSecretsAction/summary.jelly | 21 +++--- .../plugins/workflow/cps/DSLTest.java | 48 ++++++------- .../cps/actions/ArgumentsActionImplTest.java | 39 +++++++---- 6 files changed, 148 insertions(+), 104 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 795aefba6..a008a55ae 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -251,7 +251,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { allEnv = context.get(EnvVars.class); EnvironmentExpander envExpander = context.get(EnvironmentExpander.class); if (envExpander != null) { - sensitiveVariables = envExpander.getSensitiveVariables(); + sensitiveVariables = new HashSet<>(envExpander.getSensitiveVariables()); } } catch (IOException | InterruptedException e) { LOGGER.log(Level.WARNING, "Unable to retrieve environment variables", e); @@ -363,20 +363,10 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc .collect(Collectors.toList()); if (scanResults != null && !scanResults.isEmpty()) { - Map> interpolatedArguments = new HashMap<>(); - // inspect the sanitized arguments to see if any interpolated strings were used - for (Entry> sanitizedEntries : argumentsAction.getSanitizedArguments().entrySet()) { - List sanitizedVariables = sanitizedEntries.getValue(); - for (String interpolatedVariable : scanResults) { - if (sanitizedVariables.contains(interpolatedVariable)) { - interpolatedArguments.computeIfAbsent(sanitizedEntries.getKey(), k -> new ArrayList<>()).add(interpolatedVariable); - } - } - } - String warning = String.format("Warning: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure.%n\t\t Affected argument(s) used the following variable(s): %s%n\t\t See for details.", - stepName, interpolatedArguments.toString()); + stepName, scanResults.toString()); listener.getLogger().println(warning); + FlowExecutionOwner owner = exec.getOwner(); if (owner != null && owner.getExecutable() instanceof Run) { InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); @@ -384,7 +374,7 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc runReport = new InterpolatedSecretsAction(); ((Run) owner.getExecutable()).addAction(runReport); } - runReport.record(stepName, interpolatedArguments); + runReport.record(stepName, argumentsAction.getArguments(), scanResults); } else { LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); } @@ -405,7 +395,6 @@ private static String loadSoleArgumentKey(StepDescriptor d) { */ @SuppressWarnings({"unchecked", "rawtypes"}) protected Object invokeDescribable(String symbol, Object _args) { - Set interpolatedStrings = new HashSet<>(); List metaSteps = StepDescriptor.metaStepsOf(symbol); StepDescriptor metaStep = metaSteps.size()==1 ? metaSteps.get(0) : null; @@ -419,7 +408,7 @@ protected Object invokeDescribable(String symbol, Object _args) { // The only time a closure is valid is when the resulting Describable is immediately executed via a meta-step NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), - UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, interpolatedStrings); + UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly); UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); if (metaStep==null) { @@ -479,7 +468,7 @@ protected Object invokeDescribable(String symbol, Object _args) { ud = new UninstantiatedDescribable(symbol, null, dargs); margs.put(p.getName(),ud); - return invokeStep(metaStep, symbol, new NamedArgsAndClosure(margs, args.body, interpolatedStrings)); + return invokeStep(metaStep, symbol, new NamedArgsAndClosure(margs, args.body, args.interpolatedStrings)); } catch (Exception e) { throw new IllegalArgumentException("Failed to prepare "+symbol+" step",e); } @@ -518,18 +507,18 @@ private static int preallocatedHashmapCapacity(int elementsToHold) { * This class holds the argument map and optional body of the step that is to be invoked. * The UninstantiatedDescribable field is set when the associated symbol is being built as the parameter of a step to be invoked. */ - static class NamedArgsAndClosure { + static class NamedArgsAndClosure implements Serializable { final Map namedArgs; final Closure body; final List msgs; final Set interpolatedStrings; UninstantiatedDescribable uninstantiatedDescribable = null; - private NamedArgsAndClosure(Map namedArgs, Closure body, @Nonnull Set interpolatedStrings) { + private NamedArgsAndClosure(Map namedArgs, Closure body, @Nonnull Set foundInterpolatedStrings) { this.namedArgs = new LinkedHashMap<>(preallocatedHashmapCapacity(namedArgs.size())); this.body = body; this.msgs = new ArrayList<>(); - this.interpolatedStrings = interpolatedStrings; + this.interpolatedStrings = new HashSet<>(foundInterpolatedStrings); for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more @@ -612,6 +601,10 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, interpolatedStrings); } + static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg) { + return parseArgs(arg, expectsBlock, soleArgumentKey, singleRequiredArg, new HashSet<>()); + } + /** * Given the Groovy style argument packing used in the sole object parameter of {@link GroovyObject#invokeMethod(String, Object)}, * compute the named argument map and an optional closure that represents the body. diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java index 8af1ea817..c61461da0 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java @@ -64,10 +64,7 @@ public class ArgumentsActionImpl extends ArgumentsAction { @CheckForNull private Map arguments; - private transient Set sensitiveVariables; - - private List sanitizedArgumentVariables = new ArrayList<>(); - private Map> sanitizedArguments = new HashMap<>(); + private final Set sensitiveVariables; boolean isUnmodifiedBySanitization = true; @@ -90,16 +87,16 @@ public ArgumentsActionImpl(@Nonnull Map stepArguments) { this.sensitiveVariables = sensitiveVariables; } - /** See if sensitive environment variable content is in a string and return the variable or null, if there is none*/ - @Nonnull - public static List getAffectedVariables(@CheckForNull String input, @CheckForNull EnvVars variables, @CheckForNull Set sensitiveVariables) { - if (input == null || variables == null || variables.size() == 0 || sensitiveVariables == null || sensitiveVariables.size() ==0) { - return Collections.emptyList(); + /** See if sensitive environment variable content is in a string and replace the content with its associated variable name, otherwise return string unmodified*/ + public static String replaceSensitiveVariables(@Nonnull String input, @CheckForNull EnvVars variables, @Nonnull Set sensitiveVariables) { + if (variables == null || variables.size() == 0 || sensitiveVariables.size() ==0) { + return input; } - - return sensitiveVariables.stream() - .filter(e -> input.contains(variables.get(e))) - .collect(Collectors.toList()); + String modded = input; + for (String sensitive : sensitiveVariables) { + modded = modded.replace(variables.get(sensitive), "${" + sensitive + "}"); + } + return modded; } /** Restrict stored arguments to a reasonable subset of types so we don't retain totally arbitrary objects @@ -180,7 +177,7 @@ Object sanitizeArrayAndRecordMutation(@Nonnull Object[] objects, @CheckForNull E /** Recursively sanitize a single object by: * - Exploding {@link Step}s and {@link UninstantiatedDescribable}s into their Maps to sanitize - * - Removing unsafe strings using {@link #getAffectedVariables(String, EnvVars, Set)} (String, EnvVars, Set)} and replace with {@link NotStoredReason#MASKED_VALUE} + * - Removing unsafe strings using {@link #replaceSensitiveVariables(String, EnvVars, Set)} and replace with the variable name * - Removing oversized objects using {@link #isOversized(Object)} and replacing with {@link NotStoredReason#OVERSIZE_VALUE} * While making an effort not to retain needless copies of objects and to re-use originals where possible * (including the Step or UninstantiatedDescribable) @@ -233,12 +230,11 @@ Object sanitizeObjectAndRecordMutation(@CheckForNull Object o, @CheckForNull Env return NotStoredReason.UNSERIALIZABLE; } } else if (modded instanceof String && vars != null && !vars.isEmpty()) { - List affectedVariables = getAffectedVariables((String)modded, vars, sensitiveVariables); - if (affectedVariables != null && !affectedVariables.isEmpty()) { - sanitizedArgumentVariables.addAll(affectedVariables); + String replaced = replaceSensitiveVariables((String)modded, vars, sensitiveVariables); + if (!replaced.equals(modded)) { this.isUnmodifiedBySanitization = false; - return NotStoredReason.MASKED_VALUE; } + return replaced; } if (modded != tempVal) { @@ -300,13 +296,10 @@ Map sanitizeMapAndRecordMutation(@Nonnull Map map boolean isMutated = false; for (Map.Entry param : mapContents.entrySet()) { - Object modded = sanitizeObjectAndRecordMutation(param.getValue(), variables); - + Object modded = sanitizeObjectAndRecordMutation(param.getValue(), variables);//, sanitizedArgumentVariables); if (modded != param.getValue()) { // Sanitization stripped out some values, so we need to store the mutated object output.put(param.getKey(), modded); - sanitizedArguments.put(param.getKey(), new ArrayList<>(sanitizedArgumentVariables)); - sanitizedArgumentVariables.clear(); isMutated = true; //isUnmodifiedBySanitization was already set } else { // Any mutation was just from exploding step/uninstantiated describable, and we can just use the original output.put(param.getKey(), param.getValue()); @@ -321,10 +314,6 @@ static int getMaxRetainedLength() { return MAX_RETAINED_LENGTH; } - public Map> getSanitizedArguments() { - return sanitizedArguments.isEmpty() ? Collections.EMPTY_MAP : sanitizedArguments; - } - @Nonnull @Override protected Map getArgumentsInternal() { diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index c2f12d75a..d4f356149 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -2,19 +2,24 @@ import hudson.model.Run; import jenkins.model.RunAction2; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; +import org.kohsuke.stapler.export.Exported; +import org.kohsuke.stapler.export.ExportedBean; import javax.annotation.Nonnull; +import java.io.Serializable; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; /** * Action to generate the UI report for watched environment variables */ +@Restricted(NoExternalUse.class) public class InterpolatedSecretsAction implements RunAction2 { - private List> interpolatedWarnings; + private List interpolatedWarnings; private transient Run run; public String getIconFileName() { @@ -29,15 +34,39 @@ public String getUrlName() { return null; } - public void record(@Nonnull String stepName, @Nonnull Map> stepArguments) { + public void record(@Nonnull String stepName, @Nonnull Map stepArguments, @Nonnull List interpolatedVariables) { if (interpolatedWarnings == null) { interpolatedWarnings = new ArrayList<>(); } - interpolatedWarnings.add(Arrays.asList(stepName, stepArguments)); + interpolatedWarnings.add(new InterpolatedWarnings(stepName, stepArguments, interpolatedVariables)); } - public List> getWarnings() { - return interpolatedWarnings; + public List getWarningsOutput() { + List outputList = new ArrayList<>(); + StringBuilder sb = new StringBuilder(); + + for (InterpolatedWarnings warning : interpolatedWarnings) { + sb.append(warning.stepName + "("); + for (Map.Entry argEntry : warning.stepArguments.entrySet()) { + sb.append(argEntry.getKey() + ": " + argEntry.getValue().toString()); + } + sb.append(")\n"); + sb.append("interpolated variable(s): " + warning.interpolatedVariables.toString()); + } + outputList.add(sb.toString()); + return outputList; + } + + public List getWarnings() { + return interpolatedWarnings; + } + + public boolean getHasWarnings() { + if (interpolatedWarnings == null || interpolatedWarnings.isEmpty()) { + return false; + } else { + return true; + } } public boolean getInProgress() { @@ -53,4 +82,33 @@ public void onAttached(Run run) { public void onLoad(Run run) { this.run = run; } + + @ExportedBean + static class InterpolatedWarnings implements Serializable { + final String stepName; + final Map stepArguments; + final List interpolatedVariables; + + private InterpolatedWarnings(@Nonnull String stepName, @Nonnull Map stepArguments, @Nonnull List interpolatedVariables) { + this.stepName = stepName; + this.stepArguments = stepArguments; + this.interpolatedVariables = interpolatedVariables; + } + + @Exported + public String getStepSignature() { + StringBuilder sb = new StringBuilder(); + sb.append(stepName + "("); + for (Map.Entry argEntry : stepArguments.entrySet()) { + sb.append(argEntry.getKey() + ": " + argEntry.getValue().toString()); + } + sb.append(")"); + return sb.toString(); + } + + @Override + public String toString() { + return getStepSignature(); + } + } } diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index a1a3d58ce..33d29012a 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -20,23 +20,22 @@ THE SOFTWARE. --> - + ${%Possible insecure use of sensitive variables} (${%click here for an explanation}): (${%in progress}) - -
          - -
        • ${warning.get(0)}( - - ${arguments} - - )
          • -           -
        + +
          + +
        • ${warning}
          • +
            +
          • ${warning.interpolatedVariables}
            • +
          +           +
        \ No newline at end of file diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index eba152a96..97a4ab9d5 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -33,7 +33,6 @@ import hudson.model.Descriptor; import hudson.model.Result; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -41,6 +40,7 @@ import static org.hamcrest.Matchers.containsString; import org.hamcrest.MatcherAssert; +import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable; import org.jenkinsci.plugins.workflow.actions.ArgumentsAction; import org.jenkinsci.plugins.workflow.cps.view.InterpolatedSecretsAction; import org.jenkinsci.plugins.workflow.graph.FlowNode; @@ -446,21 +446,15 @@ public void namedSoleParamForStep() throws Exception { + "}", true)); WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); r.assertLogContains("Warning: A secret was passed to \""+ shellStep + "\"", run); - r.assertLogContains("Affected argument(s) used the following variable(s): {script=[PASSWORD]}", run); + r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - List> reportResults = reportAction.getWarnings(); - MatcherAssert.assertThat(reportResults.size(), is(1)); - List warning = reportResults.get(0); - MatcherAssert.assertThat(warning.get(0), is(shellStep)); - Map args = (Map)warning.get(1); - MatcherAssert.assertThat(args.size(), is(1)); - MatcherAssert.assertThat(args.get("script"), is(Arrays.asList("PASSWORD"))); + MatcherAssert.assertThat(reportAction.getWarningsOutput(), is(shellStep + "(script: echo ${PASSWORD})\n interpolated variable(s): [PASSWORD]")); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(shellStep)); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); Assert.assertFalse(argAction.isUnmodifiedArguments()); - MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); + MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), is("echo ${PASSWORD}")); } @Test public void describableInterpolation() throws Exception { @@ -477,16 +471,10 @@ public void namedSoleParamForStep() throws Exception { + "}", true)); WorkflowRun run = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); r.assertLogContains("Warning: A secret was passed to \"archiveArtifacts\"", run); - r.assertLogContains("Affected argument(s) used the following variable(s): {=[PASSWORD]}", run); + r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - List> reportResults = reportAction.getWarnings(); - MatcherAssert.assertThat(reportResults.size(), is(1)); - List warning = reportResults.get(0); - MatcherAssert.assertThat(warning.get(0), is("archiveArtifacts")); - Map args = (Map)warning.get(1); - MatcherAssert.assertThat(args.size(), is(1)); - MatcherAssert.assertThat(args.get(""), is(Arrays.asList("PASSWORD"))); + MatcherAssert.assertThat(reportAction.getWarningsOutput(), is("archiveArtifacts(delegate: @archiveArtifacts(=${PASSWORD}))\n interpolated variable(s): [PASSWORD]")); } @Test public void multipleSensitiveVariables() throws Exception { @@ -504,21 +492,15 @@ public void namedSoleParamForStep() throws Exception { + "}", true)); WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); r.assertLogContains("Warning: A secret was passed to \""+ shellStep + "\"", run); - r.assertLogContains("Affected argument(s) used the following variable(s): {script=[PASSWORD, USERNAME]}", run); + r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD, USERNAME]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - List> reportResults = reportAction.getWarnings(); - MatcherAssert.assertThat(reportResults.size(), is(1)); - List warning = reportResults.get(0); - MatcherAssert.assertThat(warning.get(0), is(shellStep)); - Map args = (Map)warning.get(1); - MatcherAssert.assertThat(args.size(), is(1)); - MatcherAssert.assertThat(args.get("script"), is(Arrays.asList("PASSWORD", "USERNAME"))); + MatcherAssert.assertThat(reportAction.getWarningsOutput(), is(shellStep + "(script: echo ${PASSWORD} ${USERNAME} ${PASSWORD})\n interpolated variable(s): [PASSWORD, USERNAME]")); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(shellStep)); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); Assert.assertFalse(argAction.isUnmodifiedArguments()); - MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), instanceOf(ArgumentsAction.NotStoredReason.class)); + MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), is("echo ${PASSWORD} ${USERNAME} ${PASSWORD}")); } @Test public void describableNoMetaStep() throws Exception { @@ -536,7 +518,17 @@ public void namedSoleParamForStep() throws Exception { WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); r.assertLogContains("First arg: ****, second arg: two", run); r.assertLogContains("Warning: A secret was passed to \"monomorphWithSymbolStep\"", run); - r.assertLogContains("Affected argument(s) used the following variable(s): {firstArg=[PASSWORD]}", run); + r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD]", run); + InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); + Assert.assertNotNull(reportAction); + MatcherAssert.assertThat(reportAction.getWarningsOutput(), is("monomorphWithSymbolStep(data: @monomorphSymbol(secondArg=two,firstArg=${PASSWORD}))\n interpolated variable(s): [PASSWORD]")); + LinearScanner scan = new LinearScanner(); + FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("monomorphWithSymbolStep")); + ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); + Assert.assertFalse(argAction.isUnmodifiedArguments()); + Object var = argAction.getArguments().values().iterator().next(); + MatcherAssert.assertThat(var, instanceOf(UninstantiatedDescribable.class)); + MatcherAssert.assertThat(((UninstantiatedDescribable)var).getArguments().toString(), is("{secondArg=two, firstArg=${PASSWORD}}")); } @Test public void noBodyError() throws Exception { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java index 47aa482fc..f203b429a 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java @@ -159,9 +159,9 @@ public void testStringSafetyTest() throws Exception { passwordBinding.put("mypass", "p4ssw0rd"); Set sensitiveVariables = new HashSet<>(); sensitiveVariables.add("mypass"); - MatcherAssert.assertThat("Input with no variables is safe", ArgumentsActionImpl.getAffectedVariables(input, new EnvVars(), sensitiveVariables).isEmpty()); - MatcherAssert.assertThat("Input containing bound value is unsafe", Arrays.asList("mypass").equals(ArgumentsActionImpl.getAffectedVariables(input, new EnvVars(passwordBinding), sensitiveVariables))); - MatcherAssert.assertThat("EnvVars that do not occur are safe", ArgumentsActionImpl.getAffectedVariables("I have no passwords", new EnvVars(passwordBinding), sensitiveVariables).isEmpty()); + MatcherAssert.assertThat("Input with no variables is safe", ArgumentsActionImpl.replaceSensitiveVariables(input, new EnvVars(), sensitiveVariables), is(input)); + MatcherAssert.assertThat("Input containing bound value is unsafe", ArgumentsActionImpl.replaceSensitiveVariables(input, new EnvVars(passwordBinding), sensitiveVariables), is("I have a secret ${mypass}")); + MatcherAssert.assertThat("EnvVars that do not occur are safe", ArgumentsActionImpl.replaceSensitiveVariables("I have no passwords", new EnvVars(passwordBinding), sensitiveVariables), is("I have no passwords")); } @Test @@ -181,7 +181,7 @@ public void testRecursiveSanitizationOfContent() { String oversizedString = new String (oversized); // Simplest masking of secret and oversized value - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, impl.sanitizeObjectAndRecordMutation(secretUsername, env)); + Assert.assertEquals("${USERVARIABLE}", impl.sanitizeObjectAndRecordMutation(secretUsername, env)); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; @@ -193,12 +193,12 @@ public void testRecursiveSanitizationOfContent() { Step mystep = new EchoStep("I have a "+secretUsername); Map singleSanitization = (Map)(impl.sanitizeObjectAndRecordMutation(mystep, env)); Assert.assertEquals(1, singleSanitization.size()); - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, singleSanitization.get("message")); + Assert.assertEquals("I have a ${USERVARIABLE}", singleSanitization.get("message")); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; singleSanitization = ((UninstantiatedDescribable) (impl.sanitizeObjectAndRecordMutation(mystep.getDescriptor().uninstantiate(mystep), env))).getArguments(); Assert.assertEquals(1, singleSanitization.size()); - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, singleSanitization.get("message")); + Assert.assertEquals("I have a ${USERVARIABLE}", singleSanitization.get("message")); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; @@ -207,7 +207,7 @@ public void testRecursiveSanitizationOfContent() { dangerous.put("name", secretUsername); Map sanitizedMap = impl.sanitizeMapAndRecordMutation(dangerous, env); Assert.assertNotEquals(sanitizedMap, dangerous); - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, sanitizedMap.get("name")); + Assert.assertEquals("${USERVARIABLE}", sanitizedMap.get("name")); Assert.assertFalse(impl.isUnmodifiedArguments()); impl.isUnmodifiedBySanitization = true; @@ -220,7 +220,7 @@ public void testRecursiveSanitizationOfContent() { List sanitized = (List)impl.sanitizeListAndRecordMutation(unsanitizedList, env); Assert.assertEquals(3, sanitized.size()); Assert.assertFalse(impl.isUnmodifiedArguments()); - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, sanitized.get(2)); + Assert.assertEquals("${USERVARIABLE}", sanitized.get(2)); impl.isUnmodifiedBySanitization = true; Assert.assertEquals(unsanitizedList, impl.sanitizeObjectAndRecordMutation(unsanitizedList, new EnvVars())); @@ -245,7 +245,7 @@ public void testArraySanitization() { Assert.assertThat(filteredArgs, IsMapContaining.hasEntry("ints", ArgumentsAction.NotStoredReason.UNSERIALIZABLE)); Assert.assertThat(filteredArgs, IsMapContaining.hasKey("strings")); Object[] contents = (Object[])(filteredArgs.get("strings")); - Assert.assertArrayEquals(new Object[]{"heh", ArgumentsAction.NotStoredReason.MASKED_VALUE, "lumberjack"}, (Object[])(filteredArgs.get("strings"))); + Assert.assertArrayEquals(new Object[]{"heh", "${USERVARIABLE}", "lumberjack"}, (Object[])(filteredArgs.get("strings"))); } @Test @@ -296,9 +296,9 @@ public void testBasicCreateAndMask() throws Exception { // Test sanitizing arguments now argumentsActionImpl = new ArgumentsActionImpl(arguments, new EnvVars(passwordBinding), sensitiveVariables); Assert.assertFalse(argumentsActionImpl.isUnmodifiedArguments()); - Assert.assertEquals(ArgumentsActionImpl.NotStoredReason.MASKED_VALUE, argumentsActionImpl.getArgumentValueOrReason("message")); + Assert.assertEquals("I have a secret ${mypass}", argumentsActionImpl.getArgumentValueOrReason("message")); Assert.assertEquals(1, argumentsActionImpl.getArguments().size()); - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, argumentsActionImpl.getArguments().get("message")); + Assert.assertEquals("I have a secret ${mypass}", argumentsActionImpl.getArguments().get("message")); // Mask oversized values arguments.clear(); @@ -425,8 +425,21 @@ public void testBasicCredentials() throws Exception { filtered = scanner.filteredNodes(exec, new DescriptorMatchPredicate(EchoStep.DescriptorImpl.class)); for (FlowNode f : filtered) { act = f.getPersistentAction(ArgumentsActionImpl.class); - Assert.assertEquals(ArgumentsAction.NotStoredReason.MASKED_VALUE, act.getArguments().get("message")); - Assert.assertNull(ArgumentsAction.getStepArgumentsAsString(f)); + String id = f.getId(); + switch (id) { + case "7": + Assert.assertEquals("${PASSWORD}'", act.getArguments().get("message")); + break; + case "8": + case "9": + Assert.assertEquals("${USERNAME}", act.getArguments().get("message")); + break; + case "16": + Assert.assertEquals("${USERNAME} ${PASSWORD}", act.getArguments().get("message")); + break; + default: + Assert.fail(); + } } List allStepped = scanner.filteredNodes(run.getExecution().getCurrentHeads(), FlowScanningUtils.hasActionPredicate(ArgumentsActionImpl.class)); From 4f9997debdfc6ddb2016ba498745d1447f2d20f5 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 1 Oct 2020 13:48:04 -0600 Subject: [PATCH 47/76] fix jelly output --- .../workflow/cps/view/InterpolatedSecretsAction.java | 7 ++++++- .../cps/view/InterpolatedSecretsAction/summary.jelly | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index d4f356149..679b82957 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -84,7 +84,7 @@ public void onLoad(Run run) { } @ExportedBean - static class InterpolatedWarnings implements Serializable { + public static class InterpolatedWarnings implements Serializable { final String stepName; final Map stepArguments; final List interpolatedVariables; @@ -106,6 +106,11 @@ public String getStepSignature() { return sb.toString(); } + @Exported + public List getInterpolatedVariables() { + return interpolatedVariables; + } + @Override public String toString() { return getStepSignature(); diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index 33d29012a..5b2094772 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -30,9 +30,9 @@ THE SOFTWARE.
          -
        • ${warning}
          • +
        • ${warning.stepSignature}
            • -
          • ${warning.interpolatedVariables}
            • +
          • interpolated variables: ${warning.interpolatedVariables}
        From 850621464818983bcd4412625135cc55c2c1d86c Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 1 Oct 2020 14:29:41 -0600 Subject: [PATCH 48/76] fix unit tests, some clean up --- .../cps/view/InterpolatedSecretsAction.java | 21 ---------------- .../plugins/workflow/cps/DSLTest.java | 25 ++++++++++++++++--- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 679b82957..a2a0c9551 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -41,22 +41,6 @@ public void record(@Nonnull String stepName, @Nonnull Map stepAr interpolatedWarnings.add(new InterpolatedWarnings(stepName, stepArguments, interpolatedVariables)); } - public List getWarningsOutput() { - List outputList = new ArrayList<>(); - StringBuilder sb = new StringBuilder(); - - for (InterpolatedWarnings warning : interpolatedWarnings) { - sb.append(warning.stepName + "("); - for (Map.Entry argEntry : warning.stepArguments.entrySet()) { - sb.append(argEntry.getKey() + ": " + argEntry.getValue().toString()); - } - sb.append(")\n"); - sb.append("interpolated variable(s): " + warning.interpolatedVariables.toString()); - } - outputList.add(sb.toString()); - return outputList; - } - public List getWarnings() { return interpolatedWarnings; } @@ -110,10 +94,5 @@ public String getStepSignature() { public List getInterpolatedVariables() { return interpolatedVariables; } - - @Override - public String toString() { - return getStepSignature(); - } } } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 97a4ab9d5..8a738fa08 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -33,6 +33,7 @@ import hudson.model.Descriptor; import hudson.model.Result; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -449,7 +450,11 @@ public void namedSoleParamForStep() throws Exception { r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - MatcherAssert.assertThat(reportAction.getWarningsOutput(), is(shellStep + "(script: echo ${PASSWORD})\n interpolated variable(s): [PASSWORD]")); + List warnings = reportAction.getWarnings(); + MatcherAssert.assertThat(warnings.size(), is(1)); + InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is(shellStep + "(script: echo ${PASSWORD})")); + MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(shellStep)); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); @@ -474,7 +479,11 @@ public void namedSoleParamForStep() throws Exception { r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - MatcherAssert.assertThat(reportAction.getWarningsOutput(), is("archiveArtifacts(delegate: @archiveArtifacts(=${PASSWORD}))\n interpolated variable(s): [PASSWORD]")); + List warnings = reportAction.getWarnings(); + MatcherAssert.assertThat(warnings.size(), is(1)); + InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("archiveArtifacts(delegate: @archiveArtifacts(=${PASSWORD}))")); + MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); } @Test public void multipleSensitiveVariables() throws Exception { @@ -495,7 +504,11 @@ public void namedSoleParamForStep() throws Exception { r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD, USERNAME]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - MatcherAssert.assertThat(reportAction.getWarningsOutput(), is(shellStep + "(script: echo ${PASSWORD} ${USERNAME} ${PASSWORD})\n interpolated variable(s): [PASSWORD, USERNAME]")); + List warnings = reportAction.getWarnings(); + MatcherAssert.assertThat(warnings.size(), is(1)); + InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is(shellStep + "(script: echo ${PASSWORD} ${USERNAME} ${PASSWORD})")); + MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD", "USERNAME"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(shellStep)); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); @@ -521,7 +534,11 @@ public void namedSoleParamForStep() throws Exception { r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD]", run); InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); Assert.assertNotNull(reportAction); - MatcherAssert.assertThat(reportAction.getWarningsOutput(), is("monomorphWithSymbolStep(data: @monomorphSymbol(secondArg=two,firstArg=${PASSWORD}))\n interpolated variable(s): [PASSWORD]")); + List warnings = reportAction.getWarnings(); + MatcherAssert.assertThat(warnings.size(), is(1)); + InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(secondArg=two,firstArg=${PASSWORD}))")); + MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("monomorphWithSymbolStep")); ArgumentsAction argAction = node.getPersistentAction(ArgumentsAction.class); From e6284e9db84cec3c0af9314667fd53b7b26015f0 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 1 Oct 2020 17:34:49 -0600 Subject: [PATCH 49/76] add redirect to console warning --- src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index a008a55ae..06eeb0687 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -363,7 +363,7 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc .collect(Collectors.toList()); if (scanResults != null && !scanResults.isEmpty()) { - String warning = String.format("Warning: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure.%n\t\t Affected argument(s) used the following variable(s): %s%n\t\t See for details.", + String warning = String.format("Warning: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure.%n\t\t Affected argument(s) used the following variable(s): %s%n\t\t See https://jenkins.io/redirect/groovy-string-interpolation for details.", stepName, scanResults.toString()); listener.getLogger().println(warning); From ef157e94dd4582ae7bff9aa9bae27737196efb47 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 8 Oct 2020 11:22:48 -0600 Subject: [PATCH 50/76] address review comments --- .../jenkinsci/plugins/workflow/cps/DSL.java | 36 ++++---- .../cps/actions/ArgumentsActionImpl.java | 6 +- .../cps/view/InterpolatedSecretsAction.java | 84 ++++++++++++++----- .../InterpolatedSecretsAction/summary.jelly | 2 +- 4 files changed, 89 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 06eeb0687..f64ac40c6 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -282,7 +282,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { ClassLoader originalLoader = Thread.currentThread().getContextClassLoader(); try { TaskListener listener = context.get(TaskListener.class); - logInterpolationWarnings(name, argumentsAction, ps.interpolatedStrings, allEnv, sensitiveVariables, listener); + logInterpolationWarnings(name, argumentsAction, an.getId(), ps.interpolatedStrings, allEnv, sensitiveVariables, listener); if (unreportedAmbiguousFunctions.remove(name)) { reportAmbiguousStepInvocation(context, d, listener); } @@ -353,7 +353,7 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } } - private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsActionImpl argumentsAction, Set interpolatedStrings, @CheckForNull EnvVars envVars, @Nonnull Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { + private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsActionImpl argumentsAction, String nodeId, Set interpolatedStrings, @CheckForNull EnvVars envVars, @Nonnull Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { if (argumentsAction == null || interpolatedStrings.isEmpty() || envVars == null || envVars.isEmpty() || sensitiveVariables.isEmpty()) { return; } @@ -374,7 +374,7 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc runReport = new InterpolatedSecretsAction(); ((Run) owner.getExecutable()).addAction(runReport); } - runReport.record(stepName, argumentsAction.getArguments(), scanResults); + runReport.record(stepName, scanResults, nodeId); } else { LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); } @@ -505,13 +505,13 @@ private static int preallocatedHashmapCapacity(int elementsToHold) { /** * This class holds the argument map and optional body of the step that is to be invoked. - * The UninstantiatedDescribable field is set when the associated symbol is being built as the parameter of a step to be invoked. */ static class NamedArgsAndClosure implements Serializable { final Map namedArgs; final Closure body; final List msgs; final Set interpolatedStrings; + // UninstantiatedDescribable is set when the associated symbol is being built as the parameter of a step to be invoked. UninstantiatedDescribable uninstantiatedDescribable = null; private NamedArgsAndClosure(Map namedArgs, Closure body, @Nonnull Set foundInterpolatedStrings) { @@ -577,13 +577,6 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { Set interpolatedStrings = new HashSet<>(); if (arg instanceof NamedArgsAndClosure) { interpolatedStrings = ((NamedArgsAndClosure) arg).interpolatedStrings; - } else if (arg instanceof Object[]) { - Object[] array = (Object[]) arg; - for (Object o : array) { - if (o instanceof NamedArgsAndClosure) { - interpolatedStrings.addAll(((NamedArgsAndClosure) o).interpolatedStrings); - } - } } boolean singleArgumentOnly = false; @@ -631,17 +624,28 @@ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String so * The collection of interpolated Groovy strings. */ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nonnull Set interpolatedStrings) { - if (arg instanceof NamedArgsAndClosure) + if (arg instanceof NamedArgsAndClosure) { + interpolatedStrings.addAll(((NamedArgsAndClosure) arg).interpolatedStrings); return (NamedArgsAndClosure) arg; - if (arg instanceof Map) // TODO is this clause actually used? + } + if (arg instanceof Map) { // TODO is this clause actually used? return new NamedArgsAndClosure((Map) arg, null, interpolatedStrings); - if (arg instanceof Closure && expectsBlock) + } + if (arg instanceof Closure && expectsBlock) { return new NamedArgsAndClosure(Collections.emptyMap(),(Closure)arg, interpolatedStrings); + } if (arg instanceof Object[]) {// this is how Groovy appears to pack argument list into one Object for invokeMethod List a = Arrays.asList((Object[])arg); - if (a.size()==0) - return new NamedArgsAndClosure(Collections.emptyMap(),null, interpolatedStrings); + if (a.size()==0) { + return new NamedArgsAndClosure(Collections.emptyMap(), null, interpolatedStrings); + } else { + for (Object o : a) { + if (o instanceof NamedArgsAndClosure) { + interpolatedStrings.addAll(((NamedArgsAndClosure) o).interpolatedStrings); + } + } + } Closure c=null; diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java index c61461da0..f8f7b6bbf 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java @@ -46,13 +46,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Implements {@link ArgumentsAction} by storing step arguments, with sanitization. @@ -71,7 +71,7 @@ public class ArgumentsActionImpl extends ArgumentsAction { private static final Logger LOGGER = Logger.getLogger(ArgumentsActionImpl.class.getName()); public ArgumentsActionImpl(@Nonnull Map stepArguments, @CheckForNull EnvVars env, @Nonnull Set sensitiveVariables) { - this.sensitiveVariables = sensitiveVariables; + this.sensitiveVariables = new HashSet<>(sensitiveVariables); arguments = serializationCheck(sanitizeMapAndRecordMutation(stepArguments, env)); } @@ -296,7 +296,7 @@ Map sanitizeMapAndRecordMutation(@Nonnull Map map boolean isMutated = false; for (Map.Entry param : mapContents.entrySet()) { - Object modded = sanitizeObjectAndRecordMutation(param.getValue(), variables);//, sanitizedArgumentVariables); + Object modded = sanitizeObjectAndRecordMutation(param.getValue(), variables); if (modded != param.getValue()) { // Sanitization stripped out some values, so we need to store the mutated object output.put(param.getKey(), modded); diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index a2a0c9551..737d5a05a 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -2,13 +2,16 @@ import hudson.model.Run; import jenkins.model.RunAction2; +import org.jenkinsci.plugins.workflow.actions.ArgumentsAction; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; +import org.jenkinsci.plugins.workflow.graph.FlowNode; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.export.Exported; import org.kohsuke.stapler.export.ExportedBean; import javax.annotation.Nonnull; -import java.io.Serializable; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -19,7 +22,7 @@ @Restricted(NoExternalUse.class) public class InterpolatedSecretsAction implements RunAction2 { - private List interpolatedWarnings; + private List interpolatedWarnings = new ArrayList<>(); private transient Run run; public String getIconFileName() { @@ -34,26 +37,23 @@ public String getUrlName() { return null; } - public void record(@Nonnull String stepName, @Nonnull Map stepArguments, @Nonnull List interpolatedVariables) { - if (interpolatedWarnings == null) { - interpolatedWarnings = new ArrayList<>(); - } - interpolatedWarnings.add(new InterpolatedWarnings(stepName, stepArguments, interpolatedVariables)); + public void record(@Nonnull String stepName, @Nonnull List interpolatedVariables, @Nonnull String nodeId) { + interpolatedWarnings.add(new InterpolatedWarnings(stepName, interpolatedVariables, run, nodeId)); } public List getWarnings() { return interpolatedWarnings; } - public boolean getHasWarnings() { - if (interpolatedWarnings == null || interpolatedWarnings.isEmpty()) { + public boolean hasWarnings() { + if (interpolatedWarnings.isEmpty()) { return false; } else { return true; } } - public boolean getInProgress() { + public boolean isInProgress() { return run.isBuilding(); } @@ -68,26 +68,72 @@ public void onLoad(Run run) { } @ExportedBean - public static class InterpolatedWarnings implements Serializable { + public static class InterpolatedWarnings { final String stepName; - final Map stepArguments; final List interpolatedVariables; + final Run run; + final String nodeId; - private InterpolatedWarnings(@Nonnull String stepName, @Nonnull Map stepArguments, @Nonnull List interpolatedVariables) { + private InterpolatedWarnings(@Nonnull String stepName, @Nonnull List interpolatedVariables, @Nonnull Run run, @Nonnull String nodeId) { this.stepName = stepName; - this.stepArguments = stepArguments; this.interpolatedVariables = interpolatedVariables; + this.run = run; + this.nodeId = nodeId; } @Exported public String getStepSignature() { StringBuilder sb = new StringBuilder(); - sb.append(stepName + "("); - for (Map.Entry argEntry : stepArguments.entrySet()) { - sb.append(argEntry.getKey() + ": " + argEntry.getValue().toString()); + String failReason = null; + if (run instanceof FlowExecutionOwner.Executable) { + try { + FlowExecutionOwner owner = ((FlowExecutionOwner.Executable) run).asFlowExecutionOwner(); + if (owner != null) { + FlowNode node = owner.get().getNode(nodeId); + if (node != null) { + ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); + if (argumentsAction != null) { + Map stepArguments = argumentsAction.getArguments(); + sb.append(stepName + "("); + for (Map.Entry argEntry : stepArguments.entrySet()) { + Object value = argEntry.getValue(); + String valueString = String.valueOf(value); + if (value instanceof ArgumentsAction.NotStoredReason) { + switch ((ArgumentsAction.NotStoredReason) value) { + case OVERSIZE_VALUE: + valueString = "argument omitted due to length"; + break; + case UNSERIALIZABLE: + valueString = "unable to serialize argument"; + break; + default: + break; + } + } + sb.append(argEntry.getKey() + ": " + valueString); + } + sb.append(")"); + } else { + failReason = "null arguments action"; + } + } else { + failReason = "null flow node"; + } + } else { + failReason = "null flow execution owner"; + } + } catch (IOException e) { + failReason = "could not get flow node"; + } + } else { + failReason = "not an instance of FlowExecutionOwner.Executable"; + } + + if (failReason != null) { + return "Unable to construct " + stepName; + } else { + return sb.toString(); } - sb.append(")"); - return sb.toString(); } @Exported diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index 5b2094772..89aae7fd9 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -27,7 +27,7 @@ THE SOFTWARE. (${%in progress}) - +
        • ${warning.stepSignature}
          • From 00d220b6432cd65f86a24ae12245d1fe57d676f9 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 8 Oct 2020 12:16:02 -0600 Subject: [PATCH 51/76] simplify parseArgs --- .../java/org/jenkinsci/plugins/workflow/cps/DSL.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index f64ac40c6..63fe7233c 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -45,6 +45,7 @@ import java.lang.annotation.Annotation; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -574,11 +575,6 @@ private static Object flattenGString(Object v, @Nonnull Set interpolated } static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { - Set interpolatedStrings = new HashSet<>(); - if (arg instanceof NamedArgsAndClosure) { - interpolatedStrings = ((NamedArgsAndClosure) arg).interpolatedStrings; - } - boolean singleArgumentOnly = false; try { DescribableModel stepModel = DescribableModel.of(d.clazz); @@ -586,12 +582,12 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { if (singleArgumentOnly) { // Can fetch the one argument we need DescribableParameter dp = stepModel.getSoleRequiredParameter(); String paramName = (dp != null) ? dp.getName() : null; - return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, interpolatedStrings); + return parseArgs(arg, d.takesImplicitBlockArgument(), paramName, singleArgumentOnly, new HashSet<>()); } } catch (NoStaplerConstructorException e) { // Ignore steps without databound constructors and treat them as normal. } - return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, interpolatedStrings); + return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, new HashSet<>()); } static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg) { From a8df396d68dc4f00386db06297a6ec44f923723b Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 12 Oct 2020 10:12:42 -0600 Subject: [PATCH 52/76] centralize parsing of NamedArgsAndClosure --- .../jenkinsci/plugins/workflow/cps/DSL.java | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 63fe7233c..5362074f6 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -45,9 +45,7 @@ import java.lang.annotation.Annotation; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -422,6 +420,9 @@ protected Object invokeDescribable(String symbol, Object _args) { // also note that in this case 'd' is not trustworthy, as depending on // where this UninstantiatedDescribable is ultimately used, the symbol // might be resolved with a specific type. + + // we returning the NamedArgsAndClosure instead of the UninstantiatedDescribable in order to preserve + // the discovered interpolated strings. args.uninstantiatedDescribable = ud; return args; } else { @@ -521,12 +522,39 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nonnull Set(); this.interpolatedStrings = new HashSet<>(foundInterpolatedStrings); + namedArgs = (Map) collectInterpolatedStrings(namedArgs, interpolatedStrings); for (Map.Entry entry : namedArgs.entrySet()) { String k = entry.getKey().toString().intern(); // coerces GString and more Object v = flattenGString(entry.getValue(), interpolatedStrings); this.namedArgs.put(k, v); } } + + /** + * Recursively search argument values for instances of {@link NamedArgsAndClosure} and convert them to {@link UninstantiatedDescribable}. + * These instances were created in {@link DSL#invokeDescribable(String, Object)} for symbols with no meta-step. + * Gather all the interpolated strings from each instance of {@link NamedArgsAndClosure}. + */ + private static Object collectInterpolatedStrings(Object argValue, Set interpolatedStrings) { + if (argValue instanceof NamedArgsAndClosure) { + interpolatedStrings.addAll(((NamedArgsAndClosure) argValue).interpolatedStrings); + return ((NamedArgsAndClosure) argValue).uninstantiatedDescribable; + } else if (argValue instanceof Map) { + Map r = new LinkedHashMap<>(preallocatedHashmapCapacity(((Map) argValue).size())); + for (Map.Entry e : ((Map) argValue).entrySet()) { + r.put(e.getKey(), collectInterpolatedStrings(e.getValue(), interpolatedStrings)); + } + return r; + } else if (argValue instanceof List) { + List r = new ArrayList<>(); + for (int i = 0; i < ((List) argValue).size(); i++) { + r.add(collectInterpolatedStrings(((List) argValue).get(i), interpolatedStrings)); + } + return r; + } else { + return argValue; + } + } } /** @@ -566,9 +594,6 @@ private static Object flattenGString(Object v, @Nonnull Set interpolated r.put(k2, o2); } return mutated ? r : v; - } else if (v instanceof NamedArgsAndClosure) { - UninstantiatedDescribable ud = ((NamedArgsAndClosure) v).uninstantiatedDescribable; - return ud != null? ud : v; } else { return v; } @@ -621,7 +646,6 @@ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String so */ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg, @Nonnull Set interpolatedStrings) { if (arg instanceof NamedArgsAndClosure) { - interpolatedStrings.addAll(((NamedArgsAndClosure) arg).interpolatedStrings); return (NamedArgsAndClosure) arg; } if (arg instanceof Map) { // TODO is this clause actually used? @@ -635,12 +659,6 @@ static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String so List a = Arrays.asList((Object[])arg); if (a.size()==0) { return new NamedArgsAndClosure(Collections.emptyMap(), null, interpolatedStrings); - } else { - for (Object o : a) { - if (o instanceof NamedArgsAndClosure) { - interpolatedStrings.addAll(((NamedArgsAndClosure) o).interpolatedStrings); - } - } } Closure c=null; From 7c8022b5eec0e7ecad65341b3fac3b4c05fbae2b Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 12 Oct 2020 16:02:50 -0600 Subject: [PATCH 53/76] Sort arguments in step signature --- .../cps/view/InterpolatedSecretsAction.java | 74 ++++++++++++------- .../plugins/workflow/cps/DSLTest.java | 40 +++++++++- .../cps/actions/ArgumentsActionImplTest.java | 16 +--- 3 files changed, 85 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 737d5a05a..736d23688 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -13,8 +13,11 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; /** * Action to generate the UI report for watched environment variables @@ -84,7 +87,48 @@ private InterpolatedWarnings(@Nonnull String stepName, @Nonnull List int @Exported public String getStepSignature() { StringBuilder sb = new StringBuilder(); - String failReason = null; + Map stepArguments; + try { + stepArguments = getStepArguments(run, nodeId); + } catch (IllegalStateException e) { + return "Unable to construct " + stepName + ": " + e.getMessage(); + } + + sb.append(stepName + "("); + Set> entrySet = stepArguments.entrySet(); + if (!entrySet.isEmpty()) { + // Give some sort of order to the step arguments + TreeSet sortedArgs = new TreeSet<>(); + for (Map.Entry argEntry : stepArguments.entrySet()) { + Object value = argEntry.getValue(); + String valueString = String.valueOf(value); + if (value instanceof ArgumentsAction.NotStoredReason) { + switch ((ArgumentsAction.NotStoredReason) value) { + case OVERSIZE_VALUE: + valueString = "argument omitted due to length"; + break; + case UNSERIALIZABLE: + valueString = "unable to serialize argument"; + break; + default: + break; + } + } + sortedArgs.add(argEntry.getKey() + ": " + valueString); + } + Iterator it = sortedArgs.iterator(); + sb.append(it.next()); + while (it.hasNext()) { + sb.append(", " + it.next()); + } + } + sb.append(")"); + return sb.toString(); + } + + @Nonnull + private Map getStepArguments(Run run, String nodeId) throws IllegalStateException { + String failReason; if (run instanceof FlowExecutionOwner.Executable) { try { FlowExecutionOwner owner = ((FlowExecutionOwner.Executable) run).asFlowExecutionOwner(); @@ -93,26 +137,7 @@ public String getStepSignature() { if (node != null) { ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); if (argumentsAction != null) { - Map stepArguments = argumentsAction.getArguments(); - sb.append(stepName + "("); - for (Map.Entry argEntry : stepArguments.entrySet()) { - Object value = argEntry.getValue(); - String valueString = String.valueOf(value); - if (value instanceof ArgumentsAction.NotStoredReason) { - switch ((ArgumentsAction.NotStoredReason) value) { - case OVERSIZE_VALUE: - valueString = "argument omitted due to length"; - break; - case UNSERIALIZABLE: - valueString = "unable to serialize argument"; - break; - default: - break; - } - } - sb.append(argEntry.getKey() + ": " + valueString); - } - sb.append(")"); + return argumentsAction.getArguments(); } else { failReason = "null arguments action"; } @@ -128,12 +153,7 @@ public String getStepSignature() { } else { failReason = "not an instance of FlowExecutionOwner.Executable"; } - - if (failReason != null) { - return "Unable to construct " + stepName; - } else { - return sb.toString(); - } + throw new IllegalStateException(failReason); } @Exported diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 8a738fa08..d3b6680aa 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -59,6 +59,7 @@ import org.jenkinsci.plugins.workflow.testMetaStep.AmbiguousEchoLowerStep; import org.jenkinsci.plugins.workflow.testMetaStep.AmbiguousEchoUpperStep; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.junit.Assert.*; @@ -432,7 +433,8 @@ public void namedSoleParamForStep() throws Exception { "'org.jenkinsci.plugins.workflow.steps.SleepStep': comment,units", b); } - @Test public void sensitiveVarsLogging() throws Exception { + @Issue("JENKINS-63254") + @Test public void sensitiveVariableInterpolation() throws Exception { final String credentialsId = "creds"; final String username = "bob"; final String password = "secr3t"; @@ -462,7 +464,8 @@ public void namedSoleParamForStep() throws Exception { MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), is("echo ${PASSWORD}")); } - @Test public void describableInterpolation() throws Exception { + @Issue("JENKINS-63254") + @Test public void sensitiveVariableInterpolationWithMetaStep() throws Exception { final String credentialsId = "creds"; final String username = "bob"; final String password = "secr3t"; @@ -516,7 +519,8 @@ public void namedSoleParamForStep() throws Exception { MatcherAssert.assertThat(argAction.getArguments().values().iterator().next(), is("echo ${PASSWORD} ${USERNAME} ${PASSWORD}")); } - @Test public void describableNoMetaStep() throws Exception { + @Issue("JENKINS-63254") + @Test public void sensitiveVariableInterpolationWithNestedDescribable() throws Exception { final String credentialsId = "creds"; final String username = "bob"; final String password = "secr3t"; @@ -548,6 +552,36 @@ public void namedSoleParamForStep() throws Exception { MatcherAssert.assertThat(((UninstantiatedDescribable)var).getArguments().toString(), is("{secondArg=two, firstArg=${PASSWORD}}")); } + @Issue("JENKINS-63254") + @Test public void complexSensitiveVariableInterpolationWithNestedDescribable() throws Exception { + final String credentialsId = "creds"; + final String username = "bob"; + final String password = "secr3t"; + UsernamePasswordCredentialsImpl c = new UsernamePasswordCredentialsImpl(CredentialsScope.GLOBAL, credentialsId, "sample", username, password); + CredentialsProvider.lookupStores(r.jenkins).iterator().next().addCredentials(Domain.global(), c); + p.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + "withCredentials([usernamePassword(credentialsId: 'creds', usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {\n" + + "monomorphListSymbolStep([monomorphSymbol(firstArg: monomorphWithSymbolStep(monomorphSymbol([firstArg: \"innerFirstArgIs${PASSWORD}\", secondArg: \"innerSecondArgIs${USERNAME}\"])), secondArg: \"hereismy${PASSWORD}\"), monomorphSymbol(firstArg: \"${PASSWORD}\", secondArg: \"${USERNAME}\")])" + + "}\n" + + "}", true)); + WorkflowRun run = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + r.assertLogContains("Warning: A secret was passed to \"monomorphWithSymbolStep\"", run); + r.assertLogContains("Affected argument(s) used the following variable(s): [PASSWORD, USERNAME]", run); + r.assertLogContains("Warning: A secret was passed to \"monomorphListSymbolStep\"", run); + r.assertLogNotContains("Affected argument(s) used the following variable(s): [PASSWORD]", run); + InterpolatedSecretsAction reportAction = run.getAction(InterpolatedSecretsAction.class); + Assert.assertNotNull(reportAction); + List warnings = reportAction.getWarnings(); + MatcherAssert.assertThat(warnings.size(), is(2)); + InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(secondArg=innerSecondArgIs${USERNAME},firstArg=innerFirstArgIs${PASSWORD}))")); + MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); + InterpolatedSecretsAction.InterpolatedWarnings listStepWarning = warnings.get(1); + MatcherAssert.assertThat(listStepWarning.getStepSignature(), is("monomorphListSymbolStep(data: [@monomorphSymbol(secondArg=hereismy${PASSWORD},firstArg=null), @monomorphSymbol(secondArg=${USERNAME},firstArg=${PASSWORD})])")); + MatcherAssert.assertThat(listStepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); + } + @Test public void noBodyError() throws Exception { p.setDefinition((new CpsFlowDefinition("timeout(time: 1, unit: 'SECONDS')", true))); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java index f203b429a..692079725 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java @@ -425,21 +425,7 @@ public void testBasicCredentials() throws Exception { filtered = scanner.filteredNodes(exec, new DescriptorMatchPredicate(EchoStep.DescriptorImpl.class)); for (FlowNode f : filtered) { act = f.getPersistentAction(ArgumentsActionImpl.class); - String id = f.getId(); - switch (id) { - case "7": - Assert.assertEquals("${PASSWORD}'", act.getArguments().get("message")); - break; - case "8": - case "9": - Assert.assertEquals("${USERNAME}", act.getArguments().get("message")); - break; - case "16": - Assert.assertEquals("${USERNAME} ${PASSWORD}", act.getArguments().get("message")); - break; - default: - Assert.fail(); - } + MatcherAssert.assertThat((String) act.getArguments().get("message"), allOf(not(containsString("bob")), not(containsString("s3cr3t")))); } List allStepped = scanner.filteredNodes(run.getExecution().getCurrentHeads(), FlowScanningUtils.hasActionPredicate(ArgumentsActionImpl.class)); From 36050b44716f113788e2af426e209c13ed0b65ff Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 12 Oct 2020 23:54:36 -0600 Subject: [PATCH 54/76] fix comments --- src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 5362074f6..cf05d0b74 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -421,8 +421,8 @@ protected Object invokeDescribable(String symbol, Object _args) { // where this UninstantiatedDescribable is ultimately used, the symbol // might be resolved with a specific type. - // we returning the NamedArgsAndClosure instead of the UninstantiatedDescribable in order to preserve - // the discovered interpolated strings. + // we are returning the NamedArgsAndClosure instead of the UninstantiatedDescribable in order to preserve + // the discovered interpolated strings that are stored in the NamedArgsAndClosure. args.uninstantiatedDescribable = ud; return args; } else { From 163e7cb7452204f9270783c0925f0f5c0113caae Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 14 Oct 2020 12:35:22 -0600 Subject: [PATCH 55/76] make step arguments print out in order they were added --- .../workflow/cps/actions/ArgumentsActionImpl.java | 3 ++- .../cps/view/InterpolatedSecretsAction.java | 15 +++++++-------- .../jenkinsci/plugins/workflow/cps/DSLTest.java | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java index f8f7b6bbf..9ba61ee36 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImpl.java @@ -47,6 +47,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -292,7 +293,7 @@ Map serializationCheck(@Nonnull Map arguments) { @Nonnull Map sanitizeMapAndRecordMutation(@Nonnull Map mapContents, @CheckForNull EnvVars variables) { // Package scoped so we can test it directly - HashMap output = Maps.newHashMapWithExpectedSize(mapContents.size()); + LinkedHashMap output = new LinkedHashMap<>(mapContents.size()); boolean isMutated = false; for (Map.Entry param : mapContents.entrySet()) { diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 736d23688..aaf2b5574 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -97,8 +97,7 @@ public String getStepSignature() { sb.append(stepName + "("); Set> entrySet = stepArguments.entrySet(); if (!entrySet.isEmpty()) { - // Give some sort of order to the step arguments - TreeSet sortedArgs = new TreeSet<>(); + boolean first = true; for (Map.Entry argEntry : stepArguments.entrySet()) { Object value = argEntry.getValue(); String valueString = String.valueOf(value); @@ -114,12 +113,12 @@ public String getStepSignature() { break; } } - sortedArgs.add(argEntry.getKey() + ": " + valueString); - } - Iterator it = sortedArgs.iterator(); - sb.append(it.next()); - while (it.hasNext()) { - sb.append(", " + it.next()); + if (first) { + first = false; + } else { + sb.append(", "); + } + sb.append(argEntry.getKey() + ": " + valueString); } } sb.append(")"); diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index d3b6680aa..ba366ee99 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -541,7 +541,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(secondArg=two,firstArg=${PASSWORD}))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(firstArg=${PASSWORD},secondArg=two))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("monomorphWithSymbolStep")); @@ -549,7 +549,7 @@ public void namedSoleParamForStep() throws Exception { Assert.assertFalse(argAction.isUnmodifiedArguments()); Object var = argAction.getArguments().values().iterator().next(); MatcherAssert.assertThat(var, instanceOf(UninstantiatedDescribable.class)); - MatcherAssert.assertThat(((UninstantiatedDescribable)var).getArguments().toString(), is("{secondArg=two, firstArg=${PASSWORD}}")); + MatcherAssert.assertThat(((UninstantiatedDescribable)var).getArguments().toString(), is("{firstArg=${PASSWORD}, secondArg=two}")); } @Issue("JENKINS-63254") @@ -575,10 +575,10 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(2)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(secondArg=innerSecondArgIs${USERNAME},firstArg=innerFirstArgIs${PASSWORD}))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(firstArg=innerFirstArgIs${PASSWORD},secondArg=innerSecondArgIs${USERNAME}))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); InterpolatedSecretsAction.InterpolatedWarnings listStepWarning = warnings.get(1); - MatcherAssert.assertThat(listStepWarning.getStepSignature(), is("monomorphListSymbolStep(data: [@monomorphSymbol(secondArg=hereismy${PASSWORD},firstArg=null), @monomorphSymbol(secondArg=${USERNAME},firstArg=${PASSWORD})])")); + MatcherAssert.assertThat(listStepWarning.getStepSignature(), is("monomorphListSymbolStep(data: [@monomorphSymbol(firstArg=null,secondArg=hereismy${PASSWORD}), @monomorphSymbol(firstArg=${PASSWORD},secondArg=${USERNAME})])")); MatcherAssert.assertThat(listStepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); } From 3740ed12f82faf404f87e97ea0b92e57a91c4b92 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Wed, 14 Oct 2020 22:57:57 -0600 Subject: [PATCH 56/76] update workflow-step-api and credentials-binding to release versions --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 77fb0fcce..7f7d27fc0 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 8 false 1.32 - 2.23-rc567.0fe52fbdf6b5 + 2.23 @@ -162,7 +162,7 @@ org.jenkins-ci.plugins credentials-binding - 1.24-rc375.06428a051632 + 1.24 test From a20db3d299a91e88a24076f791cf80ba02cb08de Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Sun, 18 Oct 2020 23:36:46 -0600 Subject: [PATCH 57/76] update bom --- pom.xml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index 7f7d27fc0..d3cf879ef 100644 --- a/pom.xml +++ b/pom.xml @@ -69,14 +69,13 @@ 8 false 1.32 - 2.23 io.jenkins.tools.bom bom-2.176.x - 13 + 14 import pom @@ -86,7 +85,6 @@ org.jenkins-ci.plugins.workflow workflow-step-api - ${workflow-step-api.version} org.jenkins-ci.plugins.workflow @@ -128,7 +126,6 @@ org.jenkins-ci.plugins.workflow workflow-step-api - ${workflow-step-api.version} tests test @@ -162,7 +159,6 @@ org.jenkins-ci.plugins credentials-binding - 1.24 test From b14f8cf2ce07227ade95c88d2346ced7b14ad544 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 19 Oct 2020 09:59:18 -0600 Subject: [PATCH 58/76] bump bom to v15 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d3cf879ef..c7edb9a5f 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,7 @@ io.jenkins.tools.bom bom-2.176.x - 14 + 15 import pom From d489f73f2d33ad7eecf0b99bbea2e1291d72a35f Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 19 Oct 2020 12:55:11 -0600 Subject: [PATCH 59/76] address review comments --- .../org/jenkinsci/plugins/workflow/cps/DSL.java | 14 +++++++------- .../cps/view/InterpolatedSecretsAction.java | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index cf05d0b74..54389c5ee 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -407,7 +407,7 @@ protected Object invokeDescribable(String symbol, Object _args) { // The only time a closure is valid is when the resulting Describable is immediately executed via a meta-step NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), - UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly); + UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, new HashSet<>()); UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); if (metaStep==null) { @@ -507,9 +507,13 @@ private static int preallocatedHashmapCapacity(int elementsToHold) { /** * This class holds the argument map and optional body of the step that is to be invoked. + * + *

          Some steps have complex argument types (e.g. `checkout` takes {@link hudson.scm.SCM}). When user use symbol-based + * syntax with those arguments, an instance of this class is created as the result of {@link DSL#invokeDescribable(String, Object)}. + * The instance is returned to the Groovy program so that it can be passed to {@link DSL#invokeStep(StepDescriptor, String, Object)} + * later, so it must implement {@link Serializable}. */ - static class NamedArgsAndClosure implements Serializable { - final Map namedArgs; + static class NamedArgsAndClosure implements Serializable { final Map namedArgs; final Closure body; final List msgs; final Set interpolatedStrings; @@ -615,10 +619,6 @@ static NamedArgsAndClosure parseArgs(Object arg, StepDescriptor d) { return parseArgs(arg,d.takesImplicitBlockArgument(), loadSoleArgumentKey(d), singleArgumentOnly, new HashSet<>()); } - static NamedArgsAndClosure parseArgs(Object arg, boolean expectsBlock, String soleArgumentKey, boolean singleRequiredArg) { - return parseArgs(arg, expectsBlock, soleArgumentKey, singleRequiredArg, new HashSet<>()); - } - /** * Given the Groovy style argument packing used in the sole object parameter of {@link GroovyObject#invokeMethod(String, Object)}, * compute the named argument map and an optional closure that represents the body. diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index aaf2b5574..e3712c07e 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -98,7 +98,7 @@ public String getStepSignature() { Set> entrySet = stepArguments.entrySet(); if (!entrySet.isEmpty()) { boolean first = true; - for (Map.Entry argEntry : stepArguments.entrySet()) { + for (Map.Entry argEntry : entrySet) { Object value = argEntry.getValue(); String valueString = String.valueOf(value); if (value instanceof ArgumentsAction.NotStoredReason) { From 11b12ba5c88ec52b17f2c2ef1b8009c25d29a099 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 20 Oct 2020 00:39:56 -0600 Subject: [PATCH 60/76] make getStepSignature recursive --- .../jenkinsci/plugins/workflow/cps/DSL.java | 2 +- .../cps/view/InterpolatedSecretsAction.java | 72 ++++++++++--------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 54389c5ee..1b2c7c043 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -537,7 +537,7 @@ private NamedArgsAndClosure(Map namedArgs, Closure body, @Nonnull Set interpolatedStrings) { if (argValue instanceof NamedArgsAndClosure) { diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index e3712c07e..da63d25a3 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -13,11 +13,9 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; +import java.util.stream.Collectors; /** * Action to generate the UI report for watched environment variables @@ -77,7 +75,7 @@ public static class InterpolatedWarnings { final Run run; final String nodeId; - private InterpolatedWarnings(@Nonnull String stepName, @Nonnull List interpolatedVariables, @Nonnull Run run, @Nonnull String nodeId) { + InterpolatedWarnings(@Nonnull String stepName, @Nonnull List interpolatedVariables, @Nonnull Run run, @Nonnull String nodeId) { this.stepName = stepName; this.interpolatedVariables = interpolatedVariables; this.run = run; @@ -86,7 +84,6 @@ private InterpolatedWarnings(@Nonnull String stepName, @Nonnull List int @Exported public String getStepSignature() { - StringBuilder sb = new StringBuilder(); Map stepArguments; try { stepArguments = getStepArguments(run, nodeId); @@ -94,35 +91,9 @@ public String getStepSignature() { return "Unable to construct " + stepName + ": " + e.getMessage(); } - sb.append(stepName + "("); - Set> entrySet = stepArguments.entrySet(); - if (!entrySet.isEmpty()) { - boolean first = true; - for (Map.Entry argEntry : entrySet) { - Object value = argEntry.getValue(); - String valueString = String.valueOf(value); - if (value instanceof ArgumentsAction.NotStoredReason) { - switch ((ArgumentsAction.NotStoredReason) value) { - case OVERSIZE_VALUE: - valueString = "argument omitted due to length"; - break; - case UNSERIALIZABLE: - valueString = "unable to serialize argument"; - break; - default: - break; - } - } - if (first) { - first = false; - } else { - sb.append(", "); - } - sb.append(argEntry.getKey() + ": " + valueString); - } - } - sb.append(")"); - return sb.toString(); + return stepArguments.entrySet().stream() + .map(InterpolatedSecretsAction::argumentToString) + .collect(Collectors.joining(", ", stepName + "(", ")")); } @Nonnull @@ -160,4 +131,37 @@ public List getInterpolatedVariables() { return interpolatedVariables; } } + + private static String argumentToString(Map.Entry argEntry) { + Object value = argEntry.getValue(); + String valueString; + if (value instanceof ArgumentsAction.NotStoredReason) { + switch ((ArgumentsAction.NotStoredReason) value) { + case OVERSIZE_VALUE: + valueString = "argument omitted due to length"; + break; + case UNSERIALIZABLE: + valueString = "unable to serialize argument"; + break; + default: + valueString = String.valueOf(value); + break; + } + } else if (value instanceof Map) { + valueString = mapToString((Map) value); + } else if (value instanceof List) { + valueString = ((List>) value).stream() + .map(InterpolatedSecretsAction::mapToString) + .collect(Collectors.joining(", ", "[", "]")); + } else { + valueString = String.valueOf(value); + } + return argEntry.getKey() + ": " + valueString; + } + + private static String mapToString(Map valueMap) { + return valueMap.entrySet().stream() + .map(InterpolatedSecretsAction::argumentToString) + .collect(Collectors.joining(", ", "[", "]")); + } } From bd0dfd285dee8361e9aceaaa2bc87ff1e2b6059e Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 20 Oct 2020 01:18:08 -0600 Subject: [PATCH 61/76] make recursion more generic, add unit test for getStepSignature() --- .../cps/view/InterpolatedSecretsAction.java | 51 +++++++++++-------- .../view/InterpolatedSecretsActionTest.java | 47 +++++++++++++++++ 2 files changed, 78 insertions(+), 20 deletions(-) create mode 100644 src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index da63d25a3..8a34ea31f 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -132,31 +132,42 @@ public List getInterpolatedVariables() { } } - private static String argumentToString(Map.Entry argEntry) { - Object value = argEntry.getValue(); + private static String argumentToString(Object arg) { String valueString; - if (value instanceof ArgumentsAction.NotStoredReason) { - switch ((ArgumentsAction.NotStoredReason) value) { - case OVERSIZE_VALUE: - valueString = "argument omitted due to length"; - break; - case UNSERIALIZABLE: - valueString = "unable to serialize argument"; - break; - default: - valueString = String.valueOf(value); - break; + if (arg instanceof Map.Entry) { + Map.Entry argEntry = (Map.Entry) arg; + Object value = argEntry.getValue(); + if (value instanceof ArgumentsAction.NotStoredReason) { + switch ((ArgumentsAction.NotStoredReason) value) { + case OVERSIZE_VALUE: + valueString = "argument omitted due to length"; + break; + case UNSERIALIZABLE: + valueString = "unable to serialize argument"; + break; + default: + valueString = String.valueOf(value); + break; + } + } else if (value instanceof Map || value instanceof List) { + valueString = argumentToString(value); + } else { + valueString = String.valueOf(value); } - } else if (value instanceof Map) { - valueString = mapToString((Map) value); - } else if (value instanceof List) { - valueString = ((List>) value).stream() - .map(InterpolatedSecretsAction::mapToString) + return argEntry.getKey() + ": " + valueString; + } else if (arg instanceof Map) { + valueString = ((Map) arg).entrySet().stream() + .map(InterpolatedSecretsAction::argumentToString) + .collect(Collectors.joining(", ", "[", "]")); + } else if (arg instanceof List) { + valueString = ((List) arg).stream() + .map(InterpolatedSecretsAction::argumentToString) .collect(Collectors.joining(", ", "[", "]")); } else { - valueString = String.valueOf(value); + valueString = String.valueOf(arg); } - return argEntry.getKey() + ": " + valueString; + + return valueString; } private static String mapToString(Map valueMap) { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java new file mode 100644 index 000000000..ff03a673a --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java @@ -0,0 +1,47 @@ +package org.jenkinsci.plugins.workflow.cps.view; + +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.graph.FlowNode; +import org.jenkinsci.plugins.workflow.graphanalysis.LinearScanner; +import org.jenkinsci.plugins.workflow.graphanalysis.NodeStepTypePredicate; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.BuildWatcher; +import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.LoggerRule; + +import java.util.Collections; + +public class InterpolatedSecretsActionTest { + @ClassRule + public static BuildWatcher buildWatcher = new BuildWatcher(); + + @ClassRule + public static JenkinsRule r = new JenkinsRule(); + + @Rule + public LoggerRule logging = new LoggerRule(); + + private WorkflowJob p; + @Before + public void newProject() throws Exception { + p = r.createProject(WorkflowJob.class); + } + + @Test + public void testStepSignature() throws Exception { + p.setDefinition(new CpsFlowDefinition("monomorphListStep([[firstArg:'one', secondArg:'two'], [firstArg:'three', secondArg:'four']])", true)); + WorkflowRun b = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); + LinearScanner scan = new LinearScanner(); + FlowNode node = scan.findFirstMatch(b.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("monomorphListStep")); + InterpolatedSecretsAction.InterpolatedWarnings warning = + new InterpolatedSecretsAction.InterpolatedWarnings("monomorphListStep", Collections.emptyList(), b, node.getId()); + MatcherAssert.assertThat(warning.getStepSignature(), Matchers.is("monomorphListStep(data: [[firstArg: one, secondArg: two], [firstArg: three, secondArg: four]])")); + } +} From 86cf9f3b961265edbeaef8243e10fc58ef4d1c78 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 20 Oct 2020 01:31:47 -0600 Subject: [PATCH 62/76] parse UninstantiatedDescribable in getStepSignature --- .../cps/view/InterpolatedSecretsAction.java | 17 ++++++++++++++++- .../jenkinsci/plugins/workflow/cps/DSLTest.java | 4 ++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 8a34ea31f..a521b143c 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -2,6 +2,7 @@ import hudson.model.Run; import jenkins.model.RunAction2; +import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable; import org.jenkinsci.plugins.workflow.actions.ArgumentsAction; import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; import org.jenkinsci.plugins.workflow.graph.FlowNode; @@ -149,7 +150,7 @@ private static String argumentToString(Object arg) { valueString = String.valueOf(value); break; } - } else if (value instanceof Map || value instanceof List) { + } else if (value instanceof Map || value instanceof List || value instanceof UninstantiatedDescribable) { valueString = argumentToString(value); } else { valueString = String.valueOf(value); @@ -163,6 +164,20 @@ private static String argumentToString(Object arg) { valueString = ((List) arg).stream() .map(InterpolatedSecretsAction::argumentToString) .collect(Collectors.joining(", ", "[", "]")); + } else if (arg instanceof UninstantiatedDescribable) { + UninstantiatedDescribable ud = (UninstantiatedDescribable) arg; + String udName = null; + if (ud.getSymbol() != null) { + udName = '@' + ud.getSymbol(); + } + if (ud.getKlass() != null) { + udName = '$' + ud.getKlass() + udName; + } + + Map udArgs = ud.getArguments(); + valueString = udArgs.entrySet().stream() + .map(InterpolatedSecretsAction::argumentToString) + .collect(Collectors.joining(", ", udName + "(", ")")); } else { valueString = String.valueOf(arg); } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index ba366ee99..d4608e694 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -575,10 +575,10 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(2)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(firstArg=innerFirstArgIs${PASSWORD},secondArg=innerSecondArgIs${USERNAME}))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(firstArg: innerFirstArgIs${PASSWORD}, secondArg: innerSecondArgIs${USERNAME}))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); InterpolatedSecretsAction.InterpolatedWarnings listStepWarning = warnings.get(1); - MatcherAssert.assertThat(listStepWarning.getStepSignature(), is("monomorphListSymbolStep(data: [@monomorphSymbol(firstArg=null,secondArg=hereismy${PASSWORD}), @monomorphSymbol(firstArg=${PASSWORD},secondArg=${USERNAME})])")); + MatcherAssert.assertThat(listStepWarning.getStepSignature(), is("monomorphListSymbolStep(data: [@monomorphSymbol(firstArg: null, secondArg: hereismy${PASSWORD}), @monomorphSymbol(firstArg: ${PASSWORD}, secondArg: ${USERNAME})])")); MatcherAssert.assertThat(listStepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); } From a24ffe0fb9cb442dede1dc5f54ad8e365d46dec4 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 20 Oct 2020 01:53:54 -0600 Subject: [PATCH 63/76] control warning behavior with system property --- .../jenkinsci/plugins/workflow/cps/DSL.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index 1b2c7c043..ec02cb05d 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -26,11 +26,13 @@ import com.cloudbees.groovy.cps.Continuable; import com.cloudbees.groovy.cps.Outcome; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import groovy.lang.Closure; import groovy.lang.GString; import groovy.lang.GroovyObject; import groovy.lang.GroovyObjectSupport; import groovy.lang.GroovyRuntimeException; +import hudson.AbortException; import hudson.EnvVars; import hudson.ExtensionList; import hudson.Util; @@ -87,6 +89,8 @@ import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import org.jenkinsci.plugins.workflow.steps.StepExecution; import org.jvnet.hudson.annotation_indexer.Index; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.ClassDescriptor; import org.kohsuke.stapler.NoStaplerConstructorException; @@ -99,6 +103,10 @@ */ @PersistIn(PROGRAM) public class DSL extends GroovyObjectSupport implements Serializable { + @SuppressFBWarnings("MS_SHOULD_BE_FINAL") // Used to control warning behavior of unsafe Groovy interpolation + @Restricted(NoExternalUse.class) + public static String UNSAFE_GROOVY_INTERPOLATION = DSL.class.getName() + ".UNSAFE_GROOVY_INTERPOLATION"; + private final FlowExecutionOwner handle; private transient CpsFlowExecution exec; private transient Map functions; @@ -353,6 +361,16 @@ protected Object invokeStep(StepDescriptor d, String name, Object args) { } private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsActionImpl argumentsAction, String nodeId, Set interpolatedStrings, @CheckForNull EnvVars envVars, @Nonnull Set sensitiveVariables, TaskListener listener) throws IOException, InterruptedException { + if (UNSAFE_GROOVY_INTERPOLATION.equals("ignore")) { + return; + } + boolean shouldFail; + if (UNSAFE_GROOVY_INTERPOLATION.equals("fail")) { + shouldFail = true; + } else { + shouldFail = false; + } + if (argumentsAction == null || interpolatedStrings.isEmpty() || envVars == null || envVars.isEmpty() || sensitiveVariables.isEmpty()) { return; } @@ -362,8 +380,14 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc .collect(Collectors.toList()); if (scanResults != null && !scanResults.isEmpty()) { - String warning = String.format("Warning: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure.%n\t\t Affected argument(s) used the following variable(s): %s%n\t\t See https://jenkins.io/redirect/groovy-string-interpolation for details.", - stepName, scanResults.toString()); + String warningType; + if (shouldFail) { + warningType = "Error"; + } else { + warningType = "Warning"; + } + String warning = String.format("%s: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure.%n\t\t Affected argument(s) used the following variable(s): %s%n\t\t See https://jenkins.io/redirect/groovy-string-interpolation for details.", + warningType, stepName, scanResults.toString()); listener.getLogger().println(warning); FlowExecutionOwner owner = exec.getOwner(); @@ -377,6 +401,9 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc } else { LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); } + if (shouldFail) { + throw new AbortException("Unsafe Groovy interpolation"); + } } } From cd434255a1f54b7c5b979825945680d0d94085b9 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 20 Oct 2020 09:17:17 -0600 Subject: [PATCH 64/76] update unit tests with new UninstantiatedDescribable output --- src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index d4608e694..ca846963e 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -485,7 +485,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("archiveArtifacts(delegate: @archiveArtifacts(=${PASSWORD}))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("archiveArtifacts(delegate: @archiveArtifacts(: ${PASSWORD}))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); } @@ -541,7 +541,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(firstArg=${PASSWORD},secondArg=two))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(firstArg: ${PASSWORD}, secondArg: two))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("monomorphWithSymbolStep")); From 41e50e673469336413362acaeb5acbc896057d36 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 2 Nov 2020 14:29:01 -0700 Subject: [PATCH 65/76] address review comments --- .../jenkinsci/plugins/workflow/cps/DSL.java | 8 +++---- .../cps/view/InterpolatedSecretsAction.java | 23 +++++++++++++++++++ .../view/InterpolatedSecretsActionTest.java | 23 +++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index ec02cb05d..eed9bbc29 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -105,7 +105,7 @@ public class DSL extends GroovyObjectSupport implements Serializable { @SuppressFBWarnings("MS_SHOULD_BE_FINAL") // Used to control warning behavior of unsafe Groovy interpolation @Restricted(NoExternalUse.class) - public static String UNSAFE_GROOVY_INTERPOLATION = DSL.class.getName() + ".UNSAFE_GROOVY_INTERPOLATION"; + public static String UNSAFE_GROOVY_INTERPOLATION = System.getProperty(DSL.class.getName() + ".UNSAFE_GROOVY_INTERPOLATION", "warn"); private final FlowExecutionOwner handle; private transient CpsFlowExecution exec; @@ -388,8 +388,6 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc } String warning = String.format("%s: A secret was passed to \"%s\" using Groovy String interpolation, which is insecure.%n\t\t Affected argument(s) used the following variable(s): %s%n\t\t See https://jenkins.io/redirect/groovy-string-interpolation for details.", warningType, stepName, scanResults.toString()); - listener.getLogger().println(warning); - FlowExecutionOwner owner = exec.getOwner(); if (owner != null && owner.getExecutable() instanceof Run) { InterpolatedSecretsAction runReport = ((Run) owner.getExecutable()).getAction(InterpolatedSecretsAction.class); @@ -402,7 +400,9 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); } if (shouldFail) { - throw new AbortException("Unsafe Groovy interpolation"); + throw new AbortException(warning); + } else { + listener.getLogger().println(warning); } } } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index a521b143c..8ed37c107 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -1,3 +1,26 @@ +/* + * The MIT License + * + * Copyright (c) 2020, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package org.jenkinsci.plugins.workflow.cps.view; import hudson.model.Run; diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java index ff03a673a..67ddd8754 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java @@ -1,3 +1,26 @@ +/* + * The MIT License + * + * Copyright (c) 2020, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package org.jenkinsci.plugins.workflow.cps.view; import org.hamcrest.MatcherAssert; From 3e54d3691a6488743c19c0670f8022b1e02602cd Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Mon, 2 Nov 2020 15:54:00 -0700 Subject: [PATCH 66/76] make InterpolatedWarnings.run transient field --- .../cps/view/InterpolatedSecretsAction.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 8ed37c107..37c76bffe 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -45,6 +45,7 @@ * Action to generate the UI report for watched environment variables */ @Restricted(NoExternalUse.class) +@ExportedBean public class InterpolatedSecretsAction implements RunAction2 { private List interpolatedWarnings = new ArrayList<>(); @@ -66,6 +67,7 @@ public void record(@Nonnull String stepName, @Nonnull List interpolatedV interpolatedWarnings.add(new InterpolatedWarnings(stepName, interpolatedVariables, run, nodeId)); } + @Exported public List getWarnings() { return interpolatedWarnings; } @@ -93,11 +95,11 @@ public void onLoad(Run run) { } @ExportedBean - public static class InterpolatedWarnings { + public static class InterpolatedWarnings implements RunAction2 { final String stepName; final List interpolatedVariables; - final Run run; final String nodeId; + private transient Run run; InterpolatedWarnings(@Nonnull String stepName, @Nonnull List interpolatedVariables, @Nonnull Run run, @Nonnull String nodeId) { this.stepName = stepName; @@ -154,6 +156,31 @@ private Map getStepArguments(Run run, String nodeId) throws Ille public List getInterpolatedVariables() { return interpolatedVariables; } + + @Override + public void onAttached(Run run) { + this.run = run; + } + + @Override + public void onLoad(Run run) { + this.run = run; + } + + @Override + public String getIconFileName() { + return null; + } + + @Override + public String getDisplayName() { + return null; + } + + @Override + public String getUrlName() { + return null; + } } private static String argumentToString(Object arg) { From b52438dc7576b6620015f24d08baa6990b6fa120 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 3 Nov 2020 00:49:06 -0700 Subject: [PATCH 67/76] update UninstantiatedDescribable $class toString --- .../cps/view/InterpolatedSecretsAction.java | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 37c76bffe..a9f101ef2 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -216,28 +216,24 @@ private static String argumentToString(Object arg) { .collect(Collectors.joining(", ", "[", "]")); } else if (arg instanceof UninstantiatedDescribable) { UninstantiatedDescribable ud = (UninstantiatedDescribable) arg; - String udName = null; + Map udArgs = ud.getArguments(); if (ud.getSymbol() != null) { - udName = '@' + ud.getSymbol(); - } - if (ud.getKlass() != null) { - udName = '$' + ud.getKlass() + udName; + valueString = udArgs.entrySet().stream() + .map(InterpolatedSecretsAction::argumentToString) + .collect(Collectors.joining(", ", "@" + ud.getSymbol() + "(", ")")); + } else { + if (udArgs.isEmpty()) { + valueString = "[$class: " + ud.getKlass() + "]"; + } else { + valueString = udArgs.entrySet().stream() + .map(InterpolatedSecretsAction::argumentToString) + .collect(Collectors.joining(", ", "[$class: " + ud.getKlass() + ",", "]")); + } } - - Map udArgs = ud.getArguments(); - valueString = udArgs.entrySet().stream() - .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", udName + "(", ")")); } else { valueString = String.valueOf(arg); } return valueString; } - - private static String mapToString(Map valueMap) { - return valueMap.entrySet().stream() - .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", "[", "]")); - } } From 49695e08dc0b1b386e1544805feba6b511120fc8 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Tue, 3 Nov 2020 16:27:32 -0700 Subject: [PATCH 68/76] update InterpolatedSecretesAction onLoad and onAttached Remove @ symbol for UninstantiatedDescribable getStepSignature --- .../cps/view/InterpolatedSecretsAction.java | 34 +++++-------------- .../plugins/workflow/cps/DSLTest.java | 8 ++--- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index a9f101ef2..cfd1eb233 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -87,15 +87,21 @@ public boolean isInProgress() { @Override public void onAttached(Run run) { this.run = run; + for (InterpolatedWarnings warning : interpolatedWarnings) { + warning.run = run; + } } @Override public void onLoad(Run run) { this.run = run; + for (InterpolatedWarnings warning : interpolatedWarnings) { + warning.run = run; + } } @ExportedBean - public static class InterpolatedWarnings implements RunAction2 { + public static class InterpolatedWarnings { final String stepName; final List interpolatedVariables; final String nodeId; @@ -157,30 +163,6 @@ public List getInterpolatedVariables() { return interpolatedVariables; } - @Override - public void onAttached(Run run) { - this.run = run; - } - - @Override - public void onLoad(Run run) { - this.run = run; - } - - @Override - public String getIconFileName() { - return null; - } - - @Override - public String getDisplayName() { - return null; - } - - @Override - public String getUrlName() { - return null; - } } private static String argumentToString(Object arg) { @@ -220,7 +202,7 @@ private static String argumentToString(Object arg) { if (ud.getSymbol() != null) { valueString = udArgs.entrySet().stream() .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", "@" + ud.getSymbol() + "(", ")")); + .collect(Collectors.joining(", ", ud.getSymbol() + "(", ")")); } else { if (udArgs.isEmpty()) { valueString = "[$class: " + ud.getKlass() + "]"; diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index ca846963e..b2eda00db 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -485,7 +485,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("archiveArtifacts(delegate: @archiveArtifacts(: ${PASSWORD}))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("archiveArtifacts(delegate: archiveArtifacts(: ${PASSWORD}))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); } @@ -541,7 +541,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(firstArg: ${PASSWORD}, secondArg: two))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: monomorphSymbol(firstArg: ${PASSWORD}, secondArg: two))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("monomorphWithSymbolStep")); @@ -575,10 +575,10 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(2)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: @monomorphSymbol(firstArg: innerFirstArgIs${PASSWORD}, secondArg: innerSecondArgIs${USERNAME}))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: monomorphSymbol(firstArg: innerFirstArgIs${PASSWORD}, secondArg: innerSecondArgIs${USERNAME}))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); InterpolatedSecretsAction.InterpolatedWarnings listStepWarning = warnings.get(1); - MatcherAssert.assertThat(listStepWarning.getStepSignature(), is("monomorphListSymbolStep(data: [@monomorphSymbol(firstArg: null, secondArg: hereismy${PASSWORD}), @monomorphSymbol(firstArg: ${PASSWORD}, secondArg: ${USERNAME})])")); + MatcherAssert.assertThat(listStepWarning.getStepSignature(), is("monomorphListSymbolStep(data: [monomorphSymbol(firstArg: null, secondArg: hereismy${PASSWORD}), monomorphSymbol(firstArg: ${PASSWORD}, secondArg: ${USERNAME})])")); MatcherAssert.assertThat(listStepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); } From 7d8672c1ed723d6ce88e908736b053407238c754 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 5 Nov 2020 01:19:17 -0700 Subject: [PATCH 69/76] Update getStepSignature to better reflect pipeline input --- .../jenkinsci/plugins/workflow/cps/DSL.java | 4 +- .../cps/view/InterpolatedSecretsAction.java | 48 ++++++++++++++----- .../plugins/workflow/cps/DSLTest.java | 2 +- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index eed9bbc29..f81cbf0b0 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -425,9 +425,10 @@ protected Object invokeDescribable(String symbol, Object _args) { StepDescriptor metaStep = metaSteps.size()==1 ? metaSteps.get(0) : null; boolean singleArgumentOnly = false; + DescribableModel symbolModel = null; if (metaStep != null) { Descriptor symbolDescriptor = SymbolLookup.get().findDescriptor((Class)(metaStep.getMetaStepArgumentType()), symbol); - DescribableModel symbolModel = DescribableModel.of(symbolDescriptor.clazz); + symbolModel = DescribableModel.of(symbolDescriptor.clazz); singleArgumentOnly = symbolModel.hasSingleRequiredParameter() && symbolModel.getParameters().size() == 1; } @@ -436,6 +437,7 @@ protected Object invokeDescribable(String symbol, Object _args) { NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, new HashSet<>()); UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); + ud.setModel(symbolModel); if (metaStep==null) { // there's no meta-step associated with it, so this symbol is not executable. diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index cfd1eb233..2b49232d7 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -25,10 +25,14 @@ import hudson.model.Run; import jenkins.model.RunAction2; +import org.jenkinsci.plugins.structs.describable.DescribableModel; +import org.jenkinsci.plugins.structs.describable.DescribableParameter; import org.jenkinsci.plugins.structs.describable.UninstantiatedDescribable; import org.jenkinsci.plugins.workflow.actions.ArgumentsAction; import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; import org.jenkinsci.plugins.workflow.graph.FlowNode; +import org.jenkinsci.plugins.workflow.graph.StepNode; +import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.NoExternalUse; import org.kohsuke.stapler.export.Exported; @@ -117,19 +121,40 @@ public static class InterpolatedWarnings { @Exported public String getStepSignature() { Map stepArguments; + FlowNode node; try { - stepArguments = getStepArguments(run, nodeId); + node = getFlowNode(run, nodeId); + ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); + if (argumentsAction == null) { + throw new IllegalStateException("null arguments action"); + } + stepArguments = argumentsAction.getArguments(); } catch (IllegalStateException e) { return "Unable to construct " + stepName + ": " + e.getMessage(); } + if (node instanceof StepNode) { + StepDescriptor descriptor = ((StepNode)node).getDescriptor(); + if (descriptor != null && descriptor.isMetaStep()) { + DescribableParameter p = DescribableModel.of(descriptor.clazz).getFirstRequiredParameter(); + if (p != null) { + Object arg = ArgumentsAction.getResolvedArguments(node).get(p.getName()); + if (arg instanceof UninstantiatedDescribable) { + return argumentToString(arg); + } else { + return stepName + "(" + argumentToString(arg) + ")"; + } + } + } + } + return stepArguments.entrySet().stream() .map(InterpolatedSecretsAction::argumentToString) .collect(Collectors.joining(", ", stepName + "(", ")")); } @Nonnull - private Map getStepArguments(Run run, String nodeId) throws IllegalStateException { + private FlowNode getFlowNode(Run run, String nodeId) { String failReason; if (run instanceof FlowExecutionOwner.Executable) { try { @@ -137,12 +162,7 @@ private Map getStepArguments(Run run, String nodeId) throws Ille if (owner != null) { FlowNode node = owner.get().getNode(nodeId); if (node != null) { - ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); - if (argumentsAction != null) { - return argumentsAction.getArguments(); - } else { - failReason = "null arguments action"; - } + return node; } else { failReason = "null flow node"; } @@ -162,7 +182,6 @@ private Map getStepArguments(Run run, String nodeId) throws Ille public List getInterpolatedVariables() { return interpolatedVariables; } - } private static String argumentToString(Object arg) { @@ -200,9 +219,14 @@ private static String argumentToString(Object arg) { UninstantiatedDescribable ud = (UninstantiatedDescribable) arg; Map udArgs = ud.getArguments(); if (ud.getSymbol() != null) { - valueString = udArgs.entrySet().stream() - .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", ud.getSymbol() + "(", ")")); + String prefix = ud.getSymbol() + "("; + if (ud.hasSoleRequiredArgument() && udArgs.size() == 1) { + valueString = prefix + argumentToString(udArgs.values().iterator().next()) + ")"; + } else { + valueString = udArgs.entrySet().stream() + .map(InterpolatedSecretsAction::argumentToString) + .collect(Collectors.joining(", ", prefix, ")")); + } } else { if (udArgs.isEmpty()) { valueString = "[$class: " + ud.getKlass() + "]"; diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index b2eda00db..6733f9bfb 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -485,7 +485,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("archiveArtifacts(delegate: archiveArtifacts(: ${PASSWORD}))")); + MatcherAssert.assertThat(stepWarning.getStepSignature(), is("archiveArtifacts(${PASSWORD})")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); } From 611326c083b82b5edf782e0da26361686b0a7ffc Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 5 Nov 2020 14:20:11 -0700 Subject: [PATCH 70/76] Remove printing of step signature --- .../jenkinsci/plugins/workflow/cps/DSL.java | 2 +- .../cps/view/InterpolatedSecretsAction.java | 134 +----------------- .../InterpolatedSecretsAction/summary.jelly | 7 +- .../plugins/workflow/cps/DSLTest.java | 13 +- .../view/InterpolatedSecretsActionTest.java | 11 -- 5 files changed, 15 insertions(+), 152 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index f81cbf0b0..ea939da19 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -395,7 +395,7 @@ private void logInterpolationWarnings(String stepName, @CheckForNull ArgumentsAc runReport = new InterpolatedSecretsAction(); ((Run) owner.getExecutable()).addAction(runReport); } - runReport.record(stepName, scanResults, nodeId); + runReport.record(stepName, scanResults); } else { LOGGER.log(Level.FINE, "Unable to generate Interpolated Secrets Report"); } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java index 2b49232d7..c2261f68a 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction.java @@ -67,8 +67,8 @@ public String getUrlName() { return null; } - public void record(@Nonnull String stepName, @Nonnull List interpolatedVariables, @Nonnull String nodeId) { - interpolatedWarnings.add(new InterpolatedWarnings(stepName, interpolatedVariables, run, nodeId)); + public void record(@Nonnull String stepName, @Nonnull List interpolatedVariables) { + interpolatedWarnings.add(new InterpolatedWarnings(stepName, interpolatedVariables)); } @Exported @@ -91,91 +91,26 @@ public boolean isInProgress() { @Override public void onAttached(Run run) { this.run = run; - for (InterpolatedWarnings warning : interpolatedWarnings) { - warning.run = run; - } } @Override public void onLoad(Run run) { this.run = run; - for (InterpolatedWarnings warning : interpolatedWarnings) { - warning.run = run; - } } @ExportedBean public static class InterpolatedWarnings { final String stepName; final List interpolatedVariables; - final String nodeId; - private transient Run run; - InterpolatedWarnings(@Nonnull String stepName, @Nonnull List interpolatedVariables, @Nonnull Run run, @Nonnull String nodeId) { + InterpolatedWarnings(@Nonnull String stepName, @Nonnull List interpolatedVariables) { this.stepName = stepName; this.interpolatedVariables = interpolatedVariables; - this.run = run; - this.nodeId = nodeId; } @Exported - public String getStepSignature() { - Map stepArguments; - FlowNode node; - try { - node = getFlowNode(run, nodeId); - ArgumentsAction argumentsAction = node.getPersistentAction(ArgumentsAction.class); - if (argumentsAction == null) { - throw new IllegalStateException("null arguments action"); - } - stepArguments = argumentsAction.getArguments(); - } catch (IllegalStateException e) { - return "Unable to construct " + stepName + ": " + e.getMessage(); - } - - if (node instanceof StepNode) { - StepDescriptor descriptor = ((StepNode)node).getDescriptor(); - if (descriptor != null && descriptor.isMetaStep()) { - DescribableParameter p = DescribableModel.of(descriptor.clazz).getFirstRequiredParameter(); - if (p != null) { - Object arg = ArgumentsAction.getResolvedArguments(node).get(p.getName()); - if (arg instanceof UninstantiatedDescribable) { - return argumentToString(arg); - } else { - return stepName + "(" + argumentToString(arg) + ")"; - } - } - } - } - - return stepArguments.entrySet().stream() - .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", stepName + "(", ")")); - } - - @Nonnull - private FlowNode getFlowNode(Run run, String nodeId) { - String failReason; - if (run instanceof FlowExecutionOwner.Executable) { - try { - FlowExecutionOwner owner = ((FlowExecutionOwner.Executable) run).asFlowExecutionOwner(); - if (owner != null) { - FlowNode node = owner.get().getNode(nodeId); - if (node != null) { - return node; - } else { - failReason = "null flow node"; - } - } else { - failReason = "null flow execution owner"; - } - } catch (IOException e) { - failReason = "could not get flow node"; - } - } else { - failReason = "not an instance of FlowExecutionOwner.Executable"; - } - throw new IllegalStateException(failReason); + public String getStepName() { + return stepName; } @Exported @@ -183,63 +118,4 @@ public List getInterpolatedVariables() { return interpolatedVariables; } } - - private static String argumentToString(Object arg) { - String valueString; - if (arg instanceof Map.Entry) { - Map.Entry argEntry = (Map.Entry) arg; - Object value = argEntry.getValue(); - if (value instanceof ArgumentsAction.NotStoredReason) { - switch ((ArgumentsAction.NotStoredReason) value) { - case OVERSIZE_VALUE: - valueString = "argument omitted due to length"; - break; - case UNSERIALIZABLE: - valueString = "unable to serialize argument"; - break; - default: - valueString = String.valueOf(value); - break; - } - } else if (value instanceof Map || value instanceof List || value instanceof UninstantiatedDescribable) { - valueString = argumentToString(value); - } else { - valueString = String.valueOf(value); - } - return argEntry.getKey() + ": " + valueString; - } else if (arg instanceof Map) { - valueString = ((Map) arg).entrySet().stream() - .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", "[", "]")); - } else if (arg instanceof List) { - valueString = ((List) arg).stream() - .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", "[", "]")); - } else if (arg instanceof UninstantiatedDescribable) { - UninstantiatedDescribable ud = (UninstantiatedDescribable) arg; - Map udArgs = ud.getArguments(); - if (ud.getSymbol() != null) { - String prefix = ud.getSymbol() + "("; - if (ud.hasSoleRequiredArgument() && udArgs.size() == 1) { - valueString = prefix + argumentToString(udArgs.values().iterator().next()) + ")"; - } else { - valueString = udArgs.entrySet().stream() - .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", prefix, ")")); - } - } else { - if (udArgs.isEmpty()) { - valueString = "[$class: " + ud.getKlass() + "]"; - } else { - valueString = udArgs.entrySet().stream() - .map(InterpolatedSecretsAction::argumentToString) - .collect(Collectors.joining(", ", "[$class: " + ud.getKlass() + ",", "]")); - } - } - } else { - valueString = String.valueOf(arg); - } - - return valueString; - } } diff --git a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly index 89aae7fd9..0ece94ec1 100644 --- a/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly +++ b/src/main/resources/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsAction/summary.jelly @@ -22,7 +22,7 @@ THE SOFTWARE. - ${%Possible insecure use of sensitive variables} + ${%The following steps that have been detected may have insecure interpolation of sensitive variables} (${%click here for an explanation}): (${%in progress}) @@ -30,10 +30,7 @@ THE SOFTWARE.

            -
          • ${warning.stepSignature}
            • -
              -
            • interpolated variables: ${warning.interpolatedVariables}
              • -
            +
          • ${warning.stepName}: ${warning.interpolatedVariables}
            •
          diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java index 6733f9bfb..fc522a79e 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/DSLTest.java @@ -455,7 +455,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is(shellStep + "(script: echo ${PASSWORD})")); + MatcherAssert.assertThat(stepWarning.getStepName(), is(shellStep)); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(shellStep)); @@ -485,7 +485,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("archiveArtifacts(${PASSWORD})")); + MatcherAssert.assertThat(stepWarning.getStepName(), is("archiveArtifacts")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); } @@ -510,7 +510,7 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is(shellStep + "(script: echo ${PASSWORD} ${USERNAME} ${PASSWORD})")); + MatcherAssert.assertThat(stepWarning.getStepName(), is(shellStep)); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD", "USERNAME"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate(shellStep)); @@ -541,7 +541,8 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(1)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: monomorphSymbol(firstArg: ${PASSWORD}, secondArg: two))")); + MatcherAssert.assertThat(stepWarning.getStepName(), is("monomorphWithSymbolStep")); +// MatcherAssert.assertThat(stepWarning.getStepName(), is("monomorphWithSymbolStep(data: monomorphSymbol(firstArg: ${PASSWORD}, secondArg: two))")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), is(Arrays.asList("PASSWORD"))); LinearScanner scan = new LinearScanner(); FlowNode node = scan.findFirstMatch(run.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("monomorphWithSymbolStep")); @@ -575,10 +576,10 @@ public void namedSoleParamForStep() throws Exception { List warnings = reportAction.getWarnings(); MatcherAssert.assertThat(warnings.size(), is(2)); InterpolatedSecretsAction.InterpolatedWarnings stepWarning = warnings.get(0); - MatcherAssert.assertThat(stepWarning.getStepSignature(), is("monomorphWithSymbolStep(data: monomorphSymbol(firstArg: innerFirstArgIs${PASSWORD}, secondArg: innerSecondArgIs${USERNAME}))")); + MatcherAssert.assertThat(stepWarning.getStepName(), is("monomorphWithSymbolStep")); MatcherAssert.assertThat(stepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); InterpolatedSecretsAction.InterpolatedWarnings listStepWarning = warnings.get(1); - MatcherAssert.assertThat(listStepWarning.getStepSignature(), is("monomorphListSymbolStep(data: [monomorphSymbol(firstArg: null, secondArg: hereismy${PASSWORD}), monomorphSymbol(firstArg: ${PASSWORD}, secondArg: ${USERNAME})])")); + MatcherAssert.assertThat(listStepWarning.getStepName(), is("monomorphListSymbolStep")); MatcherAssert.assertThat(listStepWarning.getInterpolatedVariables(), equalTo(Arrays.asList("PASSWORD", "USERNAME"))); } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java index 67ddd8754..41327de73 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java @@ -56,15 +56,4 @@ public class InterpolatedSecretsActionTest { public void newProject() throws Exception { p = r.createProject(WorkflowJob.class); } - - @Test - public void testStepSignature() throws Exception { - p.setDefinition(new CpsFlowDefinition("monomorphListStep([[firstArg:'one', secondArg:'two'], [firstArg:'three', secondArg:'four']])", true)); - WorkflowRun b = r.assertBuildStatusSuccess(p.scheduleBuild2(0)); - LinearScanner scan = new LinearScanner(); - FlowNode node = scan.findFirstMatch(b.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("monomorphListStep")); - InterpolatedSecretsAction.InterpolatedWarnings warning = - new InterpolatedSecretsAction.InterpolatedWarnings("monomorphListStep", Collections.emptyList(), b, node.getId()); - MatcherAssert.assertThat(warning.getStepSignature(), Matchers.is("monomorphListStep(data: [[firstArg: one, secondArg: two], [firstArg: three, secondArg: four]])")); - } } From 96c2f30f69ccbad8d2b4345597906e30d9532a34 Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 5 Nov 2020 14:28:54 -0700 Subject: [PATCH 71/76] remove setting the model for the Uninstantiated Describable --- src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java index ea939da19..a930b0a7b 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/cps/DSL.java @@ -425,10 +425,9 @@ protected Object invokeDescribable(String symbol, Object _args) { StepDescriptor metaStep = metaSteps.size()==1 ? metaSteps.get(0) : null; boolean singleArgumentOnly = false; - DescribableModel symbolModel = null; if (metaStep != null) { Descriptor symbolDescriptor = SymbolLookup.get().findDescriptor((Class)(metaStep.getMetaStepArgumentType()), symbol); - symbolModel = DescribableModel.of(symbolDescriptor.clazz); + DescribableModel symbolModel = DescribableModel.of(symbolDescriptor.clazz); singleArgumentOnly = symbolModel.hasSingleRequiredParameter() && symbolModel.getParameters().size() == 1; } @@ -437,7 +436,6 @@ protected Object invokeDescribable(String symbol, Object _args) { NamedArgsAndClosure args = parseArgs(_args, metaStep!=null && metaStep.takesImplicitBlockArgument(), UninstantiatedDescribable.ANONYMOUS_KEY, singleArgumentOnly, new HashSet<>()); UninstantiatedDescribable ud = new UninstantiatedDescribable(symbol, null, args.namedArgs); - ud.setModel(symbolModel); if (metaStep==null) { // there's no meta-step associated with it, so this symbol is not executable. From 78111484e999fe68e87b7345d45ff68121c0d396 Mon Sep 17 00:00:00 2001 From: Devin Nusbaum Date: Thu, 5 Nov 2020 17:08:39 -0500 Subject: [PATCH 72/76] Make sure password parameters are masked in step arguments --- pom.xml | 2 +- .../cps/actions/ArgumentsActionImplTest.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 48f893a98..1a6be4f6c 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,7 @@ org.jenkins-ci.plugins.workflow workflow-support + 3.6-rc759.3e4ba3120e7a org.jenkins-ci.plugins.workflow @@ -101,7 +102,6 @@ org.jenkins-ci.plugins script-security - 1.75 org.jenkins-ci.plugins diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java index 692079725..8b1c03260 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/cps/actions/ArgumentsActionImplTest.java @@ -13,6 +13,10 @@ import hudson.Functions; import hudson.XmlFile; import hudson.model.Action; +import hudson.model.ParametersAction; +import hudson.model.ParametersDefinitionProperty; +import hudson.model.PasswordParameterDefinition; +import hudson.model.PasswordParameterValue; import hudson.tasks.ArtifactArchiver; import org.apache.commons.lang.RandomStringUtils; import org.hamcrest.MatcherAssert; @@ -48,6 +52,7 @@ import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; import org.jvnet.hudson.test.BuildWatcher; @@ -621,6 +626,22 @@ public void testReallyUnusualStepInstantiations() throws Exception { equalTo(NotStoredReason.UNSERIALIZABLE)); } + @Issue("JENKINS-47101") + @Test public void passwordParametersSanitized() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class); + p.addProperty(new ParametersDefinitionProperty( + Arrays.asList(new PasswordParameterDefinition("MYPASSWORD", "mysecret", "description")))); + p.setDefinition(new CpsFlowDefinition("echo(\"$MYPASSWORD\")", true)); + ParametersAction paramsAction = new ParametersAction(Arrays.asList(new PasswordParameterValue("MYPASSWORD", "mysecret"))); + WorkflowRun b = p.scheduleBuild2(0, paramsAction).waitForStart(); + r.assertBuildStatusSuccess(r.waitForCompletion(b)); + LinearScanner scan = new LinearScanner(); + FlowNode shNode = scan.findFirstMatch(b.getExecution().getCurrentHeads().get(0), new NodeStepTypePredicate("echo")); + ArgumentsAction args = shNode.getPersistentAction(ArgumentsAction.class); + assertThat(args.isUnmodifiedArguments(), equalTo(false)); + assertThat(args.getArguments(), hasEntry("message", "${MYPASSWORD}")); + } + public static class NopStep extends Step { @DataBoundConstructor public NopStep(Object value) {} From f7798af3036da3c8f1cf14267a27f5b54122bc62 Mon Sep 17 00:00:00 2001 From: Devin Nusbaum Date: Thu, 5 Nov 2020 17:25:10 -0500 Subject: [PATCH 73/76] Remove InterpolatedSecretsActionTest.java --- .../view/InterpolatedSecretsActionTest.java | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100644 src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java diff --git a/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java b/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java deleted file mode 100644 index 41327de73..000000000 --- a/src/test/java/org/jenkinsci/plugins/workflow/cps/view/InterpolatedSecretsActionTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * The MIT License - * - * Copyright (c) 2020, CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package org.jenkinsci.plugins.workflow.cps.view; - -import org.hamcrest.MatcherAssert; -import org.hamcrest.Matchers; -import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; -import org.jenkinsci.plugins.workflow.graph.FlowNode; -import org.jenkinsci.plugins.workflow.graphanalysis.LinearScanner; -import org.jenkinsci.plugins.workflow.graphanalysis.NodeStepTypePredicate; -import org.jenkinsci.plugins.workflow.job.WorkflowJob; -import org.jenkinsci.plugins.workflow.job.WorkflowRun; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.jvnet.hudson.test.BuildWatcher; -import org.jvnet.hudson.test.JenkinsRule; -import org.jvnet.hudson.test.LoggerRule; - -import java.util.Collections; - -public class InterpolatedSecretsActionTest { - @ClassRule - public static BuildWatcher buildWatcher = new BuildWatcher(); - - @ClassRule - public static JenkinsRule r = new JenkinsRule(); - - @Rule - public LoggerRule logging = new LoggerRule(); - - private WorkflowJob p; - @Before - public void newProject() throws Exception { - p = r.createProject(WorkflowJob.class); - } -} From 40eb64a3318bfc95646e163310f55a174fd663ae Mon Sep 17 00:00:00 2001 From: Devin Nusbaum Date: Thu, 5 Nov 2020 17:29:11 -0500 Subject: [PATCH 74/76] Align workflow-support tests jar with incremental version --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1a6be4f6c..80f7332c0 100644 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,7 @@ 8 false 1.32 + 3.6-rc759.3e4ba3120e7a @@ -93,7 +94,7 @@ org.jenkins-ci.plugins.workflow workflow-support - 3.6-rc759.3e4ba3120e7a + ${workflow-support-plugin.version} org.jenkins-ci.plugins.workflow @@ -132,6 +133,7 @@ org.jenkins-ci.plugins.workflow workflow-support + ${workflow-support-plugin.version} tests test From bc8d268faa4195b4de8097bccd5ed368956577d4 Mon Sep 17 00:00:00 2001 From: Devin Nusbaum Date: Thu, 5 Nov 2020 17:50:15 -0500 Subject: [PATCH 75/76] Update to latest workflow-support incremental --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 80f7332c0..8a56cd39b 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 8 false 1.32 - 3.6-rc759.3e4ba3120e7a + 3.6-rc759.b1425935abde From 4524d26f0664098320b29a6848d9644c8e7ecc4c Mon Sep 17 00:00:00 2001 From: Carroll Chiou Date: Thu, 5 Nov 2020 19:52:14 -0700 Subject: [PATCH 76/76] update pom, update changelog to prepare for release --- CHANGELOG.md | 8 ++++++++ pom.xml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a68067b..289dba700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## Changelog +### 2.85 + +Release date: 2020-11-05 + +* Improvement: Add warnings when secrets are used with Groovy String interpolation. ([JENKINS-63254](https://issues.jenkins-ci.org/browse/JENKINS-63254)) +* Fix: Allow masking of secret variables that use the same name as system variables. ([JENKINS-47101](https://issues.jenkins-ci.org/browse/JENKINS-47101)) +* Fix: Throw an error when a step that requires a body has no body. ([PR #370](https://github.com/jenkinsci/workflow-cps-plugin/pull/370)) + ### 2.84 Release date: 2020-10-30 diff --git a/pom.xml b/pom.xml index 000325915..a07addba6 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ 8 false 1.32 - 3.6-rc759.b1425935abde + 3.6 12.19.0 6.14.8 1.10.0