diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 9ff818c0da19..f6641f7a1cea 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -13,6 +13,7 @@ Note that these environment variables may be removed at any time. | `ARCHIVED_WORKFLOW_GC_PERIOD` | `time.Duration` | The periodicity for GC of archived workflows. | | `ARGO_TRACE` | `bool` | Whether to enable tracing statements in Argo components. | | `DEFAULT_REQUEUE_TIME` | `time.Duration` | The requeue time for the rate limiter of the workflow queue. | +| `EXPRESSION_TEMPLATES` | `bool` | Escape hatch to disable expression templates. Default `true`. | | `GZIP_IMPLEMENTATION` | `string` | The implementation of compression/decompression. Currently only "PGZip" and "GZip" are supported. Defaults to "PGZip". | | `LEADER_ELECTION_IDENTITY` | `string` | The ID used for workflow controllers to elect a leader. | | `LEADER_ELECTION_LEASE_DURATION` | `time.Duration` | The duration that non-leader candidates will wait to force acquire leadership. | diff --git a/docs/fields.md b/docs/fields.md index 9a369c019cc1..4ff24f8c8b62 100644 --- a/docs/fields.md +++ b/docs/fields.md @@ -94,6 +94,8 @@ Workflow is the definition of a workflow resource - [`exit-handlers.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`forever.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/forever.yaml) - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) @@ -445,6 +447,8 @@ WorkflowSpec is the specification of a Workflow. - [`exit-handlers.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`forever.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/forever.yaml) - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) @@ -814,6 +818,8 @@ CronWorkflowSpec is the specification of a CronWorkflow - [`exit-handlers.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`forever.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/forever.yaml) - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) @@ -1141,6 +1147,8 @@ WorkflowTemplateSpec is a spec of WorkflowTemplate. - [`exit-handlers.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`forever.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/forever.yaml) - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) @@ -1452,6 +1460,8 @@ Arguments to a template - [`exit-handler-step-level.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handler-step-level.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`global-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/global-outputs.yaml) - [`global-parameters.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/global-parameters.yaml) @@ -1798,6 +1808,8 @@ Template is a reusable and composable unit of execution in a workflow - [`exit-handlers.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`forever.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/forever.yaml) - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) @@ -2176,6 +2188,8 @@ Outputs hold parameters, artifacts, and results from a step - [`data-transformations.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/data-transformations.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) - [`global-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/global-outputs.yaml) @@ -2393,6 +2407,8 @@ Parameter indicate a passed string parameter to a service template with an optio - [`exit-handler-step-level.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handler-step-level.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`global-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/global-outputs.yaml) - [`global-parameters.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/global-parameters.yaml) @@ -2653,6 +2669,8 @@ DAGTemplate is a template subtype for directed acyclic graph templates - [`exit-handler-dag-level.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handler-dag-level.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`key-only-artifact.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/key-only-artifact.yaml) - [`loops-dag.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/loops-dag.yaml) @@ -2810,6 +2828,8 @@ Inputs are the mechanism for passing parameters, artifacts, volumes from one tem - [`exit-handler-step-level.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handler-step-level.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`global-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/global-outputs.yaml) - [`handle-large-output-results.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/handle-large-output-results.yaml) @@ -3544,6 +3564,8 @@ ValueFrom describes a location in which to obtain the value to a parameter - [`data-transformations.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/data-transformations.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`global-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/global-outputs.yaml) - [`handle-large-output-results.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/handle-large-output-results.yaml) @@ -3648,6 +3670,8 @@ MetricLabel is a single label for a prometheus metric - [`daemoned-stateful-set-with-service.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/daemoned-stateful-set-with-service.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`hello-world.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/hello-world.yaml) - [`k8s-owner-reference.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-owner-reference.yaml) @@ -3715,6 +3739,8 @@ DAGTask represents a node in the graph during DAG execution - [`exit-handler-dag-level.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handler-dag-level.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`key-only-artifact.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/key-only-artifact.yaml) - [`loops-dag.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/loops-dag.yaml) @@ -4151,6 +4177,8 @@ ObjectMeta is metadata that all persisted resources must have, which includes al - [`exit-handlers.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`forever.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/forever.yaml) - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) @@ -4721,6 +4749,8 @@ A single application container that you want to run within a pod. - [`exit-handlers.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`forever.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/forever.yaml) - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) @@ -5352,6 +5382,8 @@ PersistentVolumeClaimSpec describes the common attributes of storage devices and - [`exit-handlers.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/exit-handlers.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`forever.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/forever.yaml) - [`fun-with-gifs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/fun-with-gifs.yaml) @@ -5979,6 +6011,8 @@ EnvVarSource represents a source for the value of an EnvVar. - [`data-transformations.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/data-transformations.yaml) +- [`expression-tag-template-workflow.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/expression-tag-template-workflow.yaml) + - [`global-outputs.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/global-outputs.yaml) - [`handle-large-output-results.yaml`](https://github.com/argoproj/argo-workflows/blob/master/examples/handle-large-output-results.yaml) diff --git a/docs/variables.md b/docs/variables.md index c74ec85b6ac9..a21d0902499d 100644 --- a/docs/variables.md +++ b/docs/variables.md @@ -2,42 +2,128 @@ Some fields in a workflow specification allow for variable references which are automatically substituted by Argo. -??? note "How to use variables" - Variables are enclosed in curly braces and **may** include whitespace between the brackets and variable. - - ``` yaml - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: hello-world-parameters- - spec: - entrypoint: whalesay - arguments: +## How to use variables + +Variables are enclosed in curly braces: + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: hello-world-parameters- +spec: + entrypoint: whalesay + arguments: + parameters: + - name: message + value: hello world + templates: + - name: whalesay + inputs: parameters: - - name: message - value: hello world - templates: - - name: whalesay - inputs: - parameters: - name: message - container: - image: docker/whalesay - command: [cowsay] - # args: ["{{ inputs.parameters.message }}"] <- good - args: ["{{inputs.parameters.message}}"] # good - ``` + container: + image: docker/whalesay + command: [ cowsay ] + args: [ "{{inputs.parameters.message}}" ] +``` The following variables are made available to reference various metadata of a workflow: -## All Templates +## Template Tag Kinds + +There are two kinds of template tag: + +* **simple** The default, e.g. `{{workflow.name}}` +* **expression** Where`{{` is immediately followed by `=`, e.g. `{{=workflow.name}}`. + +### Simple + +The tag is substituted with the variable that has a name the same as the tag. + +Simple tags **may** have whitespace between the brackets and variable. + +```yaml +args: [ "{{ inputs.parameters.message }}" ] +``` + +### Expression + +> Since v3.1 + +The tag is substituted with the result of evaluating the tag as an expression. + +[Learn about the expression syntax](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md). + +#### Examples + +Plain list: + +``` +[1, 2] +``` + +Filter a list: + +``` +filter([1, 2], { # > 1}) +``` + +Map a list: + +``` +map([1, 2], { # * 2 }) +``` + +We provide some core functions: + +Cast to int: + +``` +asInt(inputs.parameters["my-int-param"]) +``` + +Cast to flloat: + +``` +asFloat(inputs.parameters["my-float-param"]) +``` + +Cast to string: + +``` +string(1) +``` + +Convert to a JSON string (needed for `withParam`): + +``` +toJson([1, 2]) +``` + +You can also use [Sprig functions](http://masterminds.github.io/sprig/): + +Trim a string: + +``` +sprig.trim(inputs.parameters["my-string-param"]) +``` + +!!! Warning In Sprig functions, errors are not often not raised. E.g. if `int` is used on an invalid value, it +returns `0`. Please review the Sprig documentation to understand which functions do and which do not. + +## Reference + +### All Templates + | Variable | Description| |----------|------------| | `inputs.parameters.`| Input parameter to a template | | `inputs.parameters`| All input parameters to a template as a JSON string | | `inputs.artifacts.` | Input artifact to a template | -## Steps Templates +### Steps Templates + | Variable | Description| |----------|------------| | `steps..id` | unique id of container step | @@ -51,7 +137,8 @@ The following variables are made available to reference various metadata of a wo | `steps..outputs.parameters.` | Output parameter of any previous step. When the previous step uses 'withItems' or 'withParams', this contains a JSON array of the output parameter values of each invocation | | `steps..outputs.artifacts.` | Output artifact of any previous step | -## DAG Templates +### DAG Templates + | Variable | Description| |----------|------------| | `tasks..id` | unique id of container task | @@ -65,7 +152,8 @@ The following variables are made available to reference various metadata of a wo | `tasks..outputs.parameters.` | Output parameter of any previous task. When the previous task uses 'withItems' or 'withParams', this contains a JSON array of the output parameter values of each invocation | | `tasks..outputs.artifacts.` | Output artifact of any previous task | -## Container/Script Templates +### Container/Script Templates + | Variable | Description| |----------|------------| | `pod.name` | Pod name of the container/script | @@ -74,13 +162,15 @@ The following variables are made available to reference various metadata of a wo | `outputs.artifacts..path` | Local path of the output artifact | | `outputs.parameters..path` | Local path of the output parameter | -## Loops (withItems / withParam) +### Loops (withItems / withParam) + | Variable | Description| |----------|------------| | `item` | Value of the item in a list | | `item.` | Field value of the item in a list of maps | -## Metrics +### Metrics + When emitting custom metrics in a `template`, special variables are available that allow self-reference to the current step. @@ -92,7 +182,8 @@ step. | `inputs.parameters.` | Input parameter of the metric-emitting template | | `outputs.parameters.` | Output parameter of the metric-emitting template | | `outputs.result` | Output result of the metric-emitting template | -| `resourcesDuration.{cpu,memory}` | Resources duration **in seconds**. Must be one of `resourcesDuration.cpu` or `resourcesDuration.memory`, if available. For more info, see the [Resource Duration](resource-duration.md) doc.| +| `resourcesDuration.{cpu,memory}` | Resources duration **in +seconds**. Must be one of `resourcesDuration.cpu` or `resourcesDuration.memory`, if available. For more info, see the [Resource Duration](resource-duration.md) doc.| ### Realtime Metrics @@ -101,12 +192,15 @@ real time, set `realtime: true` under `gauge` (note: only Gauge metrics allow fo currently available for real time emission: For `Workflow`-level metrics: + * `workflow.duration` For `Template`-level metrics: + * `duration` -## Global +### Global + | Variable | Description| |----------|------------| | `workflow.name` | Workflow name | @@ -124,7 +218,8 @@ For `Template`-level metrics: | `workflow.priority` | Workflow priority | | `workflow.duration` | Workflow duration estimate, may differ from actual duration by a couple of seconds | -## Exit Handler +### Exit Handler + | Variable | Description| |----------|------------| | `workflow.status` | Workflow status. One of: `Succeeded`, `Failed`, `Error` | diff --git a/examples/expression-tag-template-workflow.yaml b/examples/expression-tag-template-workflow.yaml new file mode 100644 index 000000000000..685a777be1bb --- /dev/null +++ b/examples/expression-tag-template-workflow.yaml @@ -0,0 +1,45 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: expression-tag-template- + labels: + workflows.argoproj.io/test: "true" + annotations: + # available in v3.1.0 + workflows.argoproj.io/version: ">= 3.1.0" + workflows.argoproj.io/verify.py: | + assert status["phase"] == "Succeeded" + assert nodes["task-0(0:3)"]["phase"] == "Succeeded" + assert nodes["task-0(0:3)"]["outputs"]["parameters"][0]["value"] == "hello 30 @ 2021" +spec: + entrypoint: main + templates: + - name: main + dag: + tasks: + - name: task-0 + template: pod-0 + arguments: + parameters: + - name: foo + value: "{{=item}}" + # withParam must be a JSON list encoded as a string, so you use `toJson` to perform the conversion + withParam: "{{=toJson(filter([1, 3], {# > 1}))}}" + - name: pod-0 + inputs: + parameters: + - name: foo + container: + image: argoproj/argosay:v2 + # in this example, we use `asInt` to cast a parameter (which are ALWAYS strings) to an int so we can + # multiple by 10 + args: + - echo + - | + hello {{=asInt(inputs.parameters.foo) * 10}} @ {{=sprig.date('2006', workflow.creationTimestamp)}} + - /output + outputs: + parameters: + - name: output + valueFrom: + path: /output \ No newline at end of file diff --git a/go.mod b/go.mod index 645944fb40b5..3a7b35184a4a 100644 --- a/go.mod +++ b/go.mod @@ -9,16 +9,18 @@ require ( github.com/Azure/go-autorest/autorest v0.11.1 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible + github.com/Masterminds/sprig v2.22.0+incompatible github.com/TwinProduction/go-color v0.0.3 github.com/aliyun/aliyun-oss-go-sdk v2.1.5+incompatible github.com/antonmedv/expr v1.8.8 github.com/argoproj/argo-events v1.2.0 - github.com/argoproj/pkg v0.6.0 + github.com/argoproj/pkg v0.7.0 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/blushft/go-diagrams v0.0.0-20201006005127-c78c821223d9 github.com/colinmarc/hdfs v1.1.4-0.20180805212432-9746310a4d31 github.com/coreos/go-oidc v2.2.1+incompatible github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect + github.com/doublerebel/bellows v0.0.0-20160303004610-f177d92a03d3 github.com/emicklei/go-restful v2.15.0+incompatible // indirect github.com/evanphx/json-patch v4.9.0+incompatible github.com/fatih/structs v1.1.0 // indirect diff --git a/go.sum b/go.sum index 1a070c190ea7..3ac5b97e0ad8 100644 --- a/go.sum +++ b/go.sum @@ -80,8 +80,11 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -121,8 +124,8 @@ github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAF github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= github.com/argoproj/argo-events v1.2.0 h1:CjF8hVUkeflhaOt9uWjJK6ai6b4pw0CCUUmOnmhWnNY= github.com/argoproj/argo-events v1.2.0/go.mod h1:eY+egQNBLXAz/AF4mqgHsMMa4Aur7frHjUfBg+RpX04= -github.com/argoproj/pkg v0.6.0 h1:q6413Dtl8keWXjScb99GD5tzTuZugXQeTnd/+mRj3yI= -github.com/argoproj/pkg v0.6.0/go.mod h1:DmT4fN1ihGS0VBa0kHwpL+hoRAi8WdMFsUDa1QTZd+M= +github.com/argoproj/pkg v0.7.0 h1:fhCONBGjX5FuB+AlpXHc1Lo3FGuNDB7Zr+Q7S0my5ck= +github.com/argoproj/pkg v0.7.0/go.mod h1:ra+bQPmbVAoEL+gYSKesuigt4m49i3Qa3mE/xQcjCiA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -212,6 +215,8 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/doublerebel/bellows v0.0.0-20160303004610-f177d92a03d3 h1:7nllYTGLnq4CqBL27lV6oNfXzM2tJ2mrKF8E+aBXOV0= +github.com/doublerebel/bellows v0.0.0-20160303004610-f177d92a03d3/go.mod h1:v/MTKot4he5oRHGirOYGN4/hEOONNnWtDBLAzllSGMw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -539,6 +544,7 @@ github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pN github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.3.0 h1:gvV6jG9dTgFEncxo+AF7PH6MZXi/vZl25owA/8Dg8Wo= github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/strcase v0.1.1/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -668,6 +674,7 @@ github.com/minio/minio-go/v7 v7.0.2/go.mod h1:dJ80Mv2HeGkYLH1sqS/ksz07ON6csH3S6J github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -684,6 +691,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -719,6 +727,8 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 h1:Yl0tPBa8QPjGmesFh1D0rDy+q1Twx6FyU7VWHi8wZbI= +github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852/go.mod h1:eqOVx5Vwu4gd2mmMZvVZsgIqNSaW3xxRThUJ0k/TPk4= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/mkdocs.yml b/mkdocs.yml index 411994859be8..0299a47c2da5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -140,6 +140,7 @@ nav: - scaling.md - cost-optimisation.md - windows.md + - environment-variables.md - Developer Guide: - CONTRIBUTING.md - architecture.md diff --git a/util/expr/env/env.go b/util/expr/env/env.go new file mode 100644 index 000000000000..9952727491a5 --- /dev/null +++ b/util/expr/env/env.go @@ -0,0 +1,27 @@ +package env + +import ( + "encoding/json" + + "github.com/Masterminds/sprig" + exprpkg "github.com/argoproj/pkg/expr" + "github.com/doublerebel/bellows" +) + +func GetFuncMap(m map[string]interface{}) map[string]interface{} { + env := bellows.Expand(m) + for k, v := range exprpkg.GetExprEnvFunctionMap() { + env[k] = v + } + env["toJson"] = toJson + env["sprig"] = sprig.GenericFuncMap() + return env +} + +func toJson(v interface{}) string { + output, err := json.Marshal(v) + if err != nil { + panic(err) + } + return string(output) +} diff --git a/util/json/fix.go b/util/json/fix.go new file mode 100644 index 000000000000..6a7ee41ef451 --- /dev/null +++ b/util/json/fix.go @@ -0,0 +1,11 @@ +package json + +import "strings" + +func Fix(s string) string { + // https://stackoverflow.com/questions/28595664/how-to-stop-json-marshal-from-escaping-and/28596225 + s = strings.Replace(s, "\\u003c", "<", -1) + s = strings.Replace(s, "\\u003e", ">", -1) + s = strings.Replace(s, "\\u0026", "&", -1) + return s +} diff --git a/util/json/fix_test.go b/util/json/fix_test.go new file mode 100644 index 000000000000..ba066e35c37d --- /dev/null +++ b/util/json/fix_test.go @@ -0,0 +1,13 @@ +package json + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Fix(t *testing.T) { + assert.Equal(t, "<", Fix("\\u003c")) + assert.Equal(t, ">", Fix("\\u003e")) + assert.Equal(t, "&", Fix("\\u0026")) +} diff --git a/util/template/expression_template.go b/util/template/expression_template.go new file mode 100644 index 000000000000..39bd6f01a82f --- /dev/null +++ b/util/template/expression_template.go @@ -0,0 +1,37 @@ +package template + +import ( + "fmt" + "io" + "os" + + "github.com/antonmedv/expr" +) + +func init() { + if os.Getenv("EXPRESSION_TEMPLATES") != "false" { + registerKind(kindExpression) + } +} + +func expressionReplace(w io.Writer, expression string, env map[string]interface{}, allowUnresolved bool) (int, error) { + result, err := expr.Eval(expression, env) + if (err != nil || result == nil) && allowUnresolved { // result is also un-resolved, and any error can be unresolved + return w.Write([]byte(fmt.Sprintf("{{%s%s}}", kindExpression, expression))) + } + if err != nil { + return 0, fmt.Errorf("failed to evaluate expression: %w", err) + } + if result == nil { + return 0, fmt.Errorf("failed to evaluate expression %q", expression) + } + return w.Write([]byte(fmt.Sprintf("%v", result))) +} + +func envMap(replaceMap map[string]string) map[string]interface{} { + envMap := make(map[string]interface{}) + for k, v := range replaceMap { + envMap[k] = v + } + return envMap +} diff --git a/util/template/kind.go b/util/template/kind.go new file mode 100644 index 000000000000..989464f41300 --- /dev/null +++ b/util/template/kind.go @@ -0,0 +1,29 @@ +package template + +import ( + "strings" + + jsonutil "github.com/argoproj/argo-workflows/v3/util/json" +) + +type kind = string // defines the prefix symbol that determines the syntax of the tag + +const ( + kindSimple kind = "" // default is simple, i.e. no prefix + kindExpression kind = "=" +) + +var kinds []kind + +func registerKind(k kind) { + kinds = append(kinds, k) +} + +func parseTag(tag string) (kind, string) { + for _, k := range kinds { + if strings.HasPrefix(tag, k) { + return k, jsonutil.Fix(strings.TrimPrefix(tag, k)) + } + } + return kindSimple, tag +} diff --git a/util/template/kind_test.go b/util/template/kind_test.go new file mode 100644 index 000000000000..82106e3c07b8 --- /dev/null +++ b/util/template/kind_test.go @@ -0,0 +1,20 @@ +package template + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_parseTag(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + kind, tag := parseTag("tag") + assert.Equal(t, kindSimple, kind) + assert.Equal(t, "tag", tag) + }) + t.Run("Expression", func(t *testing.T) { + kind, tag := parseTag("=tag") + assert.Equal(t, kindExpression, kind) + assert.Equal(t, "tag", tag) + }) +} diff --git a/util/template/replace.go b/util/template/replace.go new file mode 100644 index 000000000000..79950dcc1c85 --- /dev/null +++ b/util/template/replace.go @@ -0,0 +1,9 @@ +package template + +func Replace(s string, replaceMap map[string]string, allowUnresolved bool) (string, error) { + t, err := NewTemplate(s) + if err != nil { + return "", err + } + return t.Replace(replaceMap, allowUnresolved) +} diff --git a/util/template/replace_test.go b/util/template/replace_test.go new file mode 100644 index 000000000000..8bfb34e024d8 --- /dev/null +++ b/util/template/replace_test.go @@ -0,0 +1,121 @@ +package template + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Replace(t *testing.T) { + t.Run("InvailedTemplate", func(t *testing.T) { + _, err := Replace("{{", nil, false) + assert.Error(t, err) + }) + t.Run("Simple", func(t *testing.T) { + t.Run("Valid", func(t *testing.T) { + r, err := Replace("{{foo}}", map[string]string{"foo": "bar"}, false) + assert.NoError(t, err) + assert.Equal(t, "bar", r) + }) + t.Run("Unresolved", func(t *testing.T) { + t.Run("Allowed", func(t *testing.T) { + _, err := Replace("{{foo}}", nil, true) + assert.NoError(t, err) + }) + t.Run("Disallowed", func(t *testing.T) { + _, err := Replace("{{foo}}", nil, false) + assert.EqualError(t, err, "failed to resolve {{foo}}") + }) + }) + }) + t.Run("Expression", func(t *testing.T) { + t.Run("Valid", func(t *testing.T) { + r, err := Replace("{{=foo}}", map[string]string{"foo": "bar"}, false) + assert.NoError(t, err) + assert.Equal(t, "bar", r) + }) + t.Run("Unresolved", func(t *testing.T) { + t.Run("Allowed", func(t *testing.T) { + _, err := Replace("{{=foo}}", nil, true) + assert.NoError(t, err) + }) + t.Run("Disallowed", func(t *testing.T) { + _, err := Replace("{{=foo}}", nil, false) + assert.EqualError(t, err, "failed to evaluate expression \"foo\"") + }) + }) + t.Run("Error", func(t *testing.T) { + _, err := Replace("{{=!}}", nil, false) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "failed to evaluate expression") + } + }) + }) +} + +func TestNestedReplaceString(t *testing.T) { + replaceMap := map[string]string{"inputs.parameters.message": "hello world"} + + test := `{{- with secret "{{inputs.parameters.message}}" -}} + {{ .Data.data.gitcreds }} + {{- end }}` + replacement, err := Replace(test, replaceMap, true) + if assert.NoError(t, err) { + assert.Equal(t, "{{- with secret \"hello world\" -}}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) + } + + test = `{{- with {{ secret "{{inputs.parameters.message}}" -}} + {{ .Data.data.gitcreds }} + {{- end }}` + + replacement, err = Replace(test, replaceMap, true) + if assert.NoError(t, err) { + assert.Equal(t, "{{- with {{ secret \"hello world\" -}}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) + } + + test = `{{- with {{ secret "{{inputs.parameters.message}}" -}} }} + {{ .Data.data.gitcreds }} + {{- end }}` + + replacement, err = Replace(test, replaceMap, true) + if assert.NoError(t, err) { + assert.Equal(t, "{{- with {{ secret \"hello world\" -}} }}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) + } + + test = `{{- with secret "{{inputs.parameters.message}}" -}} }} + {{ .Data.data.gitcreds }} + {{- end }}` + + replacement, err = Replace(test, replaceMap, true) + if assert.NoError(t, err) { + assert.Equal(t, "{{- with secret \"hello world\" -}} }}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) + } + + test = `{{- with {{ {{ }} secret "{{inputs.parameters.message}}" -}} }} + {{ .Data.data.gitcreds }} + {{- end }}` + + replacement, err = Replace(test, replaceMap, true) + if assert.NoError(t, err) { + assert.Equal(t, "{{- with {{ {{ }} secret \"hello world\" -}} }}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) + } + + test = `{{- with {{ {{ }} secret "{{does-not-exist}}" -}} }} + {{ .Data.data.gitcreds }} + {{- end }}` + + replacement, err = Replace(test, replaceMap, true) + if assert.NoError(t, err) { + assert.Equal(t, test, replacement) + } +} + +func TestReplaceStringWithWhiteSpace(t *testing.T) { + replaceMap := map[string]string{"inputs.parameters.message": "hello world"} + + test := `{{ inputs.parameters.message }}` + replacement, err := Replace(test, replaceMap, true) + if assert.NoError(t, err) { + assert.Equal(t, "hello world", replacement) + } +} diff --git a/util/template/resolve_var.go b/util/template/resolve_var.go new file mode 100644 index 000000000000..954faa8dc20c --- /dev/null +++ b/util/template/resolve_var.go @@ -0,0 +1,31 @@ +package template + +import ( + "strings" + + "github.com/antonmedv/expr" + + "github.com/argoproj/argo-workflows/v3/errors" +) + +func ResolveVar(s string, m map[string]interface{}) (interface{}, error) { + tag := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(s, prefix), suffix)) + kind, expression := parseTag(tag) + switch kind { + case kindExpression: + result, err := expr.Eval(expression, m) + if err != nil { + return nil, errors.Errorf(errors.CodeBadRequest, "Invalid expression: %q", expression) + } + if result == nil { + return nil, errors.Errorf(errors.CodeBadRequest, "Unable to resolve: %q", tag) + } + return result, nil + default: + v, ok := m[tag] + if !ok { + return nil, errors.Errorf(errors.CodeBadRequest, "Unable to resolve: %q", tag) + } + return v, nil + } +} diff --git a/util/template/resolve_var_test.go b/util/template/resolve_var_test.go new file mode 100644 index 000000000000..6ca9219b11ba --- /dev/null +++ b/util/template/resolve_var_test.go @@ -0,0 +1,41 @@ +package template + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_ResolveVar(t *testing.T) { + t.Run("Simple", func(t *testing.T) { + t.Run("Valid", func(t *testing.T) { + v, err := ResolveVar("{{foo}}", map[string]interface{}{"foo": "bar"}) + assert.NoError(t, err) + assert.Equal(t, "bar", v) + }) + t.Run("Whitespace", func(t *testing.T) { + v, err := ResolveVar("{{ foo }}", map[string]interface{}{"foo": "bar"}) + assert.NoError(t, err) + assert.Equal(t, "bar", v) + }) + t.Run("Unresolved", func(t *testing.T) { + _, err := ResolveVar("{{foo}}", nil) + assert.EqualError(t, err, "Unable to resolve: \"foo\"") + }) + }) + t.Run("Expression", func(t *testing.T) { + t.Run("Valid", func(t *testing.T) { + v, err := ResolveVar("{{=foo}}", map[string]interface{}{"foo": "bar"}) + assert.NoError(t, err) + assert.Equal(t, "bar", v) + }) + t.Run("Unresolved", func(t *testing.T) { + _, err := ResolveVar("{{=foo}}", nil) + assert.EqualError(t, err, "Unable to resolve: \"=foo\"") + }) + t.Run("Error", func(t *testing.T) { + _, err := ResolveVar("{{=!}}", nil) + assert.EqualError(t, err, "Invalid expression: \"!\"") + }) + }) +} diff --git a/util/template/simple_template.go b/util/template/simple_template.go new file mode 100644 index 000000000000..89bd9ddb0731 --- /dev/null +++ b/util/template/simple_template.go @@ -0,0 +1,36 @@ +package template + +import ( + "fmt" + "io" + "strconv" + "strings" + + "github.com/argoproj/argo-workflows/v3/errors" +) + +func simpleReplace(w io.Writer, tag string, replaceMap map[string]string, allowUnresolved bool) (int, error) { + replacement, ok := replaceMap[strings.TrimSpace(tag)] + if !ok { + // Attempt to resolve nested tags, if possible + if index := strings.LastIndex(tag, "{{"); index > 0 { + nestedTagPrefix := tag[:index] + nestedTag := tag[index+2:] + if replacement, ok := replaceMap[nestedTag]; ok { + replacement = strconv.Quote(replacement) + replacement = replacement[1 : len(replacement)-1] + return w.Write([]byte("{{" + nestedTagPrefix + replacement)) + } + } + if allowUnresolved { + // just write the same string back + return w.Write([]byte(fmt.Sprintf("{{%s}}", tag))) + } + return 0, errors.Errorf(errors.CodeBadRequest, "failed to resolve {{%s}}", tag) + } + // The following escapes any special characters (e.g. newlines, tabs, etc...) + // in preparation for substitution + replacement = strconv.Quote(replacement) + replacement = replacement[1 : len(replacement)-1] + return w.Write([]byte(replacement)) +} diff --git a/util/template/template.go b/util/template/template.go new file mode 100644 index 000000000000..273c92d67b78 --- /dev/null +++ b/util/template/template.go @@ -0,0 +1,46 @@ +package template + +import ( + "bytes" + "io" + + "github.com/valyala/fasttemplate" + + exprenv "github.com/argoproj/argo-workflows/v3/util/expr/env" +) + +const ( + prefix = "{{" + suffix = "}}" +) + +type Template interface { + Replace(replaceMap map[string]string, allowUnresolved bool) (string, error) +} + +func NewTemplate(s string) (Template, error) { + template, err := fasttemplate.NewTemplate(s, prefix, suffix) + if err != nil { + return nil, err + } + return &impl{template}, nil +} + +type impl struct { + *fasttemplate.Template +} + +func (t *impl) Replace(replaceMap map[string]string, allowUnresolved bool) (string, error) { + env := exprenv.GetFuncMap(envMap(replaceMap)) + replacedTmpl := &bytes.Buffer{} + _, err := t.Template.ExecuteFunc(replacedTmpl, func(w io.Writer, tag string) (int, error) { + kind, expression := parseTag(tag) + switch kind { + case kindExpression: + return expressionReplace(w, expression, env, allowUnresolved) + default: + return simpleReplace(w, tag, replaceMap, allowUnresolved) + } + }) + return replacedTmpl.String(), err +} diff --git a/util/template/validate.go b/util/template/validate.go new file mode 100644 index 000000000000..08a043be8d2c --- /dev/null +++ b/util/template/validate.go @@ -0,0 +1,25 @@ +package template + +import ( + "io" + "io/ioutil" + + "github.com/valyala/fasttemplate" +) + +func Validate(s string, validator func(tag string) error) error { + t, err := fasttemplate.NewTemplate(s, prefix, suffix) + if err != nil { + return err + } + _, err = t.ExecuteFunc(ioutil.Discard, func(w io.Writer, tag string) (int, error) { + kind, _ := parseTag(tag) + switch kind { + case kindExpression: + return 0, nil // we do not validate expression templates + default: + return 0, validator(tag) + } + }) + return err +} diff --git a/util/template/validate_test.go b/util/template/validate_test.go new file mode 100644 index 000000000000..65fa77c91835 --- /dev/null +++ b/util/template/validate_test.go @@ -0,0 +1,27 @@ +package template + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_Validate(t *testing.T) { + t.Run("InvalidTemplate", func(t *testing.T) { + err := Validate("{{", func(tag string) error { return fmt.Errorf("") }) + assert.Error(t, err) + }) + t.Run("InvalidTag", func(t *testing.T) { + err := Validate("{{foo}}", func(tag string) error { return fmt.Errorf(tag) }) + assert.EqualError(t, err, "foo") + }) + t.Run("Simple", func(t *testing.T) { + err := Validate("{{foo}}", func(tag string) error { return nil }) + assert.NoError(t, err) + }) + t.Run("Expression", func(t *testing.T) { + err := Validate("{{=foo}}", func(tag string) error { return fmt.Errorf(tag) }) + assert.NoError(t, err) + }) +} diff --git a/workflow/common/util.go b/workflow/common/util.go index 81c8dab98f9c..536170ea1597 100644 --- a/workflow/common/util.go +++ b/workflow/common/util.go @@ -5,17 +5,14 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "os/exec" "runtime" - "strconv" "strings" "time" "github.com/gorilla/websocket" log "github.com/sirupsen/logrus" - "github.com/valyala/fasttemplate" apiv1 "k8s.io/api/core/v1" apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,6 +26,7 @@ import ( wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/util" errorsutil "github.com/argoproj/argo-workflows/v3/util/errors" + "github.com/argoproj/argo-workflows/v3/util/template" waitutil "github.com/argoproj/argo-workflows/v3/util/wait" ) @@ -180,11 +178,7 @@ func SubstituteParams(tmpl *wfv1.Template, globalParams, localParams Parameters) } // First replace globals & locals, then replace inputs because globals could be referenced in the inputs replaceMap := globalParams.Merge(localParams) - fstTmpl, err := fasttemplate.NewTemplate(string(tmplBytes), "{{", "}}") - if err != nil { - return nil, fmt.Errorf("unable to parse argo variable: %w", err) - } - globalReplacedTmplStr, err := Replace(fstTmpl, replaceMap, true) + globalReplacedTmplStr, err := template.Replace(string(tmplBytes), replaceMap, true) if err != nil { return nil, err } @@ -223,11 +217,7 @@ func SubstituteParams(tmpl *wfv1.Template, globalParams, localParams Parameters) } } - fstTmpl, err = fasttemplate.NewTemplate(globalReplacedTmplStr, "{{", "}}") - if err != nil { - return nil, fmt.Errorf("unable to parse argo variable: %w", err) - } - s, err := Replace(fstTmpl, replaceMap, true) + s, err := template.Replace(globalReplacedTmplStr, replaceMap, true) if err != nil { return nil, err } @@ -239,43 +229,6 @@ func SubstituteParams(tmpl *wfv1.Template, globalParams, localParams Parameters) return &newTmpl, nil } -// Replace executes basic string substitution of a template with replacement values. -// allowUnresolved indicates whether or not it is acceptable to have unresolved variables -// remaining in the substituted template. -func Replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allowUnresolved bool) (string, error) { - var unresolvedErr error - replacedTmpl := fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { - replacement, ok := replaceMap[strings.TrimSpace(tag)] - if !ok { - // Attempt to resolve nested tags, if possible - if index := strings.LastIndex(tag, "{{"); index > 0 { - nestedTagPrefix := tag[:index] - nestedTag := tag[index+2:] - if replacement, ok := replaceMap[nestedTag]; ok { - replacement = strconv.Quote(replacement) - replacement = replacement[1 : len(replacement)-1] - return w.Write([]byte("{{" + nestedTagPrefix + replacement)) - } - } - if allowUnresolved { - // just write the same string back - return w.Write([]byte(fmt.Sprintf("{{%s}}", tag))) - } - unresolvedErr = errors.Errorf(errors.CodeBadRequest, "failed to resolve {{%s}}", tag) - return 0, nil - } - // The following escapes any special characters (e.g. newlines, tabs, etc...) - // in preparation for substitution - replacement = strconv.Quote(replacement) - replacement = replacement[1 : len(replacement)-1] - return w.Write([]byte(replacement)) - }) - if unresolvedErr != nil { - return "", unresolvedErr - } - return replacedTmpl, nil -} - // RunCommand is a convenience function to run/log a command and log the stderr upon failure func RunCommand(name string, arg ...string) ([]byte, error) { cmd := exec.Command(name, arg...) diff --git a/workflow/common/util_test.go b/workflow/common/util_test.go index fa13c319fb5f..86ddbac75d42 100644 --- a/workflow/common/util_test.go +++ b/workflow/common/util_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/valyala/fasttemplate" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -88,86 +87,3 @@ func TestDeletePod(t *testing.T) { assert.NoError(t, err) }) } - -func TestNestedReplaceString(t *testing.T) { - replaceMap := map[string]string{"inputs.parameters.message": "hello world"} - - test := `{{- with secret "{{inputs.parameters.message}}" -}} - {{ .Data.data.gitcreds }} - {{- end }}` - fstTmpl, err := fasttemplate.NewTemplate(test, "{{", "}}") - if assert.NoError(t, err) { - replacement, err := Replace(fstTmpl, replaceMap, true) - if assert.NoError(t, err) { - assert.Equal(t, "{{- with secret \"hello world\" -}}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) - } - } - - test = `{{- with {{ secret "{{inputs.parameters.message}}" -}} - {{ .Data.data.gitcreds }} - {{- end }}` - fstTmpl, err = fasttemplate.NewTemplate(test, "{{", "}}") - if assert.NoError(t, err) { - replacement, err := Replace(fstTmpl, replaceMap, true) - if assert.NoError(t, err) { - assert.Equal(t, "{{- with {{ secret \"hello world\" -}}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) - } - } - - test = `{{- with {{ secret "{{inputs.parameters.message}}" -}} }} - {{ .Data.data.gitcreds }} - {{- end }}` - fstTmpl, err = fasttemplate.NewTemplate(test, "{{", "}}") - if assert.NoError(t, err) { - replacement, err := Replace(fstTmpl, replaceMap, true) - if assert.NoError(t, err) { - assert.Equal(t, "{{- with {{ secret \"hello world\" -}} }}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) - } - } - - test = `{{- with secret "{{inputs.parameters.message}}" -}} }} - {{ .Data.data.gitcreds }} - {{- end }}` - fstTmpl, err = fasttemplate.NewTemplate(test, "{{", "}}") - if assert.NoError(t, err) { - replacement, err := Replace(fstTmpl, replaceMap, true) - if assert.NoError(t, err) { - assert.Equal(t, "{{- with secret \"hello world\" -}} }}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) - } - } - - test = `{{- with {{ {{ }} secret "{{inputs.parameters.message}}" -}} }} - {{ .Data.data.gitcreds }} - {{- end }}` - fstTmpl, err = fasttemplate.NewTemplate(test, "{{", "}}") - if assert.NoError(t, err) { - replacement, err := Replace(fstTmpl, replaceMap, true) - if assert.NoError(t, err) { - assert.Equal(t, "{{- with {{ {{ }} secret \"hello world\" -}} }}\n {{ .Data.data.gitcreds }}\n {{- end }}", replacement) - } - } - - test = `{{- with {{ {{ }} secret "{{does-not-exist}}" -}} }} - {{ .Data.data.gitcreds }} - {{- end }}` - fstTmpl, err = fasttemplate.NewTemplate(test, "{{", "}}") - if assert.NoError(t, err) { - replacement, err := Replace(fstTmpl, replaceMap, true) - if assert.NoError(t, err) { - assert.Equal(t, test, replacement) - } - } -} - -func TestReplaceStringWithWhiteSpace(t *testing.T) { - replaceMap := map[string]string{"inputs.parameters.message": "hello world"} - - test := `{{ inputs.parameters.message }}` - fstTmpl, err := fasttemplate.NewTemplate(test, "{{", "}}") - if assert.NoError(t, err) { - replacement, err := Replace(fstTmpl, replaceMap, true) - if assert.NoError(t, err) { - assert.Equal(t, "hello world", replacement) - } - } -} diff --git a/workflow/controller/dag.go b/workflow/controller/dag.go index fbd575bf46b9..839f860d88e1 100644 --- a/workflow/controller/dag.go +++ b/workflow/controller/dag.go @@ -9,10 +9,10 @@ import ( "time" "github.com/antonmedv/expr" - "github.com/valyala/fasttemplate" "github.com/argoproj/argo-workflows/v3/errors" wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/argoproj/argo-workflows/v3/util/template" "github.com/argoproj/argo-workflows/v3/workflow/common" "github.com/argoproj/argo-workflows/v3/workflow/templateresolution" ) @@ -554,12 +554,7 @@ func (woc *wfOperationCtx) resolveDependencyReferences(dagCtx *dagContext, task if err != nil { return nil, errors.InternalWrapError(err) } - fstTmpl, err := fasttemplate.NewTemplate(string(taskBytes), "{{", "}}") - if err != nil { - return nil, fmt.Errorf("unable to parse argo variable: %w", err) - } - - newTaskStr, err := common.Replace(fstTmpl, woc.globalParams.Merge(scope.getParameters()), true) + newTaskStr, err := template.Replace(string(taskBytes), woc.globalParams.Merge(scope.getParameters()), true) if err != nil { return nil, err } @@ -659,14 +654,14 @@ func expandTask(task wfv1.DAGTask) ([]wfv1.DAGTask, error) { task.WithParam = "" task.WithSequence = nil - fstTmpl, err := fasttemplate.NewTemplate(string(taskBytes), "{{", "}}") + tmpl, err := template.NewTemplate(string(taskBytes)) if err != nil { return nil, fmt.Errorf("unable to parse argo variable: %w", err) } expandedTasks := make([]wfv1.DAGTask, 0) for i, item := range items { var newTask wfv1.DAGTask - newTaskName, err := processItem(fstTmpl, task.Name, i, item, &newTask) + newTaskName, err := processItem(tmpl, task.Name, i, item, &newTask) if err != nil { return nil, err } diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go index 98377773ad7d..134ded517aab 100644 --- a/workflow/controller/operator.go +++ b/workflow/controller/operator.go @@ -20,7 +20,6 @@ import ( "github.com/argoproj/pkg/strftime" jsonpatch "github.com/evanphx/json-patch" log "github.com/sirupsen/logrus" - "github.com/valyala/fasttemplate" apiv1 "k8s.io/api/core/v1" policyv1beta "k8s.io/api/policy/v1beta1" apierr "k8s.io/apimachinery/pkg/api/errors" @@ -44,6 +43,7 @@ import ( "github.com/argoproj/argo-workflows/v3/util/intstr" "github.com/argoproj/argo-workflows/v3/util/resource" "github.com/argoproj/argo-workflows/v3/util/retry" + "github.com/argoproj/argo-workflows/v3/util/template" waitutil "github.com/argoproj/argo-workflows/v3/util/wait" "github.com/argoproj/argo-workflows/v3/workflow/common" controllercache "github.com/argoproj/argo-workflows/v3/workflow/controller/cache" @@ -2727,7 +2727,7 @@ func parseStringToDuration(durationString string) (time.Duration, error) { return suspendDuration, nil } -func processItem(fstTmpl *fasttemplate.Template, name string, index int, item wfv1.Item, obj interface{}) (string, error) { +func processItem(tmpl template.Template, name string, index int, item wfv1.Item, obj interface{}) (string, error) { replaceMap := make(map[string]string) var newName string @@ -2768,7 +2768,7 @@ func processItem(fstTmpl *fasttemplate.Template, name string, index int, item wf default: return "", errors.Errorf(errors.CodeBadRequest, "withItems[%d] expected string, number, list, or map. received: %v", index, item) } - newStepStr, err := common.Replace(fstTmpl, replaceMap, false) + newStepStr, err := tmpl.Replace(replaceMap, false) if err != nil { return "", err } @@ -2848,11 +2848,7 @@ func (woc *wfOperationCtx) substituteParamsInVolumes(params map[string]string) e if err != nil { return errors.InternalWrapError(err) } - fstTmpl, err := fasttemplate.NewTemplate(string(volumesBytes), "{{", "}}") - if err != nil { - return fmt.Errorf("unable to parse argo variable: %w", err) - } - newVolumesStr, err := common.Replace(fstTmpl, params, true) + newVolumesStr, err := template.Replace(string(volumesBytes), params, true) if err != nil { return err } @@ -2920,12 +2916,7 @@ func (woc *wfOperationCtx) computeMetrics(metricList []*wfv1.Prometheus, localSc woc.reportMetricEmissionError(fmt.Sprintf("unable to substitute parameters for metric '%s' (marshal): %s", metricTmpl.Name, err)) continue } - fstTmpl, err := fasttemplate.NewTemplate(string(metricTmplBytes), "{{", "}}") - if err != nil { - woc.reportMetricEmissionError(fmt.Sprintf("unable to parse argo variable for metric '%s': %s", metricTmpl.Name, err)) - continue - } - replacedValue, err := common.Replace(fstTmpl, localScope, false) + replacedValue, err := template.Replace(string(metricTmplBytes), localScope, false) if err != nil { woc.reportMetricEmissionError(fmt.Sprintf("unable to substitute parameters for metric '%s': %s", metricTmpl.Name, err)) continue @@ -2980,12 +2971,7 @@ func (woc *wfOperationCtx) computeMetrics(metricList []*wfv1.Prometheus, localSc metricSpec := metricTmpl.DeepCopy() // Finally substitute value parameters - fstTmpl, err = fasttemplate.NewTemplate(metricSpec.GetValueString(), "{{", "}}") - if err != nil { - woc.reportMetricEmissionError(fmt.Sprintf("unable to parse argo variable for metric '%s': %s", metricTmpl.Name, err)) - continue - } - replacedValue, err := common.Replace(fstTmpl, localScope, false) + replacedValue, err := template.Replace(metricSpec.GetValueString(), localScope, false) if err != nil { woc.reportMetricEmissionError(fmt.Sprintf("unable to substitute parameters for metric '%s': %s", metricSpec.Name, err)) continue diff --git a/workflow/controller/operator_test.go b/workflow/controller/operator_test.go index 597f604c341a..2f8e31c7f93c 100644 --- a/workflow/controller/operator_test.go +++ b/workflow/controller/operator_test.go @@ -17,7 +17,6 @@ import ( dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/valyala/fasttemplate" apiv1 "k8s.io/api/core/v1" apierr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -34,6 +33,7 @@ import ( "github.com/argoproj/argo-workflows/v3/test" testutil "github.com/argoproj/argo-workflows/v3/test/util" intstrutil "github.com/argoproj/argo-workflows/v3/util/intstr" + "github.com/argoproj/argo-workflows/v3/util/template" "github.com/argoproj/argo-workflows/v3/workflow/common" "github.com/argoproj/argo-workflows/v3/workflow/controller/cache" hydratorfake "github.com/argoproj/argo-workflows/v3/workflow/hydrator/fake" @@ -5092,15 +5092,13 @@ func Test_processItem(t *testing.T) { } taskBytes, err := json.Marshal(task) assert.NoError(t, err) - fstTmpl, err := fasttemplate.NewTemplate(string(taskBytes), "{{", "}}") - assert.NoError(t, err) - var items []wfv1.Item err = json.Unmarshal([]byte(task.WithParam), &items) assert.NoError(t, err) var newTask wfv1.DAGTask - newTaskName, err := processItem(fstTmpl, "task-name", 0, items[0], &newTask) + tmpl, _ := template.NewTemplate(string(taskBytes)) + newTaskName, err := processItem(tmpl, "task-name", 0, items[0], &newTask) if assert.NoError(t, err) { assert.Equal(t, `task-name(0:json:{"number":2,"string":"foo","list":[0,"1"]},list:[0,"1"],number:2,string:foo)`, newTaskName) } diff --git a/workflow/controller/scope.go b/workflow/controller/scope.go index 864605f27de5..89217f0f6617 100644 --- a/workflow/controller/scope.go +++ b/workflow/controller/scope.go @@ -1,12 +1,9 @@ package controller import ( - "strings" - - "github.com/valyala/fasttemplate" - "github.com/argoproj/argo-workflows/v3/errors" wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/argoproj/argo-workflows/v3/util/template" "github.com/argoproj/argo-workflows/v3/workflow/common" ) @@ -38,24 +35,16 @@ func (s *wfScope) addArtifactToScope(key string, artifact wfv1.Artifact) { // resolveVar resolves a parameter or artifact func (s *wfScope) resolveVar(v string) (interface{}, error) { - v = strings.TrimPrefix(v, "{{") - v = strings.TrimSuffix(v, "}}") - v = strings.TrimSpace(v) - parts := strings.Split(v, ".") - prefix := parts[0] - switch prefix { - case "steps", "tasks", "workflow": - val, ok := s.scope[v] - if ok { - return val, nil - } - case "inputs": - art := s.tmpl.Inputs.GetArtifactByName(parts[2]) - if art != nil { - return *art, nil + m := make(map[string]interface{}) + for k, v := range s.scope { + m[k] = v + } + if s.tmpl != nil { + for _, a := range s.tmpl.Inputs.Artifacts { + m["inputs.artifacts."+a.Name] = a // special case for artifacts } } - return nil, errors.Errorf(errors.CodeBadRequest, "Unable to resolve: {{%s}}", v) + return template.ResolveVar(v, m) } func (s *wfScope) resolveParameter(v string) (string, error) { @@ -77,12 +66,11 @@ func (s *wfScope) resolveArtifact(v string, subPath string) (*wfv1.Artifact, err } valArt, ok := val.(wfv1.Artifact) if !ok { - return nil, errors.Errorf(errors.CodeBadRequest, "Variable {{%s}} is not an artifact", v) + return nil, errors.Errorf(errors.CodeBadRequest, "Variable {{%s}} is not an artifact: %q", v, val) } if subPath != "" { - fstTmpl := fasttemplate.New(subPath, "{{", "}}") - resolvedSubPath, err := common.Replace(fstTmpl, s.getParameters(), true) + resolvedSubPath, err := template.Replace(subPath, s.getParameters(), true) if err != nil { return nil, err } diff --git a/workflow/controller/steps.go b/workflow/controller/steps.go index 1344285f2900..785a45f95f7e 100644 --- a/workflow/controller/steps.go +++ b/workflow/controller/steps.go @@ -8,10 +8,10 @@ import ( "time" "github.com/Knetic/govaluate" - "github.com/valyala/fasttemplate" "github.com/argoproj/argo-workflows/v3/errors" wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/argoproj/argo-workflows/v3/util/template" "github.com/argoproj/argo-workflows/v3/workflow/common" "github.com/argoproj/argo-workflows/v3/workflow/templateresolution" ) @@ -353,12 +353,7 @@ func (woc *wfOperationCtx) resolveReferences(stepGroup []wfv1.WorkflowStep, scop if err != nil { return nil, errors.InternalWrapError(err) } - fstTmpl, err := fasttemplate.NewTemplate(string(stepBytes), "{{", "}}") - if err != nil { - return nil, fmt.Errorf("unable to parse argo variable: %w", err) - } - - newStepStr, err := common.Replace(fstTmpl, woc.globalParams.Merge(scope.getParameters()), true) + newStepStr, err := template.Replace(string(stepBytes), woc.globalParams.Merge(scope.getParameters()), true) if err != nil { return nil, err } @@ -468,14 +463,14 @@ func (woc *wfOperationCtx) expandStep(step wfv1.WorkflowStep) ([]wfv1.WorkflowSt if err != nil { return nil, errors.InternalWrapError(err) } - fstTmpl, err := fasttemplate.NewTemplate(string(stepBytes), "{{", "}}") + t, err := template.NewTemplate(string(stepBytes)) if err != nil { return nil, fmt.Errorf("unable to parse argo variable: %w", err) } for i, item := range items { var newStep wfv1.WorkflowStep - newStepName, err := processItem(fstTmpl, step.Name, i, item, &newStep) + newStepName, err := processItem(t, step.Name, i, item, &newStep) if err != nil { return nil, err } diff --git a/workflow/controller/workflowpod.go b/workflow/controller/workflowpod.go index 0bcd8290d705..673411f02938 100644 --- a/workflow/controller/workflowpod.go +++ b/workflow/controller/workflowpod.go @@ -4,14 +4,12 @@ import ( "context" "encoding/json" "fmt" - "io" "os" "path/filepath" "strconv" "time" log "github.com/sirupsen/logrus" - "github.com/valyala/fasttemplate" apiv1 "k8s.io/api/core/v1" apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,6 +23,7 @@ import ( wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" errorsutil "github.com/argoproj/argo-workflows/v3/util/errors" "github.com/argoproj/argo-workflows/v3/util/intstr" + "github.com/argoproj/argo-workflows/v3/util/template" "github.com/argoproj/argo-workflows/v3/workflow/common" "github.com/argoproj/argo-workflows/v3/workflow/util" ) @@ -449,11 +448,7 @@ func substitutePodParams(pod *apiv1.Pod, globalParams common.Parameters, tmpl *w if err != nil { return nil, err } - fstTmpl, err := fasttemplate.NewTemplate(string(specBytes), "{{", "}}") - if err != nil { - return nil, fmt.Errorf("unable to parse argo variable: %w", err) - } - newSpecBytes, err := common.Replace(fstTmpl, podParams, true) + newSpecBytes, err := template.Replace(string(specBytes), podParams, true) if err != nil { return nil, err } @@ -1140,16 +1135,9 @@ func verifyResolvedVariables(obj interface{}) error { if err != nil { return err } - var unresolvedErr error - fstTmpl, err := fasttemplate.NewTemplate(string(str), "{{", "}}") - if err != nil { - return fmt.Errorf("unable to parse argo variable: %w", err) - } - fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { - unresolvedErr = errors.Errorf(errors.CodeBadRequest, "failed to resolve {{%s}}", tag) - return 0, nil + return template.Validate(string(str), func(tag string) error { + return errors.Errorf(errors.CodeBadRequest, "failed to resolve {{%s}}", tag) }) - return unresolvedErr } // createSecretVolumes will retrieve and create Volumes and Volumemount object for Pod diff --git a/workflow/validate/validate.go b/workflow/validate/validate.go index c118ea63d8e5..ab0818fd557d 100644 --- a/workflow/validate/validate.go +++ b/workflow/validate/validate.go @@ -3,7 +3,6 @@ package validate import ( "encoding/json" "fmt" - "io" "reflect" "regexp" "strconv" @@ -12,7 +11,6 @@ import ( "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" - "github.com/valyala/fasttemplate" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" apivalidation "k8s.io/apimachinery/pkg/util/validation" "sigs.k8s.io/yaml" @@ -22,6 +20,7 @@ import ( "github.com/argoproj/argo-workflows/v3/util" "github.com/argoproj/argo-workflows/v3/util/intstr" "github.com/argoproj/argo-workflows/v3/util/sorting" + "github.com/argoproj/argo-workflows/v3/util/template" "github.com/argoproj/argo-workflows/v3/workflow/artifacts/hdfs" "github.com/argoproj/argo-workflows/v3/workflow/common" "github.com/argoproj/argo-workflows/v3/workflow/metrics" @@ -511,22 +510,16 @@ func validateArtifactLocation(errPrefix string, art wfv1.ArtifactLocation) error // resolveAllVariables is a helper to ensure all {{variables}} are resolveable from current scope func resolveAllVariables(scope map[string]interface{}, tmplStr string) error { - var unresolvedErr error _, allowAllItemRefs := scope[anyItemMagicValue] // 'item.*' is a magic placeholder value set by addItemsToScope _, allowAllWorkflowOutputParameterRefs := scope[anyWorkflowOutputParameterMagicValue] _, allowAllWorkflowOutputArtifactRefs := scope[anyWorkflowOutputArtifactMagicValue] - fstTmpl, err := fasttemplate.NewTemplate(tmplStr, "{{", "}}") - if err != nil { - return fmt.Errorf("unable to parse argo variable: %w", err) - } - - fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { + return template.Validate(tmplStr, func(tag string) error { // Skip the custom variable references if !checkValidWorkflowVariablePrefix(tag) { - return 0, nil + return nil } _, ok := scope[tag] - if !ok && unresolvedErr == nil { + if !ok { if (tag == "item" || strings.HasPrefix(tag, "item.")) && allowAllItemRefs { // we are *probably* referencing a undetermined item using withParam // NOTE: this is far from foolproof. @@ -538,12 +531,11 @@ func resolveAllVariables(scope map[string]interface{}, tmplStr string) error { // We are self referencing for metric emission, allow it. } else if strings.HasPrefix(tag, common.GlobalVarWorkflowCreationTimestamp) { } else { - unresolvedErr = fmt.Errorf("failed to resolve {{%s}}", tag) + return fmt.Errorf("failed to resolve {{%s}}", tag) } } - return 0, nil + return nil }) - return unresolvedErr } // checkValidWorkflowVariablePrefix is a helper methood check variable starts workflow root elements