From b52f42db9a7bab1cf03cec60e4422f566c2d3efd Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 4 Nov 2021 16:33:09 +0100 Subject: [PATCH] Added a `-hcl2-strict` flag to allow for lenient hcl variable parsing. (#11284) Co-authored-by: James Rasell --- .changelog/11284.txt | 3 ++ command/helpers.go | 6 +-- command/helpers_test.go | 53 ++++++++++++++++++- command/job_plan.go | 11 +++- command/job_run.go | 11 +++- command/job_validate.go | 17 ++++-- website/content/docs/commands/job/plan.mdx | 8 +++ website/content/docs/commands/job/run.mdx | 14 +++-- .../content/docs/commands/job/validate.mdx | 12 +++++ 9 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 .changelog/11284.txt diff --git a/.changelog/11284.txt b/.changelog/11284.txt new file mode 100644 index 000000000000..d767686e5a9a --- /dev/null +++ b/.changelog/11284.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: added `hcl2-strict` flag to control HCL2 parsing errors where variable passed without root +``` diff --git a/command/helpers.go b/command/helpers.go index ee0de86a84bc..863dc823e288 100644 --- a/command/helpers.go +++ b/command/helpers.go @@ -388,10 +388,10 @@ type JobGetter struct { // ApiJob returns the Job struct from jobfile. func (j *JobGetter) ApiJob(jpath string) (*api.Job, error) { - return j.ApiJobWithArgs(jpath, nil, nil) + return j.ApiJobWithArgs(jpath, nil, nil, true) } -func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []string) (*api.Job, error) { +func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []string, strict bool) (*api.Job, error) { var jobfile io.Reader pathName := filepath.Base(jpath) switch jpath { @@ -459,7 +459,7 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin AllowFS: true, VarFiles: varfiles, Envs: os.Environ(), - Strict: true, + Strict: strict, }) if err != nil { diff --git a/command/helpers_test.go b/command/helpers_test.go index c5e54e853ea0..275a5d249db4 100644 --- a/command/helpers_test.go +++ b/command/helpers_test.go @@ -368,7 +368,58 @@ job "example" { _, err = vf.WriteString(fileVars + "\n") require.NoError(t, err) - j, err := (&JobGetter{}).ApiJobWithArgs(hclf.Name(), cliArgs, []string{vf.Name()}) + j, err := (&JobGetter{}).ApiJobWithArgs(hclf.Name(), cliArgs, []string{vf.Name()}, true) + require.NoError(t, err) + + require.NotNil(t, j) + require.Equal(t, expected, j.Datacenters) +} + +func TestJobGetter_HCL2_Variables_StrictFalse(t *testing.T) { + t.Parallel() + + hcl := ` +variables { + var1 = "default-val" + var2 = "default-val" + var3 = "default-val" + var4 = "default-val" +} + +job "example" { + datacenters = ["${var.var1}", "${var.var2}", "${var.var3}", "${var.var4}"] +} +` + + os.Setenv("NOMAD_VAR_var4", "from-envvar") + defer os.Unsetenv("NOMAD_VAR_var4") + + // Both the CLI and var file contain variables that are not used with the + // template and therefore would error, if hcl2-strict was true. + cliArgs := []string{`var2=from-cli`,`unsedVar1=from-cli`} + fileVars := ` +var3 = "from-varfile" +unsedVar2 = "from-varfile" +` + expected := []string{"default-val", "from-cli", "from-varfile", "from-envvar"} + + hclf, err := ioutil.TempFile("", "hcl") + require.NoError(t, err) + defer os.Remove(hclf.Name()) + defer hclf.Close() + + _, err = hclf.WriteString(hcl) + require.NoError(t, err) + + vf, err := ioutil.TempFile("", "var.hcl") + require.NoError(t, err) + defer os.Remove(vf.Name()) + defer vf.Close() + + _, err = vf.WriteString(fileVars + "\n") + require.NoError(t, err) + + j, err := (&JobGetter{}).ApiJobWithArgs(hclf.Name(), cliArgs, []string{vf.Name()}, false) require.NoError(t, err) require.NotNil(t, j) diff --git a/command/job_plan.go b/command/job_plan.go index bd43b0eab317..9ab331de9418 100644 --- a/command/job_plan.go +++ b/command/job_plan.go @@ -79,6 +79,11 @@ Plan Options: -hcl1 Parses the job file as HCLv1. + -hcl2-strict + Whether an error should be produced from the HCL2 parser where a variable + has been supplied which is not defined within the root variables. Defaults + to true. + -policy-override Sets the flag to force override any soft mandatory Sentinel policies. @@ -105,6 +110,7 @@ func (c *JobPlanCommand) AutocompleteFlags() complete.Flags { "-policy-override": complete.PredictNothing, "-verbose": complete.PredictNothing, "-hcl1": complete.PredictNothing, + "-hcl2-strict": complete.PredictNothing, "-var": complete.PredictAnything, "-var-file": complete.PredictFiles("*.var"), }) @@ -116,7 +122,7 @@ func (c *JobPlanCommand) AutocompleteArgs() complete.Predictor { func (c *JobPlanCommand) Name() string { return "job plan" } func (c *JobPlanCommand) Run(args []string) int { - var diff, policyOverride, verbose bool + var diff, policyOverride, verbose, hcl2Strict bool var varArgs, varFiles flaghelper.StringFlag flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient) @@ -125,6 +131,7 @@ func (c *JobPlanCommand) Run(args []string) int { flagSet.BoolVar(&policyOverride, "policy-override", false, "") flagSet.BoolVar(&verbose, "verbose", false, "") flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") + flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") flagSet.Var(&varArgs, "var", "") flagSet.Var(&varFiles, "var-file", "") @@ -142,7 +149,7 @@ func (c *JobPlanCommand) Run(args []string) int { path := args[0] // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles) + job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 255 diff --git a/command/job_run.go b/command/job_run.go index 77bd4b2b32b7..a2ef4cb40ca3 100644 --- a/command/job_run.go +++ b/command/job_run.go @@ -89,6 +89,11 @@ Run Options: -hcl1 Parses the job file as HCLv1. + -hcl2-strict + Whether an error should be produced from the HCL2 parser where a variable + has been supplied which is not defined within the root variables. Defaults + to true. + -output Output the JSON that would be submitted to the HTTP API without submitting the job. @@ -144,6 +149,7 @@ func (c *JobRunCommand) AutocompleteFlags() complete.Flags { "-policy-override": complete.PredictNothing, "-preserve-counts": complete.PredictNothing, "-hcl1": complete.PredictNothing, + "-hcl2-strict": complete.PredictNothing, "-var": complete.PredictAnything, "-var-file": complete.PredictFiles("*.var"), }) @@ -156,7 +162,7 @@ func (c *JobRunCommand) AutocompleteArgs() complete.Predictor { func (c *JobRunCommand) Name() string { return "job run" } func (c *JobRunCommand) Run(args []string) int { - var detach, verbose, output, override, preserveCounts bool + var detach, verbose, output, override, preserveCounts, hcl2Strict bool var checkIndexStr, consulToken, consulNamespace, vaultToken, vaultNamespace string var varArgs, varFiles flaghelper.StringFlag @@ -168,6 +174,7 @@ func (c *JobRunCommand) Run(args []string) int { flagSet.BoolVar(&override, "policy-override", false, "") flagSet.BoolVar(&preserveCounts, "preserve-counts", false, "") flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") + flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") flagSet.StringVar(&checkIndexStr, "check-index", "", "") flagSet.StringVar(&consulToken, "consul-token", "", "") flagSet.StringVar(&consulNamespace, "consul-namespace", "", "") @@ -195,7 +202,7 @@ func (c *JobRunCommand) Run(args []string) int { } // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles) + job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 1 diff --git a/command/job_validate.go b/command/job_validate.go index bae38135715b..bea9d119e44b 100644 --- a/command/job_validate.go +++ b/command/job_validate.go @@ -37,12 +37,16 @@ Validate Options: -hcl1 Parses the job file as HCLv1. + -hcl2-strict + Whether an error should be produced from the HCL2 parser where a variable + has been supplied which is not defined within the root variables. Defaults + to true. + -var 'key=value' Variable for template, can be used multiple times. -var-file=path Path to HCL2 file containing user variables. - ` return strings.TrimSpace(helpText) } @@ -53,9 +57,10 @@ func (c *JobValidateCommand) Synopsis() string { func (c *JobValidateCommand) AutocompleteFlags() complete.Flags { return complete.Flags{ - "-hcl1": complete.PredictNothing, - "-var": complete.PredictAnything, - "-var-file": complete.PredictFiles("*.var"), + "-hcl1": complete.PredictNothing, + "-hcl2-strict": complete.PredictNothing, + "-var": complete.PredictAnything, + "-var-file": complete.PredictFiles("*.var"), } } @@ -67,10 +72,12 @@ func (c *JobValidateCommand) Name() string { return "job validate" } func (c *JobValidateCommand) Run(args []string) int { var varArgs, varFiles flaghelper.StringFlag + var hcl2Strict bool flagSet := c.Meta.FlagSet(c.Name(), FlagSetNone) flagSet.Usage = func() { c.Ui.Output(c.Help()) } flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") + flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") flagSet.Var(&varArgs, "var", "") flagSet.Var(&varFiles, "var-file", "") @@ -87,7 +94,7 @@ func (c *JobValidateCommand) Run(args []string) int { } // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles) + job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 1 diff --git a/website/content/docs/commands/job/plan.mdx b/website/content/docs/commands/job/plan.mdx index 263f361a64ab..fd09bc5894e2 100644 --- a/website/content/docs/commands/job/plan.mdx +++ b/website/content/docs/commands/job/plan.mdx @@ -65,6 +65,14 @@ capability for the job's namespace. - `-hcl1`: If set, HCL1 parser is used for parsing the job spec. +- `-hcl2-strict`: Whether an error should be produced from the HCL2 parser where + a variable has been supplied which is not defined within the root variables. + Defaults to true. + +- `-var=`: Variable for template, can be used multiple times. + +- `-var-file=`: Path to HCL2 file containing user variables. + - `-verbose`: Increase diff verbosity. ## Examples diff --git a/website/content/docs/commands/job/run.mdx b/website/content/docs/commands/job/run.mdx index f6cffefb943b..1f032a84398c 100644 --- a/website/content/docs/commands/job/run.mdx +++ b/website/content/docs/commands/job/run.mdx @@ -27,8 +27,8 @@ downloaded and read from URL specified. Nomad downloads the job file using By default, on successful job submission the run command will enter an interactive monitor and display log information detailing the scheduling -decisions, placement information, and [deployment status] for the provided job -if applicable ([`batch`] and [`system`] jobs don't create deployments). The monitor will +decisions, placement information, and [deployment status] for the provided job +if applicable ([`batch`] and [`system`] jobs don't create deployments). The monitor will exit after scheduling and deployment have finished or failed. On successful job submission and scheduling, exit code 0 will be returned. If @@ -72,6 +72,10 @@ that volume. - `-hcl1`: If set, HCL1 parser is used for parsing the job spec. +- `-hcl2-strict`: Whether an error should be produced from the HCL2 parser where + a variable has been supplied which is not defined within the root variables. + Defaults to true. + - `-output`: Output the JSON that would be submitted to the HTTP API without submitting the job. @@ -133,7 +137,7 @@ $ nomad job run job1.nomad Deployed Task Group Desired Placed Healthy Unhealthy Progress Deadline - cache 2 2 1 0 2021-06-09T15:32:58-07:00 + cache 2 2 1 0 2021-06-09T15:32:58-07:00 web 1 1 1 0 2021-06-09T15:32:58-07:00 ``` @@ -154,7 +158,7 @@ $ nomad job run -check-index 6 example.nomad ==> 2021-06-09T16:57:30-07:00: Evaluation "5ef16dff" finished with status "complete" ==> 2021-06-09T16:57:30-07:00: Monitoring deployment "62eb607c" ✓ Deployment "62eb607c" successful - + 2021-06-09T16:57:30-07:00 ID = 62eb607c Job ID = example @@ -218,7 +222,7 @@ $ nomad job run example.nomad ``` [`go-getter`]: https://github.com/hashicorp/go-getter -[deployment status]: /docs/commands/deployment#status +[deployment status]: /docs/commands/deployment#status [`batch`]: /docs/schedulers#batch [`system`]: /docs/schedulers#system [`job plan` command]: /docs/commands/job/plan diff --git a/website/content/docs/commands/job/validate.mdx b/website/content/docs/commands/job/validate.mdx index 9b3d172687f8..ed10d2322ea8 100644 --- a/website/content/docs/commands/job/validate.mdx +++ b/website/content/docs/commands/job/validate.mdx @@ -32,6 +32,18 @@ of 1 indicates an error. When ACLs are enabled, this command requires a token with the `read-job` capability for the job's namespace. +## Validate Options + +- `-hcl1`: If set, HCL1 parser is used for parsing the job spec. + +- `-hcl2-strict`: Whether an error should be produced from the HCL2 parser where +a variable has been supplied which is not defined within the root variables. +Defaults to true. + +- `-var=`: Variable for template, can be used multiple times. + +- `-var-file=`: Path to HCL2 file containing user variables. + ## Examples Validate a job with invalid syntax: