cli: add -json flag to support job commands (#12591)

* cli: add -json flag to support job commands

While the CLI has always supported running JSON jobs, its support has
been via HCLv2's JSON parsing. I have no idea what format it expects the
job to be in, but it's absolutely not in the same format as the API
expects.

So I ignored that and added a new -json flag to explicitly support *API*
style JSON jobspecs.

The jobspecs can even have the wrapping {"Job": {...}} envelope or not!

* docs: fix example for `nomad job validate`

We haven't been able to validate inside driver config stanzas ever since
the move to task driver plugins. 😭

.changelog/12591.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Added -json flag to `nomad job {run,plan,validate}` to support parsing JSON formatted jobs
+```

command/helpers.go

@@ -3,9 +3,9 @@ package command
 import (
 	"bufio"
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -14,6 +14,7 @@ import (
 
 	gg "github.com/hashicorp/go-getter"
 	"github.com/hashicorp/nomad/api"
+	flaghelper "github.com/hashicorp/nomad/helper/flags"
 	"github.com/hashicorp/nomad/jobspec"
 	"github.com/hashicorp/nomad/jobspec2"
 	"github.com/kr/text"
@@ -379,19 +380,54 @@ READ:
 	return l.ReadCloser.Read(p)
 }
 
+// JobGetter provides helpers for retrieving and parsing a jobpsec.
 type JobGetter struct {
-	hcl1 bool
+	HCL1     bool
+	Vars     flaghelper.StringFlag
+	VarFiles flaghelper.StringFlag
+	Strict   bool
+	JSON     bool
 
 	// The fields below can be overwritten for tests
 	testStdin io.Reader
 }
 
+func (j *JobGetter) Validate() error {
+	if j.HCL1 && j.Strict {
+		return fmt.Errorf("cannot parse job file as HCLv1 and HCLv2 strict.")
+	}
+	if j.HCL1 && j.JSON {
+		return fmt.Errorf("cannot parse job file as HCL and JSON.")
+	}
+	if len(j.Vars) > 0 && j.JSON {
+		return fmt.Errorf("cannot use variables with JSON files.")
+	}
+	if len(j.VarFiles) > 0 && j.JSON {
+		return fmt.Errorf("cannot use variables with JSON files.")
+	}
+	if len(j.Vars) > 0 && j.HCL1 {
+		return fmt.Errorf("cannot use variables with HCLv1.")
+	}
+	if len(j.VarFiles) > 0 && j.HCL1 {
+		return fmt.Errorf("cannot use variables with HCLv1.")
+	}
+	return nil
+}
+
 // ApiJob returns the Job struct from jobfile.
 func (j *JobGetter) ApiJob(jpath string) (*api.Job, error) {
 	return j.ApiJobWithArgs(jpath, nil, nil, true)
 }
 
 func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []string, strict bool) (*api.Job, error) {
+	j.Vars = vars
+	j.VarFiles = varfiles
+	j.Strict = strict
+
+	return j.Get(jpath)
+}
+
+func (j *JobGetter) Get(jpath string) (*api.Job, error) {
 	var jobfile io.Reader
 	pathName := filepath.Base(jpath)
 	switch jpath {
@@ -401,19 +437,19 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin
 		} else {
 			jobfile = os.Stdin
 		}
-		pathName = "stdin.hcl"
+		pathName = "stdin"
 	default:
 		if len(jpath) == 0 {
 			return nil, fmt.Errorf("Error jobfile path has to be specified.")
 		}
 
-		job, err := ioutil.TempFile("", "jobfile")
+		jobFile, err := os.CreateTemp("", "jobfile")
 		if err != nil {
 			return nil, err
 		}
-		defer os.Remove(job.Name())
+		defer os.Remove(jobFile.Name())
 
-		if err := job.Close(); err != nil {
+		if err := jobFile.Close(); err != nil {
 			return nil, err
 		}
 
@@ -426,13 +462,13 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin
 		client := &gg.Client{
 			Src: jpath,
 			Pwd: pwd,
-			Dst: job.Name(),
+			Dst: jobFile.Name(),
 		}
 
 		if err := client.Get(); err != nil {
 			return nil, fmt.Errorf("Error getting jobfile from %q: %v", jpath, err)
 		} else {
-			file, err := os.Open(job.Name())
+			file, err := os.Open(jobFile.Name())
 			if err != nil {
 				return nil, fmt.Errorf("Error opening file %q: %v", jpath, err)
 			}
@@ -444,9 +480,27 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin
 	// Parse the JobFile
 	var jobStruct *api.Job
 	var err error
-	if j.hcl1 {
+	switch {
+	case j.HCL1:
 		jobStruct, err = jobspec.Parse(jobfile)
-	} else {
+	case j.JSON:
+		// Support JSON files with both a top-level Job key as well as
+		// ones without.
+		eitherJob := struct {
+			NestedJob *api.Job `json:"Job"`
+			api.Job
+		}{}
+
+		if err := json.NewDecoder(jobfile).Decode(&eitherJob); err != nil {
+			return nil, fmt.Errorf("Failed to parse JSON job: %w", err)
+		}
+
+		if eitherJob.NestedJob != nil {
+			jobStruct = eitherJob.NestedJob
+		} else {
+			jobStruct = &eitherJob.Job
+		}
+	default:
 		var buf bytes.Buffer
 		_, err = io.Copy(&buf, jobfile)
 		if err != nil {
@@ -455,11 +509,11 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin
 		jobStruct, err = jobspec2.ParseWithConfig(&jobspec2.ParseConfig{
 			Path:     pathName,
 			Body:     buf.Bytes(),
-			ArgVars:  vars,
+			ArgVars:  j.Vars,
 			AllowFS:  true,
-			VarFiles: varfiles,
+			VarFiles: j.VarFiles,
 			Envs:     os.Environ(),
-			Strict:   strict,
+			Strict:   j.Strict,
 		})
 
 		if err != nil {

command/helpers_test.go

@@ -451,6 +451,84 @@ func TestJobGetter_HTTPServer(t *testing.T) {
 	}
 }
 
+func TestJobGetter_Validate(t *testing.T) {
+	cases := []struct {
+		name        string
+		jg          JobGetter
+		errContains string
+	}{
+		{
+			"StrictAndHCL1",
+			JobGetter{
+				HCL1:   true,
+				Strict: true,
+			},
+			"HCLv1 and HCLv2 strict",
+		},
+		{
+			"JSONandHCL1",
+			JobGetter{
+				HCL1: true,
+				JSON: true,
+			},
+			"HCL and JSON",
+		},
+		{
+			"VarsAndHCL1",
+			JobGetter{
+				HCL1: true,
+				Vars: []string{"foo"},
+			},
+			"variables with HCLv1",
+		},
+		{
+			"VarFilesAndHCL1",
+			JobGetter{
+				HCL1:     true,
+				VarFiles: []string{"foo.var"},
+			},
+			"variables with HCLv1",
+		},
+		{
+			"VarsAndJSON",
+			JobGetter{
+				JSON: true,
+				Vars: []string{"foo"},
+			},
+			"variables with JSON",
+		},
+		{
+			"VarFilesAndJSON",
+			JobGetter{
+				JSON:     true,
+				VarFiles: []string{"foo.var"},
+			},
+			"variables with JSON files",
+		},
+		{
+			"JSON_OK",
+			JobGetter{
+				JSON: true,
+			},
+			"",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.jg.Validate()
+
+			switch tc.errContains {
+			case "":
+				require.NoError(t, err)
+			default:
+				require.ErrorContains(t, err, tc.errContains)
+			}
+
+		})
+	}
+}
+
 func TestPrettyTimeDiff(t *testing.T) {
 	// Grab the time and truncate to the nearest second. This allows our tests
 	// to be deterministic since we don't have to worry about rounding.

command/job_plan.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/hashicorp/nomad/api"
-	flaghelper "github.com/hashicorp/nomad/helper/flags"
 	"github.com/hashicorp/nomad/scheduler"
 	"github.com/posener/complete"
 )
@@ -76,6 +75,11 @@ Plan Options:
     Determines whether the diff between the remote job and planned job is shown.
     Defaults to true.
 
+  -json
+    Parses the job file as JSON. If the outer object has a Job field, such as
+    from "nomad job inspect" or "nomad run -output", the value of the field is
+    used as the job.
+
   -hcl1
     Parses the job file as HCLv1.
 
@@ -109,6 +113,7 @@ func (c *JobPlanCommand) AutocompleteFlags() complete.Flags {
 			"-diff":            complete.PredictNothing,
 			"-policy-override": complete.PredictNothing,
 			"-verbose":         complete.PredictNothing,
+			"-json":            complete.PredictNothing,
 			"-hcl1":            complete.PredictNothing,
 			"-hcl2-strict":     complete.PredictNothing,
 			"-var":             complete.PredictAnything,
@@ -117,23 +122,27 @@ func (c *JobPlanCommand) AutocompleteFlags() complete.Flags {
 }
 
 func (c *JobPlanCommand) AutocompleteArgs() complete.Predictor {
-	return complete.PredictOr(complete.PredictFiles("*.nomad"), complete.PredictFiles("*.hcl"))
+	return complete.PredictOr(
+		complete.PredictFiles("*.nomad"),
+		complete.PredictFiles("*.hcl"),
+		complete.PredictFiles("*.json"),
+	)
 }
 
 func (c *JobPlanCommand) Name() string { return "job plan" }
 func (c *JobPlanCommand) Run(args []string) int {
-	var diff, policyOverride, verbose, hcl2Strict bool
-	var varArgs, varFiles flaghelper.StringFlag
+	var diff, policyOverride, verbose bool
 
 	flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient)
 	flagSet.Usage = func() { c.Ui.Output(c.Help()) }
 	flagSet.BoolVar(&diff, "diff", true, "")
 	flagSet.BoolVar(&policyOverride, "policy-override", false, "")
 	flagSet.BoolVar(&verbose, "verbose", false, "")
-	flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "")
-	flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "")
-	flagSet.Var(&varArgs, "var", "")
-	flagSet.Var(&varFiles, "var-file", "")
+	flagSet.BoolVar(&c.JobGetter.JSON, "json", false, "")
+	flagSet.BoolVar(&c.JobGetter.HCL1, "hcl1", false, "")
+	flagSet.BoolVar(&c.JobGetter.Strict, "hcl2-strict", true, "")
+	flagSet.Var(&c.JobGetter.Vars, "var", "")
+	flagSet.Var(&c.JobGetter.VarFiles, "var-file", "")
 
 	if err := flagSet.Parse(args); err != nil {
 		return 255
@@ -147,9 +156,14 @@ func (c *JobPlanCommand) Run(args []string) int {
 		return 255
 	}
 
+	if err := c.JobGetter.Validate(); err != nil {
+		c.Ui.Error(fmt.Sprintf("Invalid job options: %s", err))
+		return 1
+	}
+
 	path := args[0]
 	// Get Job struct from Jobfile
-	job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict)
+	job, err := c.JobGetter.Get(path)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err))
 		return 255
@@ -193,11 +207,11 @@ func (c *JobPlanCommand) Run(args []string) int {
 	}
 
 	runArgs := strings.Builder{}
-	for _, varArg := range varArgs {
+	for _, varArg := range c.JobGetter.Vars {
 		runArgs.WriteString(fmt.Sprintf("-var=%q ", varArg))
 	}
 
-	for _, varFile := range varFiles {
+	for _, varFile := range c.JobGetter.VarFiles {
 		runArgs.WriteString(fmt.Sprintf("-var-file=%q ", varFile))
 	}
 

command/job_plan_test.go

@@ -255,3 +255,19 @@ func TestPlanCommad_Preemptions(t *testing.T) {
 	require.Contains(out, "batch")
 	require.Contains(out, "service")
 }
+
+func TestPlanCommad_JSON(t *testing.T) {
+	ui := cli.NewMockUi()
+	cmd := &JobPlanCommand{
+		Meta: Meta{Ui: ui},
+	}
+
+	args := []string{
+		"-address=http://nope",
+		"-json",
+		"testdata/example-short.json",
+	}
+	code := cmd.Run(args)
+	require.Equal(t, 255, code)
+	require.Contains(t, ui.ErrorWriter.String(), "Error during plan: Put")
+}