From 764cdcc9daf875088e25875d091276210e778b5a Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 15 Apr 2022 17:13:30 -0700 Subject: [PATCH] wip add json jobspec support to cli While the CLI has always supported running JSON jobs, its support has been via HCLv2's JSON parsing. I have no idea what format it expects the job to be in, but it's absolutely not in the same format as the API expects. So I ignored that and added a new -json flag to explicitly support *API* style JSON jobspecs. The jobspecs can even have the wrapping {"Job": {...}} envelope or not! Lots left to do: - Changelog - Docs - Tests - plan and validate support - Can we remove, warn, or deprecate the weird HCL+JSON support? --- command/helpers.go | 68 +++++++++++++++++++++++++++++++++-------- command/job_plan.go | 18 +++++------ command/job_run.go | 32 +++++++++++++------ command/job_validate.go | 14 +++------ 4 files changed, 91 insertions(+), 41 deletions(-) diff --git a/command/helpers.go b/command/helpers.go index 863dc823e288..c01bf1753c0c 100644 --- a/command/helpers.go +++ b/command/helpers.go @@ -3,9 +3,9 @@ package command import ( "bufio" "bytes" + "encoding/json" "fmt" "io" - "io/ioutil" "os" "path/filepath" "strconv" @@ -14,6 +14,7 @@ import ( gg "github.com/hashicorp/go-getter" "github.com/hashicorp/nomad/api" + flaghelper "github.com/hashicorp/nomad/helper/flags" "github.com/hashicorp/nomad/jobspec" "github.com/hashicorp/nomad/jobspec2" "github.com/kr/text" @@ -379,19 +380,42 @@ READ: return l.ReadCloser.Read(p) } +// JobGetter provides helpers for retrieving and parsing a jobpsec. type JobGetter struct { - hcl1 bool + HCL1 bool + Vars flaghelper.StringFlag + VarFiles flaghelper.StringFlag + Strict bool + JSON bool // The fields below can be overwritten for tests testStdin io.Reader } +func (j *JobGetter) Validate() error { + if j.HCL1 && j.Strict { + return fmt.Errorf("cannot parse job file as HCLv1 and HCLv2 strict.") + } + if j.HCL1 && j.JSON { + return fmt.Errorf("cannot parse job file as HCL and JSON.") + } + return nil +} + // ApiJob returns the Job struct from jobfile. func (j *JobGetter) ApiJob(jpath string) (*api.Job, error) { return j.ApiJobWithArgs(jpath, nil, nil, true) } func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []string, strict bool) (*api.Job, error) { + j.Vars = vars + j.VarFiles = varfiles + j.Strict = strict + + return j.Get(jpath) +} + +func (j *JobGetter) Get(jpath string) (*api.Job, error) { var jobfile io.Reader pathName := filepath.Base(jpath) switch jpath { @@ -401,19 +425,19 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin } else { jobfile = os.Stdin } - pathName = "stdin.hcl" + pathName = "stdin" default: if len(jpath) == 0 { return nil, fmt.Errorf("Error jobfile path has to be specified.") } - job, err := ioutil.TempFile("", "jobfile") + jobFile, err := os.CreateTemp("", "jobfile") if err != nil { return nil, err } - defer os.Remove(job.Name()) + defer os.Remove(jobFile.Name()) - if err := job.Close(); err != nil { + if err := jobFile.Close(); err != nil { return nil, err } @@ -426,13 +450,13 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin client := &gg.Client{ Src: jpath, Pwd: pwd, - Dst: job.Name(), + Dst: jobFile.Name(), } if err := client.Get(); err != nil { return nil, fmt.Errorf("Error getting jobfile from %q: %v", jpath, err) } else { - file, err := os.Open(job.Name()) + file, err := os.Open(jobFile.Name()) if err != nil { return nil, fmt.Errorf("Error opening file %q: %v", jpath, err) } @@ -444,9 +468,27 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin // Parse the JobFile var jobStruct *api.Job var err error - if j.hcl1 { + switch { + case j.HCL1: jobStruct, err = jobspec.Parse(jobfile) - } else { + case j.JSON: + // Support JSON files with both a top-level Job key as well as + // ones without. + eitherJob := struct { + NestedJob *api.Job `json:"Job"` + api.Job + }{} + + if err := json.NewDecoder(jobfile).Decode(&eitherJob); err != nil { + return nil, fmt.Errorf("Failed to parse JSON job: %w", err) + } + + if eitherJob.NestedJob != nil { + jobStruct = eitherJob.NestedJob + } else { + jobStruct = &eitherJob.Job + } + default: var buf bytes.Buffer _, err = io.Copy(&buf, jobfile) if err != nil { @@ -455,11 +497,11 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin jobStruct, err = jobspec2.ParseWithConfig(&jobspec2.ParseConfig{ Path: pathName, Body: buf.Bytes(), - ArgVars: vars, + ArgVars: j.Vars, AllowFS: true, - VarFiles: varfiles, + VarFiles: j.VarFiles, Envs: os.Environ(), - Strict: strict, + Strict: j.Strict, }) if err != nil { diff --git a/command/job_plan.go b/command/job_plan.go index b78de85c5ae8..dca6aa1ccdf4 100644 --- a/command/job_plan.go +++ b/command/job_plan.go @@ -7,7 +7,6 @@ import ( "time" "github.com/hashicorp/nomad/api" - flaghelper "github.com/hashicorp/nomad/helper/flags" "github.com/hashicorp/nomad/scheduler" "github.com/posener/complete" ) @@ -122,18 +121,17 @@ func (c *JobPlanCommand) AutocompleteArgs() complete.Predictor { func (c *JobPlanCommand) Name() string { return "job plan" } func (c *JobPlanCommand) Run(args []string) int { - var diff, policyOverride, verbose, hcl2Strict bool - var varArgs, varFiles flaghelper.StringFlag + var diff, policyOverride, verbose bool flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient) flagSet.Usage = func() { c.Ui.Output(c.Help()) } flagSet.BoolVar(&diff, "diff", true, "") flagSet.BoolVar(&policyOverride, "policy-override", false, "") flagSet.BoolVar(&verbose, "verbose", false, "") - flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") - flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") - flagSet.Var(&varArgs, "var", "") - flagSet.Var(&varFiles, "var-file", "") + flagSet.BoolVar(&c.JobGetter.HCL1, "hcl1", false, "") + flagSet.BoolVar(&c.JobGetter.Strict, "hcl2-strict", true, "") + flagSet.Var(&c.JobGetter.Vars, "var", "") + flagSet.Var(&c.JobGetter.VarFiles, "var-file", "") if err := flagSet.Parse(args); err != nil { return 255 @@ -149,7 +147,7 @@ func (c *JobPlanCommand) Run(args []string) int { path := args[0] // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) + job, err := c.JobGetter.Get(path) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 255 @@ -193,11 +191,11 @@ func (c *JobPlanCommand) Run(args []string) int { } runArgs := strings.Builder{} - for _, varArg := range varArgs { + for _, varArg := range c.JobGetter.Vars { runArgs.WriteString(fmt.Sprintf("-var=%q ", varArg)) } - for _, varFile := range varFiles { + for _, varFile := range c.JobGetter.VarFiles { runArgs.WriteString(fmt.Sprintf("-var-file=%q ", varFile)) } diff --git a/command/job_run.go b/command/job_run.go index 49af470574b3..c4886d79006f 100644 --- a/command/job_run.go +++ b/command/job_run.go @@ -11,7 +11,6 @@ import ( "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/helper" - flaghelper "github.com/hashicorp/nomad/helper/flags" "github.com/posener/complete" ) @@ -90,6 +89,11 @@ Run Options: Override the priority of the evaluations produced as a result of this job submission. By default, this is set to the priority of the job. + -json + Parses the job file as JSON. If the outer object has a Job field, such as + from "nomad job inspect" or "nomad run -output", the value of the field is + used as the job. + -hcl1 Parses the job file as HCLv1. @@ -158,6 +162,7 @@ func (c *JobRunCommand) AutocompleteFlags() complete.Flags { "-output": complete.PredictNothing, "-policy-override": complete.PredictNothing, "-preserve-counts": complete.PredictNothing, + "-json": complete.PredictNothing, "-hcl1": complete.PredictNothing, "-hcl2-strict": complete.PredictNothing, "-var": complete.PredictAnything, @@ -167,15 +172,18 @@ func (c *JobRunCommand) AutocompleteFlags() complete.Flags { } func (c *JobRunCommand) AutocompleteArgs() complete.Predictor { - return complete.PredictOr(complete.PredictFiles("*.nomad"), complete.PredictFiles("*.hcl")) + return complete.PredictOr( + complete.PredictFiles("*.nomad"), + complete.PredictFiles("*.hcl"), + complete.PredictFiles("*.json"), + ) } func (c *JobRunCommand) Name() string { return "job run" } func (c *JobRunCommand) Run(args []string) int { - var detach, verbose, output, override, preserveCounts, hcl2Strict bool + var detach, verbose, output, override, preserveCounts bool var checkIndexStr, consulToken, consulNamespace, vaultToken, vaultNamespace string - var varArgs, varFiles flaghelper.StringFlag var evalPriority int flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient) @@ -185,15 +193,16 @@ func (c *JobRunCommand) Run(args []string) int { flagSet.BoolVar(&output, "output", false, "") flagSet.BoolVar(&override, "policy-override", false, "") flagSet.BoolVar(&preserveCounts, "preserve-counts", false, "") - flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") - flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") + flagSet.BoolVar(&c.JobGetter.JSON, "json", false, "") + flagSet.BoolVar(&c.JobGetter.HCL1, "hcl1", false, "") + flagSet.BoolVar(&c.JobGetter.Strict, "hcl2-strict", true, "") flagSet.StringVar(&checkIndexStr, "check-index", "", "") flagSet.StringVar(&consulToken, "consul-token", "", "") flagSet.StringVar(&consulNamespace, "consul-namespace", "", "") flagSet.StringVar(&vaultToken, "vault-token", "", "") flagSet.StringVar(&vaultNamespace, "vault-namespace", "", "") - flagSet.Var(&varArgs, "var", "") - flagSet.Var(&varFiles, "var-file", "") + flagSet.Var(&c.JobGetter.Vars, "var", "") + flagSet.Var(&c.JobGetter.VarFiles, "var-file", "") flagSet.IntVar(&evalPriority, "eval-priority", 0, "") if err := flagSet.Parse(args); err != nil { @@ -214,8 +223,13 @@ func (c *JobRunCommand) Run(args []string) int { return 1 } + if err := c.JobGetter.Validate(); err != nil { + c.Ui.Error(fmt.Sprintf("Invalid job options: %s", err)) + return 1 + } + // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) + job, err := c.JobGetter.Get(args[0]) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 1 diff --git a/command/job_validate.go b/command/job_validate.go index bea9d119e44b..3d7ffe483cf1 100644 --- a/command/job_validate.go +++ b/command/job_validate.go @@ -7,7 +7,6 @@ import ( "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/command/agent" - flaghelper "github.com/hashicorp/nomad/helper/flags" "github.com/hashicorp/nomad/nomad/structs" "github.com/posener/complete" ) @@ -71,15 +70,12 @@ func (c *JobValidateCommand) AutocompleteArgs() complete.Predictor { func (c *JobValidateCommand) Name() string { return "job validate" } func (c *JobValidateCommand) Run(args []string) int { - var varArgs, varFiles flaghelper.StringFlag - var hcl2Strict bool - flagSet := c.Meta.FlagSet(c.Name(), FlagSetNone) flagSet.Usage = func() { c.Ui.Output(c.Help()) } - flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") - flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") - flagSet.Var(&varArgs, "var", "") - flagSet.Var(&varFiles, "var-file", "") + flagSet.BoolVar(&c.JobGetter.HCL1, "hcl1", false, "") + flagSet.BoolVar(&c.JobGetter.Strict, "hcl2-strict", true, "") + flagSet.Var(&c.JobGetter.Vars, "var", "") + flagSet.Var(&c.JobGetter.VarFiles, "var-file", "") if err := flagSet.Parse(args); err != nil { return 1 @@ -94,7 +90,7 @@ func (c *JobValidateCommand) Run(args []string) int { } // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) + job, err := c.JobGetter.Get(args[0]) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 1