Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Backport of core: prevent malformed plans from crashing leader into release/1.1.x #12614

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions nomad/plan_endpoint.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package nomad

import (
"fmt"
"time"

metrics "github.com/armon/go-metrics"
Expand All @@ -22,6 +23,10 @@ func (p *Plan) Submit(args *structs.PlanRequest, reply *structs.PlanResponse) er
}
defer metrics.MeasureSince([]string{"nomad", "plan", "submit"}, time.Now())

if args.Plan == nil {
return fmt.Errorf("cannot submit nil plan")
}

// Pause the Nack timer for the eval as it is making progress as long as it
// is in the plan queue. We resume immediately after we get a result to
// handle the case that the receiving worker dies.
Expand Down
78 changes: 78 additions & 0 deletions nomad/plan_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
"github.com/stretchr/testify/require"
)

func TestPlanEndpoint_Submit(t *testing.T) {
Expand Down Expand Up @@ -49,3 +50,80 @@ func TestPlanEndpoint_Submit(t *testing.T) {
t.Fatalf("missing result")
}
}

// TestPlanEndpoint_Submit_Bad asserts that the Plan.Submit endpoint rejects
// bad data with an error instead of panicking.
func TestPlanEndpoint_Submit_Bad(t *testing.T) {
t.Parallel()

s1, cleanupS1 := TestServer(t, func(c *Config) {
c.NumSchedulers = 0
})
defer cleanupS1()
codec := rpcClient(t, s1)
testutil.WaitForLeader(t, s1.RPC)

// Mock a valid eval being dequeued by a worker
eval := mock.Eval()
s1.evalBroker.Enqueue(eval)

evalOut, _, err := s1.evalBroker.Dequeue([]string{eval.Type}, time.Second)
require.NoError(t, err)
require.Equal(t, eval, evalOut)

cases := []struct {
Name string
Plan *structs.Plan
Err string
}{
{
Name: "Nil",
Plan: nil,
Err: "cannot submit nil plan",
},
{
Name: "Empty",
Plan: &structs.Plan{},
Err: "evaluation is not outstanding",
},
{
Name: "BadEvalID",
Plan: &structs.Plan{
EvalID: "1234", // does not exist
},
Err: "evaluation is not outstanding",
},
{
Name: "MissingToken",
Plan: &structs.Plan{
EvalID: eval.ID,
},
Err: "evaluation token does not match",
},
{
Name: "InvalidToken",
Plan: &structs.Plan{
EvalID: eval.ID,
EvalToken: "1234", // invalid
},
Err: "evaluation token does not match",
},
}

for i := range cases {
tc := cases[i]
t.Run(tc.Name, func(t *testing.T) {
req := &structs.PlanRequest{
Plan: tc.Plan,
WriteRequest: structs.WriteRequest{Region: "global"},
}
var resp structs.PlanResponse
err := msgpackrpc.CallWithCodec(codec, "Plan.Submit", req, &resp)
require.EqualError(t, err, tc.Err)
require.Nil(t, resp.Result)
})
}

// Ensure no plans were enqueued
require.Zero(t, s1.planner.planQueue.Stats().Depth)
}
4 changes: 3 additions & 1 deletion nomad/structs/structs.go
Original file line number Diff line number Diff line change
Expand Up @@ -10569,7 +10569,9 @@ type Plan struct {

func (p *Plan) GoString() string {
out := fmt.Sprintf("(eval %s", p.EvalID[:8])
out += fmt.Sprintf(", job %s", p.Job.ID)
if p.Job != nil {
out += fmt.Sprintf(", job %s", p.Job.ID)
}
if p.Deployment != nil {
out += fmt.Sprintf(", deploy %s", p.Deployment.ID[:8])
}
Expand Down