Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

core: prevent malformed plans from crashing leader #11944

Merged
merged 1 commit into from
Jan 31, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions nomad/plan_endpoint.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package nomad

import (
"fmt"
"time"

metrics "github.com/armon/go-metrics"
Expand All @@ -22,6 +23,10 @@ func (p *Plan) Submit(args *structs.PlanRequest, reply *structs.PlanResponse) er
}
defer metrics.MeasureSince([]string{"nomad", "plan", "submit"}, time.Now())

if args.Plan == nil {
return fmt.Errorf("cannot submit nil plan")
}

// Pause the Nack timer for the eval as it is making progress as long as it
// is in the plan queue. We resume immediately after we get a result to
// handle the case that the receiving worker dies.
Expand Down
78 changes: 78 additions & 0 deletions nomad/plan_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/testutil"
"github.com/stretchr/testify/require"
)

func TestPlanEndpoint_Submit(t *testing.T) {
Expand Down Expand Up @@ -49,3 +50,80 @@ func TestPlanEndpoint_Submit(t *testing.T) {
t.Fatalf("missing result")
}
}

// TestPlanEndpoint_Submit_Bad asserts that the Plan.Submit endpoint rejects
// bad data with an error instead of panicking.
func TestPlanEndpoint_Submit_Bad(t *testing.T) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the name, strong and to the point.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ha, you made me notice I typo'd it in the comment. Thanks 😁

t.Parallel()

s1, cleanupS1 := TestServer(t, func(c *Config) {
c.NumSchedulers = 0
})
defer cleanupS1()
codec := rpcClient(t, s1)
testutil.WaitForLeader(t, s1.RPC)

// Mock a valid eval being dequeued by a worker
eval := mock.Eval()
s1.evalBroker.Enqueue(eval)

evalOut, _, err := s1.evalBroker.Dequeue([]string{eval.Type}, time.Second)
require.NoError(t, err)
require.Equal(t, eval, evalOut)

cases := []struct {
Name string
Plan *structs.Plan
Err string
}{
{
Name: "Nil",
Plan: nil,
Err: "cannot submit nil plan",
},
{
Name: "Empty",
Plan: &structs.Plan{},
Err: "evaluation is not outstanding",
},
{
Name: "BadEvalID",
Plan: &structs.Plan{
EvalID: "1234", // does not exist
},
Err: "evaluation is not outstanding",
},
{
Name: "MissingToken",
Plan: &structs.Plan{
EvalID: eval.ID,
},
Err: "evaluation token does not match",
},
{
Name: "InvalidToken",
Plan: &structs.Plan{
EvalID: eval.ID,
EvalToken: "1234", // invalid
},
Err: "evaluation token does not match",
},
}

for i := range cases {
tc := cases[i]
t.Run(tc.Name, func(t *testing.T) {
req := &structs.PlanRequest{
Plan: tc.Plan,
WriteRequest: structs.WriteRequest{Region: "global"},
}
var resp structs.PlanResponse
err := msgpackrpc.CallWithCodec(codec, "Plan.Submit", req, &resp)
require.EqualError(t, err, tc.Err)
require.Nil(t, resp.Result)
})
}

// Ensure no plans were enqueued
require.Zero(t, s1.planner.planQueue.Stats().Depth)
}
4 changes: 3 additions & 1 deletion nomad/structs/structs.go
Original file line number Diff line number Diff line change
Expand Up @@ -10803,7 +10803,9 @@ type Plan struct {

func (p *Plan) GoString() string {
out := fmt.Sprintf("(eval %s", p.EvalID[:8])
out += fmt.Sprintf(", job %s", p.Job.ID)
if p.Job != nil {
out += fmt.Sprintf(", job %s", p.Job.ID)
}
if p.Deployment != nil {
out += fmt.Sprintf(", deploy %s", p.Deployment.ID[:8])
}
Expand Down