From 4e71ba2e71701c216efbeafcf59e161bb85fd3fb Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Mon, 31 Jul 2017 16:44:17 -0700 Subject: [PATCH] Distinct Property supports arbitrary limit This PR enhances the distinct_property constraint such that a limit can be specified in the RTarget/value parameter. This allows constraints such as: ``` constraint { distinct_property = "${meta.rack}" value = "2" } ``` This restricts any given rack from running more than 2 allocations from the task group. Fixes https://github.com/hashicorp/nomad/issues/1146 --- helper/funcs.go | 14 + nomad/structs/structs.go | 38 +++ nomad/structs/structs_test.go | 55 ++++ scheduler/feasible_test.go | 282 ++++++++++++++++++ scheduler/generic_sched_test.go | 26 +- scheduler/propertyset.go | 121 +++++--- website/source/api/json-jobs.html.md | 4 +- .../docs/job-specification/constraint.html.md | 24 +- 8 files changed, 502 insertions(+), 62 deletions(-) diff --git a/helper/funcs.go b/helper/funcs.go index cd1e9c6d5217..3ad918dde049 100644 --- a/helper/funcs.go +++ b/helper/funcs.go @@ -76,6 +76,20 @@ func IntMin(a, b int) int { return b } +func IntMax(a, b int) int { + if a > b { + return a + } + return b +} + +func Uint64Max(a, b uint64) uint64 { + if a > b { + return a + } + return b +} + // MapStringStringSliceValueSet returns the set of values in a map[string][]string func MapStringStringSliceValueSet(m map[string][]string) []string { set := make(map[string]struct{}) diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index c40a5718e651..8143e3c9caa4 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -3874,8 +3874,24 @@ func (c *Constraint) Validate() error { mErr.Errors = append(mErr.Errors, errors.New("Missing constraint operand")) } + // requireLtarget specifies whether the constraint requires an LTarget to be + // provided. + requireLtarget := true + // Perform additional validation based on operand switch c.Operand { + case ConstraintDistinctHosts: + requireLtarget = false + if c.RTarget != "" { + mErr.Errors = append(mErr.Errors, fmt.Errorf("Distinct hosts constraint doesn't allow RTarget. Got %q", c.RTarget)) + } + if c.LTarget != "" { + mErr.Errors = append(mErr.Errors, fmt.Errorf("Distinct hosts constraint doesn't allow LTarget. Got %q", c.LTarget)) + } + case ConstraintSetContains: + if c.RTarget == "" { + mErr.Errors = append(mErr.Errors, fmt.Errorf("Set contains constraint requires an RTarget")) + } case ConstraintRegex: if _, err := regexp.Compile(c.RTarget); err != nil { mErr.Errors = append(mErr.Errors, fmt.Errorf("Regular expression failed to compile: %v", err)) @@ -3884,7 +3900,29 @@ func (c *Constraint) Validate() error { if _, err := version.NewConstraint(c.RTarget); err != nil { mErr.Errors = append(mErr.Errors, fmt.Errorf("Version constraint is invalid: %v", err)) } + case ConstraintDistinctProperty: + // If a count is set, make sure it is convertible to a uint64 + if c.RTarget != "" { + count, err := strconv.ParseUint(c.RTarget, 10, 64) + if err != nil { + mErr.Errors = append(mErr.Errors, fmt.Errorf("Failed to convert RTarget %q to uint64: %v", c.RTarget, err)) + } else if count < 1 { + mErr.Errors = append(mErr.Errors, fmt.Errorf("Distinct Property must have an allowed count of 1 or greater: %d < 1", count)) + } + } + case "=", "==", "is", "!=", "not", "<", "<=", ">", ">=": + if c.RTarget == "" { + mErr.Errors = append(mErr.Errors, fmt.Errorf("Operator %q requires an RTarget", c.Operand)) + } + default: + mErr.Errors = append(mErr.Errors, fmt.Errorf("Unknown constraint type %q", c.Operand)) } + + // Ensure we have an LTarget for the constraints that need one + if requireLtarget && c.LTarget == "" { + mErr.Errors = append(mErr.Errors, fmt.Errorf("No LTarget provided but is required by constraint")) + } + return mErr.ErrorOrNil() } diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 6562adde289d..08897665b9a7 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -1323,6 +1323,61 @@ func TestConstraint_Validate(t *testing.T) { if !strings.Contains(mErr.Errors[0].Error(), "Malformed constraint") { t.Fatalf("err: %s", err) } + + // Perform distinct_property validation + c.Operand = ConstraintDistinctProperty + c.RTarget = "0" + err = c.Validate() + mErr = err.(*multierror.Error) + if !strings.Contains(mErr.Errors[0].Error(), "count of 1 or greater") { + t.Fatalf("err: %s", err) + } + + c.RTarget = "-1" + err = c.Validate() + mErr = err.(*multierror.Error) + if !strings.Contains(mErr.Errors[0].Error(), "to uint64") { + t.Fatalf("err: %s", err) + } + + // Perform distinct_hosts validation + c.Operand = ConstraintDistinctHosts + c.RTarget = "foo" + err = c.Validate() + mErr = err.(*multierror.Error) + if !strings.Contains(mErr.Errors[0].Error(), "doesn't allow RTarget") { + t.Fatalf("err: %s", err) + } + if !strings.Contains(mErr.Errors[1].Error(), "doesn't allow LTarget") { + t.Fatalf("err: %s", err) + } + + // Perform set_contains validation + c.Operand = ConstraintSetContains + c.RTarget = "" + err = c.Validate() + mErr = err.(*multierror.Error) + if !strings.Contains(mErr.Errors[0].Error(), "requires an RTarget") { + t.Fatalf("err: %s", err) + } + + // Perform LTarget validation + c.Operand = ConstraintRegex + c.RTarget = "foo" + c.LTarget = "" + err = c.Validate() + mErr = err.(*multierror.Error) + if !strings.Contains(mErr.Errors[0].Error(), "No LTarget") { + t.Fatalf("err: %s", err) + } + + // Perform constraint type validation + c.Operand = "foo" + err = c.Validate() + mErr = err.(*multierror.Error) + if !strings.Contains(mErr.Errors[0].Error(), "Unknown constraint type") { + t.Fatalf("err: %s", err) + } } func TestUpdateStrategy_Validate(t *testing.T) { diff --git a/scheduler/feasible_test.go b/scheduler/feasible_test.go index 86cd6f6f18a4..b6b4b852e505 100644 --- a/scheduler/feasible_test.go +++ b/scheduler/feasible_test.go @@ -782,6 +782,198 @@ func TestDistinctPropertyIterator_JobDistinctProperty(t *testing.T) { } } +// This test creates allocations across task groups that use a property value to +// detect if the constraint at the job level properly considers all task groups +// when the constraint allows a count greater than one +func TestDistinctPropertyIterator_JobDistinctProperty_Count(t *testing.T) { + state, ctx := testContext(t) + nodes := []*structs.Node{ + mock.Node(), + mock.Node(), + mock.Node(), + } + + for i, n := range nodes { + n.Meta["rack"] = fmt.Sprintf("%d", i) + + // Add to state store + if err := state.UpsertNode(uint64(100+i), n); err != nil { + t.Fatalf("failed to upsert node: %v", err) + } + } + + static := NewStaticIterator(ctx, nodes) + + // Create a job with a distinct_property constraint and a task groups. + tg1 := &structs.TaskGroup{Name: "bar"} + tg2 := &structs.TaskGroup{Name: "baz"} + + job := &structs.Job{ + ID: "foo", + Constraints: []*structs.Constraint{ + { + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + RTarget: "2", + }, + }, + TaskGroups: []*structs.TaskGroup{tg1, tg2}, + } + + // Add allocs placing two allocations on both node 1 and 2 and only one on + // node 3. This should make the job unsatisfiable on all nodes but node5. + // Also mix the allocations existing in the plan and the state store. + plan := ctx.Plan() + alloc1ID := structs.GenerateUUID() + plan.NodeAllocation[nodes[0].ID] = []*structs.Allocation{ + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: job.ID, + Job: job, + ID: alloc1ID, + NodeID: nodes[0].ID, + }, + + &structs.Allocation{ + TaskGroup: tg2.Name, + JobID: job.ID, + Job: job, + ID: alloc1ID, + NodeID: nodes[0].ID, + }, + + // Should be ignored as it is a different job. + &structs.Allocation{ + TaskGroup: tg2.Name, + JobID: "ignore 2", + Job: job, + ID: structs.GenerateUUID(), + NodeID: nodes[0].ID, + }, + } + plan.NodeAllocation[nodes[1].ID] = []*structs.Allocation{ + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + NodeID: nodes[1].ID, + }, + + &structs.Allocation{ + TaskGroup: tg2.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + NodeID: nodes[1].ID, + }, + + // Should be ignored as it is a different job. + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: "ignore 2", + Job: job, + ID: structs.GenerateUUID(), + NodeID: nodes[1].ID, + }, + } + plan.NodeAllocation[nodes[2].ID] = []*structs.Allocation{ + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + NodeID: nodes[2].ID, + }, + + // Should be ignored as it is a different job. + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: "ignore 2", + Job: job, + ID: structs.GenerateUUID(), + NodeID: nodes[2].ID, + }, + } + + // Put an allocation on Node 3 but make it stopped in the plan + stoppingAllocID := structs.GenerateUUID() + plan.NodeUpdate[nodes[2].ID] = []*structs.Allocation{ + &structs.Allocation{ + TaskGroup: tg2.Name, + JobID: job.ID, + Job: job, + ID: stoppingAllocID, + NodeID: nodes[2].ID, + }, + } + + upserting := []*structs.Allocation{ + // Have one of the allocations exist in both the plan and the state + // store. This resembles an allocation update + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: job.ID, + Job: job, + ID: alloc1ID, + EvalID: structs.GenerateUUID(), + NodeID: nodes[0].ID, + }, + + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + EvalID: structs.GenerateUUID(), + NodeID: nodes[1].ID, + }, + + &structs.Allocation{ + TaskGroup: tg2.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + EvalID: structs.GenerateUUID(), + NodeID: nodes[0].ID, + }, + + // Should be ignored as it is a different job. + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: "ignore 2", + Job: job, + ID: structs.GenerateUUID(), + EvalID: structs.GenerateUUID(), + NodeID: nodes[1].ID, + }, + &structs.Allocation{ + TaskGroup: tg2.Name, + JobID: "ignore 2", + Job: job, + ID: structs.GenerateUUID(), + EvalID: structs.GenerateUUID(), + NodeID: nodes[1].ID, + }, + } + if err := state.UpsertAllocs(1000, upserting); err != nil { + t.Fatalf("failed to UpsertAllocs: %v", err) + } + + proposed := NewDistinctPropertyIterator(ctx, static) + proposed.SetJob(job) + proposed.SetTaskGroup(tg2) + proposed.Reset() + + out := collectFeasible(proposed) + if len(out) != 1 { + t.Fatalf("Bad: %#v", out) + } + if out[0].ID != nodes[2].ID { + t.Fatalf("wrong node picked") + } +} + // This test checks that if a node has an allocation on it that gets stopped, // there is a plan to re-use that for a new allocation, that the next select // won't select that node. @@ -934,6 +1126,96 @@ func TestDistinctPropertyIterator_JobDistinctProperty_Infeasible(t *testing.T) { } } +// This test creates previous allocations selecting certain property values to +// test if it detects infeasibility of property values correctly and picks the +// only feasible one +func TestDistinctPropertyIterator_JobDistinctProperty_Infeasible_Count(t *testing.T) { + state, ctx := testContext(t) + nodes := []*structs.Node{ + mock.Node(), + mock.Node(), + } + + for i, n := range nodes { + n.Meta["rack"] = fmt.Sprintf("%d", i) + + // Add to state store + if err := state.UpsertNode(uint64(100+i), n); err != nil { + t.Fatalf("failed to upsert node: %v", err) + } + } + + static := NewStaticIterator(ctx, nodes) + + // Create a job with a distinct_property constraint and a task groups. + tg1 := &structs.TaskGroup{Name: "bar"} + tg2 := &structs.TaskGroup{Name: "baz"} + tg3 := &structs.TaskGroup{Name: "bam"} + + job := &structs.Job{ + ID: "foo", + Constraints: []*structs.Constraint{ + { + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + RTarget: "2", + }, + }, + TaskGroups: []*structs.TaskGroup{tg1, tg2, tg3}, + } + + // Add allocs placing two tg1's on node1 and two tg2's on node2. This should + // make the job unsatisfiable for tg3. + plan := ctx.Plan() + plan.NodeAllocation[nodes[0].ID] = []*structs.Allocation{ + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + NodeID: nodes[0].ID, + }, + &structs.Allocation{ + TaskGroup: tg2.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + NodeID: nodes[0].ID, + }, + } + upserting := []*structs.Allocation{ + &structs.Allocation{ + TaskGroup: tg1.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + EvalID: structs.GenerateUUID(), + NodeID: nodes[1].ID, + }, + &structs.Allocation{ + TaskGroup: tg2.Name, + JobID: job.ID, + Job: job, + ID: structs.GenerateUUID(), + EvalID: structs.GenerateUUID(), + NodeID: nodes[1].ID, + }, + } + if err := state.UpsertAllocs(1000, upserting); err != nil { + t.Fatalf("failed to UpsertAllocs: %v", err) + } + + proposed := NewDistinctPropertyIterator(ctx, static) + proposed.SetJob(job) + proposed.SetTaskGroup(tg3) + proposed.Reset() + + out := collectFeasible(proposed) + if len(out) != 0 { + t.Fatalf("Bad: %#v", out) + } +} + // This test creates previous allocations selecting certain property values to // test if it detects infeasibility of property values correctly and picks the // only feasible one when the constraint is at the task group. diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 77eaed7b95e0..c6f2aeea9965 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -342,11 +342,12 @@ func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { // Create a job that uses distinct property and has count higher than what is // possible. job := mock.Job() - job.TaskGroups[0].Count = 4 + job.TaskGroups[0].Count = 8 job.Constraints = append(job.Constraints, &structs.Constraint{ Operand: structs.ConstraintDistinctProperty, LTarget: "${meta.rack}", + RTarget: "2", }) noErr(t, h.State.UpsertJob(h.NextIndex(), job)) @@ -391,7 +392,7 @@ func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { for _, allocList := range plan.NodeAllocation { planned = append(planned, allocList...) } - if len(planned) != 2 { + if len(planned) != 4 { t.Fatalf("bad: %#v", plan) } @@ -401,17 +402,17 @@ func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { noErr(t, err) // Ensure all allocations placed - if len(out) != 2 { + if len(out) != 4 { t.Fatalf("bad: %#v", out) } - // Ensure different node was used per. - used := make(map[string]struct{}) + // Ensure each node was only used twice + used := make(map[string]uint64) for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) + if count, _ := used[alloc.NodeID]; count > 2 { + t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) } - used[alloc.NodeID] = struct{}{} + used[alloc.NodeID]++ } h.AssertEvalStatus(t, structs.EvalStatusComplete) @@ -427,8 +428,7 @@ func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - // Create a job that uses distinct property and has count higher than what is - // possible. + // Create a job that uses distinct property only on one task group. job := mock.Job() job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) job.TaskGroups[0].Count = 1 @@ -439,7 +439,7 @@ func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { }) job.TaskGroups[1].Name = "tg2" - job.TaskGroups[1].Count = 1 + job.TaskGroups[1].Count = 2 noErr(t, h.State.UpsertJob(h.NextIndex(), job)) // Create a mock evaluation to register the job @@ -477,7 +477,7 @@ func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { for _, allocList := range plan.NodeAllocation { planned = append(planned, allocList...) } - if len(planned) != 2 { + if len(planned) != 3 { t.Fatalf("bad: %#v", plan) } @@ -487,7 +487,7 @@ func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { noErr(t, err) // Ensure all allocations placed - if len(out) != 2 { + if len(out) != 3 { t.Fatalf("bad: %#v", out) } diff --git a/scheduler/propertyset.go b/scheduler/propertyset.go index 5fb1465564f6..5c9d390939ad 100644 --- a/scheduler/propertyset.go +++ b/scheduler/propertyset.go @@ -2,8 +2,10 @@ package scheduler import ( "fmt" + "strconv" memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" ) @@ -21,21 +23,25 @@ type propertySet struct { // constraint is the constraint this property set is checking constraint *structs.Constraint + // allowedCount is the allowed number of allocations that can have the + // distinct property + allowedCount uint64 + // errorBuilding marks whether there was an error when building the property // set errorBuilding error - // existingValues is the set of values for the given property that have been - // used by pre-existing allocations. - existingValues map[string]struct{} + // existingValues is a mapping of the values of a property to the number of + // times the value has been used by pre-existing allocations. + existingValues map[string]uint64 - // proposedValues is the set of values for the given property that are used - // from proposed allocations. - proposedValues map[string]struct{} + // proposedValues is a mapping of the values of a property to the number of + // times the value has been used by proposed allocations. + proposedValues map[string]uint64 - // clearedValues is the set of values that are no longer being used by - // existingValues because of proposed stops. - clearedValues map[string]struct{} + // clearedValues is a mapping of the values of a property to the number of + // times the value has been used by proposed stopped allocations. + clearedValues map[string]uint64 } // NewPropertySet returns a new property set used to guarantee unique property @@ -44,7 +50,7 @@ func NewPropertySet(ctx Context, job *structs.Job) *propertySet { p := &propertySet{ ctx: ctx, jobID: job.ID, - existingValues: make(map[string]struct{}), + existingValues: make(map[string]uint64), } return p @@ -53,26 +59,42 @@ func NewPropertySet(ctx Context, job *structs.Job) *propertySet { // SetJobConstraint is used to parameterize the property set for a // distinct_property constraint set at the job level. func (p *propertySet) SetJobConstraint(constraint *structs.Constraint) { - // Store the constraint - p.constraint = constraint - p.populateExisting(constraint) - - // Populate the proposed when setting the constraint. We do this because - // when detecting if we can inplace update an allocation we stage an - // eviction and then select. This means the plan has an eviction before a - // single select has finished. - p.PopulateProposed() + p.setConstraint(constraint, "") } // SetTGConstraint is used to parameterize the property set for a // distinct_property constraint set at the task group level. The inputs are the // constraint and the task group name. func (p *propertySet) SetTGConstraint(constraint *structs.Constraint, taskGroup string) { + p.setConstraint(constraint, taskGroup) +} + +// setConstraint is a shared helper for setting a job or task group constraint. +func (p *propertySet) setConstraint(constraint *structs.Constraint, taskGroup string) { // Store that this is for a task group - p.taskGroup = taskGroup + if taskGroup != "" { + p.taskGroup = taskGroup + } // Store the constraint p.constraint = constraint + + // Determine the number of allowed allocations with the property. + if v := constraint.RTarget; v != "" { + c, err := strconv.ParseUint(v, 10, 64) + if err != nil { + p.errorBuilding = fmt.Errorf("failed to convert RTarget %q to uint64: %v", v, err) + p.ctx.Logger().Printf("[ERR] scheduler.dynamic-constraint: %v", p.errorBuilding) + return + } + + p.allowedCount = c + } else { + p.allowedCount = 1 + } + + // Determine the number of existing allocations that are using a property + // value p.populateExisting(constraint) // Populate the proposed when setting the constraint. We do this because @@ -80,6 +102,7 @@ func (p *propertySet) SetTGConstraint(constraint *structs.Constraint, taskGroup // eviction and then select. This means the plan has an eviction before a // single select has finished. p.PopulateProposed() + } // populateExisting is a helper shared when setting the constraint to populate @@ -115,8 +138,8 @@ func (p *propertySet) populateExisting(constraint *structs.Constraint) { func (p *propertySet) PopulateProposed() { // Reset the proposed properties - p.proposedValues = make(map[string]struct{}) - p.clearedValues = make(map[string]struct{}) + p.proposedValues = make(map[string]uint64) + p.clearedValues = make(map[string]uint64) // Gather the set of proposed stops. var stopping []*structs.Allocation @@ -151,7 +174,14 @@ func (p *propertySet) PopulateProposed() { // Remove any cleared value that is now being used by the proposed allocs for value := range p.proposedValues { - delete(p.clearedValues, value) + current, ok := p.clearedValues[value] + if !ok { + continue + } else if current == 0 { + delete(p.clearedValues, value) + } else if current > 1 { + p.clearedValues[value]-- + } } } @@ -171,27 +201,40 @@ func (p *propertySet) SatisfiesDistinctProperties(option *structs.Node, tg strin return false, fmt.Sprintf("missing property %q", p.constraint.LTarget) } - // both is used to iterate over both the proposed and existing used - // properties - bothAll := []map[string]struct{}{p.existingValues, p.proposedValues} - - // Check if the nodes value has already been used. - for _, usedProperties := range bothAll { - // Check if the nodes value has been used - _, used := usedProperties[nValue] - if !used { - continue + // combine the counts of how many times the property has been used by + // existing and proposed allocations + combinedUse := make(map[string]uint64, helper.IntMax(len(p.existingValues), len(p.proposedValues))) + for _, usedValues := range []map[string]uint64{p.existingValues, p.proposedValues} { + for propertyValue, usedCount := range usedValues { + combinedUse[propertyValue] += usedCount } + } - // Check if the value has been cleared from a proposed stop - if _, cleared := p.clearedValues[nValue]; cleared { + // Go through and discount the combined count when the value has been + // cleared by a proposed stop. + for propertyValue, clearedCount := range p.clearedValues { + combined, ok := combinedUse[propertyValue] + if !ok { continue } - return false, fmt.Sprintf("distinct_property: %s=%s already used", p.constraint.LTarget, nValue) + // Don't clear below 0. + combinedUse[propertyValue] = helper.Uint64Max(0, combined-clearedCount) + } + + usedCount, used := combinedUse[nValue] + if !used { + // The property value has never been used so we can use it. + return true, "" + } + + // The property value has been used but within the number of allowed + // allocations. + if usedCount < p.allowedCount { + return true, "" } - return true, "" + return false, fmt.Sprintf("distinct_property: %s=%s used by %d allocs", p.constraint.LTarget, nValue, usedCount) } // filterAllocs filters a set of allocations to just be those that are running @@ -245,7 +288,7 @@ func (p *propertySet) buildNodeMap(allocs []*structs.Allocation) (map[string]*st // populateProperties goes through all allocations and builds up the used // properties from the nodes storing the results in the passed properties map. func (p *propertySet) populateProperties(allocs []*structs.Allocation, nodes map[string]*structs.Node, - properties map[string]struct{}) { + properties map[string]uint64) { for _, alloc := range allocs { nProperty, ok := getProperty(nodes[alloc.NodeID], p.constraint.LTarget) @@ -253,7 +296,7 @@ func (p *propertySet) populateProperties(allocs []*structs.Allocation, nodes map continue } - properties[nProperty] = struct{}{} + properties[nProperty]++ } } diff --git a/website/source/api/json-jobs.html.md b/website/source/api/json-jobs.html.md index c83f58b4367d..7802f962ab78 100644 --- a/website/source/api/json-jobs.html.md +++ b/website/source/api/json-jobs.html.md @@ -537,7 +537,9 @@ The `Constraint` object supports the following keys: omitted. - `distinct_property` - If set, the scheduler selects nodes that have a - distinct value of the specified property for each allocation. This can + distinct value of the specified property. The `RTarget` specifies how + many allocations are allowed to share the value of a property. The + `RTarget` must be 1 or greater and if omitted, defaults to 1. This can be specified as a job constraint which applies the constraint to all task groups in the job, or as a task group constraint which scopes the effect to just that group. The constraint may not be specified at the diff --git a/website/source/docs/job-specification/constraint.html.md b/website/source/docs/job-specification/constraint.html.md index 5fc216949265..3a080939385b 100644 --- a/website/source/docs/job-specification/constraint.html.md +++ b/website/source/docs/job-specification/constraint.html.md @@ -75,6 +75,8 @@ all groups (and tasks) in the job. >= < <= + distinct_hosts + distinct_property regexp set_contains version @@ -124,16 +126,18 @@ constraint { ``` - `"distinct_property"` - Instructs the scheduler to select nodes that have a - distinct value of the specified property for each allocation. When specified - as a job constraint, it applies to all groups in the job. When specified as a - group constraint, the effect is constrained to that group. This constraint can - not be specified at the task level. Note that the `value` parameter should be - omitted when using this constraint. + distinct value of the specified property. The `value` parameter specifies how + many allocations are allowed to share the value of a property. The `value` + must be 1 or greater and if omitted, defaults to 1. When specified as a job + constraint, it applies to all groups in the job. When specified as a group + constraint, the effect is constrained to that group. This constraint can not + be specified at the task level. ```hcl constraint { operator = "distinct_property" attribute = "${meta.rack}" + value = "3" } ``` @@ -142,7 +146,8 @@ constraint { ```hcl constraint { - distinct_property = "${meta.rack}" + distinct_property = "${meta.rack}" + value = "3" } ``` @@ -209,13 +214,14 @@ constraint { A potential use case of the `distinct_property` constraint is to spread a service with `count > 1` across racks to minimize correlated failure. Nodes can be annotated with which rack they are on using [client -metadata][client-metadata] with values -such as "rack-12-1", "rack-12-2", etc. The following constraint would then -assure no two instances of the task group existed on the same rack. +metadata][client-metadata] with values such as "rack-12-1", "rack-12-2", etc. +The following constraint would assure that an individual rack is not running +more than 2 instances of the task group. ```hcl constraint { distinct_property = "${meta.rack}" + value = "2" } ```