Skip to content

Commit

Permalink
Support customizing full scheduler config
Browse files Browse the repository at this point in the history
  • Loading branch information
Mahmood Ali committed Jan 28, 2020
1 parent 744c9a4 commit 31025d6
Show file tree
Hide file tree
Showing 6 changed files with 70 additions and 36 deletions.
6 changes: 2 additions & 4 deletions command/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -323,10 +323,8 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
}

// handle system scheduler preemption default
if agentConfig.Server.SystemSchedulerPreemptionEnabledDefault != nil {
conf.SystemSchedulerPreemptionEnabledDefault = *agentConfig.Server.SystemSchedulerPreemptionEnabledDefault
} else {
conf.SystemSchedulerPreemptionEnabledDefault = true
if agentConfig.Server.DefaultSchedulerConfig != nil {
conf.DefaultSchedulerConfig = *agentConfig.Server.DefaultSchedulerConfig
}

// Add the Consul and Vault configs
Expand Down
50 changes: 41 additions & 9 deletions command/agent/agent_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package agent

import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
Expand Down Expand Up @@ -277,18 +276,51 @@ func TestAgent_ServerConfig(t *testing.T) {

func TestAgent_ServerConfig_SchedulerFlags(t *testing.T) {
cases := []struct {
input *bool
expected bool
name string
input *structs.SchedulerConfiguration
expected structs.SchedulerConfiguration
}{
{nil, true},
{helper.BoolToPtr(false), false},
{helper.BoolToPtr(true), true},
{
"default case",
nil,
structs.SchedulerConfiguration{
PreemptionConfig: structs.PreemptionConfig{
SystemSchedulerEnabled: true,
},
},
},
{
"empty value: preemption is disabled",
&structs.SchedulerConfiguration{},
structs.SchedulerConfiguration{
PreemptionConfig: structs.PreemptionConfig{
SystemSchedulerEnabled: false,
},
},
},
{
"all explicitly set",
&structs.SchedulerConfiguration{
PreemptionConfig: structs.PreemptionConfig{
SystemSchedulerEnabled: true,
BatchSchedulerEnabled: true,
ServiceSchedulerEnabled: true,
},
},
structs.SchedulerConfiguration{
PreemptionConfig: structs.PreemptionConfig{
SystemSchedulerEnabled: true,
BatchSchedulerEnabled: true,
ServiceSchedulerEnabled: true,
},
},
},
}

for _, c := range cases {
t.Run(fmt.Sprintf("case: %v", c.input), func(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
conf := DefaultConfig()
conf.Server.SystemSchedulerPreemptionEnabledDefault = c.input
conf.Server.DefaultSchedulerConfig = c.input

a := &Agent{config: conf}
conf.AdvertiseAddrs.Serf = "127.0.0.1:4000"
Expand All @@ -299,7 +331,7 @@ func TestAgent_ServerConfig_SchedulerFlags(t *testing.T) {

out, err := a.serverConfig()
require.NoError(t, err)
require.Equal(t, c.expected, out.SystemSchedulerPreemptionEnabledDefault)
require.Equal(t, c.expected, out.DefaultSchedulerConfig)
})
}
}
Expand Down
11 changes: 9 additions & 2 deletions command/agent/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -444,8 +444,10 @@ type ServerConfig struct {
// ServerJoin contains information that is used to attempt to join servers
ServerJoin *ServerJoin `hcl:"server_join"`

// SystemSchedulerPreemptionEnabledDefault is used to determin whether to enable system preemption by default in a new cluster
SystemSchedulerPreemptionEnabledDefault *bool `hcl:"system_scheduler_preemption_enabled_default"`
// DefaultSchedulerConfig configures the initial scheduler config to be persisted in Raft.
// Once the cluster is bootstrapped, and Raft persists the config (from here or through API),
// This value is ignored.
DefaultSchedulerConfig *structs.SchedulerConfiguration `hcl:"default_scheduler_config"`

// ExtraKeysHCL is used by hcl to surface unexpected keys
ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
Expand Down Expand Up @@ -1340,6 +1342,11 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
result.ServerJoin = result.ServerJoin.Merge(b.ServerJoin)
}

if b.DefaultSchedulerConfig != nil {
c := *b.DefaultSchedulerConfig
result.DefaultSchedulerConfig = &c
}

// Add the schedulers
result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)

Expand Down
18 changes: 13 additions & 5 deletions nomad/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -305,8 +305,10 @@ type Config struct {
// dead servers.
AutopilotInterval time.Duration

// SystemSchedulerPreemptionEnabledDefault is used to determin whether to enable system preemption by default in a new cluster
SystemSchedulerPreemptionEnabledDefault bool
// DefaultSchedulerConfig configures the initial scheduler config to be persisted in Raft.
// Once the cluster is bootstrapped, and Raft persists the config (from here or through API),
// This value is ignored.
DefaultSchedulerConfig structs.SchedulerConfiguration `hcl:"default_scheduler_config"`

// PluginLoader is used to load plugins.
PluginLoader loader.PluginCatalog
Expand Down Expand Up @@ -380,9 +382,15 @@ func DefaultConfig() *Config {
MaxTrailingLogs: 250,
ServerStabilizationTime: 10 * time.Second,
},
ServerHealthInterval: 2 * time.Second,
AutopilotInterval: 10 * time.Second,
SystemSchedulerPreemptionEnabledDefault: true,
ServerHealthInterval: 2 * time.Second,
AutopilotInterval: 10 * time.Second,
DefaultSchedulerConfig: structs.SchedulerConfiguration{
PreemptionConfig: structs.PreemptionConfig{
SystemSchedulerEnabled: true,
BatchSchedulerEnabled: false,
ServiceSchedulerEnabled: false,
},
},
}

// Enable all known schedulers by default
Expand Down
13 changes: 1 addition & 12 deletions nomad/leader.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,17 +44,6 @@ var minAutopilotVersion = version.Must(version.NewVersion("0.8.0"))

var minSchedulerConfigVersion = version.Must(version.NewVersion("0.9.0"))

// Default configuration for scheduler with preemption enabled for system jobs
func (s *Server) defaultSchedulerConfig() structs.SchedulerConfiguration {
return structs.SchedulerConfiguration{
PreemptionConfig: structs.PreemptionConfig{
SystemSchedulerEnabled: s.config.SystemSchedulerPreemptionEnabledDefault,
BatchSchedulerEnabled: false,
ServiceSchedulerEnabled: false,
},
}
}

// monitorLeadership is used to monitor if we acquire or lose our role
// as the leader in the Raft cluster. There is some work the leader is
// expected to do, so we must react to changes
Expand Down Expand Up @@ -1321,7 +1310,7 @@ func (s *Server) getOrCreateSchedulerConfig() *structs.SchedulerConfiguration {
return nil
}

req := structs.SchedulerSetConfigRequest{Config: s.defaultSchedulerConfig()}
req := structs.SchedulerSetConfigRequest{Config: s.config.DefaultSchedulerConfig}
if _, _, err = s.raftApply(structs.SchedulerConfigRequestType, req); err != nil {
s.logger.Named("core").Error("failed to initialize config", "error", err)
return nil
Expand Down
8 changes: 4 additions & 4 deletions nomad/structs/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ type AutopilotConfig struct {
type SchedulerConfiguration struct {
// PreemptionConfig specifies whether to enable eviction of lower
// priority jobs to place higher priority jobs.
PreemptionConfig PreemptionConfig
PreemptionConfig PreemptionConfig `hcl:"preemption_config"`

// CreateIndex/ModifyIndex store the create/modify indexes of this configuration.
CreateIndex uint64
Expand Down Expand Up @@ -152,13 +152,13 @@ type SchedulerSetConfigurationResponse struct {
// PreemptionConfig specifies whether preemption is enabled based on scheduler type
type PreemptionConfig struct {
// SystemSchedulerEnabled specifies if preemption is enabled for system jobs
SystemSchedulerEnabled bool
SystemSchedulerEnabled bool `hcl:"system_scheduler_enabled"`

// BatchSchedulerEnabled specifies if preemption is enabled for batch jobs
BatchSchedulerEnabled bool
BatchSchedulerEnabled bool `hcl:"batch_scheduler_enabled"`

// ServiceSchedulerEnabled specifies if preemption is enabled for service jobs
ServiceSchedulerEnabled bool
ServiceSchedulerEnabled bool `hcl:"service_Scheduler_enabled"`
}

// SchedulerSetConfigRequest is used by the Operator endpoint to update the
Expand Down

0 comments on commit 31025d6

Please sign in to comment.