scheduler: allow configuring default preemption for system scheduler

command/agent/agent.go

@@ -322,6 +322,11 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 		return nil, fmt.Errorf("server_service_name must be set when auto_advertise is enabled")
 	}
 
+	// handle system scheduler preemption default
+	if agentConfig.Server.DefaultSchedulerConfig != nil {
+		conf.DefaultSchedulerConfig = *agentConfig.Server.DefaultSchedulerConfig
+	}
+
 	// Add the Consul and Vault configs
 	conf.ConsulConfig = agentConfig.Consul
 	conf.VaultConfig = agentConfig.Vault

command/agent/agent_test.go

@@ -274,6 +274,67 @@ func TestAgent_ServerConfig(t *testing.T) {
 	}
 }
 
+func TestAgent_ServerConfig_SchedulerFlags(t *testing.T) {
+	cases := []struct {
+		name     string
+		input    *structs.SchedulerConfiguration
+		expected structs.SchedulerConfiguration
+	}{
+		{
+			"default case",
+			nil,
+			structs.SchedulerConfiguration{
+				PreemptionConfig: structs.PreemptionConfig{
+					SystemSchedulerEnabled: true,
+				},
+			},
+		},
+		{
+			"empty value: preemption is disabled",
+			&structs.SchedulerConfiguration{},
+			structs.SchedulerConfiguration{
+				PreemptionConfig: structs.PreemptionConfig{
+					SystemSchedulerEnabled: false,
+				},
+			},
+		},
+		{
+			"all explicitly set",
+			&structs.SchedulerConfiguration{
+				PreemptionConfig: structs.PreemptionConfig{
+					SystemSchedulerEnabled:  true,
+					BatchSchedulerEnabled:   true,
+					ServiceSchedulerEnabled: true,
+				},
+			},
+			structs.SchedulerConfiguration{
+				PreemptionConfig: structs.PreemptionConfig{
+					SystemSchedulerEnabled:  true,
+					BatchSchedulerEnabled:   true,
+					ServiceSchedulerEnabled: true,
+				},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			conf := DefaultConfig()
+			conf.Server.DefaultSchedulerConfig = c.input
+
+			a := &Agent{config: conf}
+			conf.AdvertiseAddrs.Serf = "127.0.0.1:4000"
+			conf.AdvertiseAddrs.RPC = "127.0.0.1:4001"
+			conf.AdvertiseAddrs.HTTP = "10.10.11.1:4005"
+			conf.ACL.Enabled = true
+			require.NoError(t, conf.normalizeAddrs())
+
+			out, err := a.serverConfig()
+			require.NoError(t, err)
+			require.Equal(t, c.expected, out.DefaultSchedulerConfig)
+		})
+	}
+}
 func TestAgent_ClientConfig(t *testing.T) {
 	t.Parallel()
 	conf := DefaultConfig()

command/agent/config.go

@@ -444,6 +444,11 @@ type ServerConfig struct {
 	// ServerJoin contains information that is used to attempt to join servers
 	ServerJoin *ServerJoin `hcl:"server_join"`
 
+	// DefaultSchedulerConfig configures the initial scheduler config to be persisted in Raft.
+	// Once the cluster is bootstrapped, and Raft persists the config (from here or through API),
+	// This value is ignored.
+	DefaultSchedulerConfig *structs.SchedulerConfiguration `hcl:"default_scheduler_config"`
+
 	// ExtraKeysHCL is used by hcl to surface unexpected keys
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
 }
@@ -1337,6 +1342,11 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 		result.ServerJoin = result.ServerJoin.Merge(b.ServerJoin)
 	}
 
+	if b.DefaultSchedulerConfig != nil {
+		c := *b.DefaultSchedulerConfig
+		result.DefaultSchedulerConfig = &c
+	}
+
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)
 

nomad/config.go

@@ -305,6 +305,11 @@ type Config struct {
 	// dead servers.
 	AutopilotInterval time.Duration
 
+	// DefaultSchedulerConfig configures the initial scheduler config to be persisted in Raft.
+	// Once the cluster is bootstrapped, and Raft persists the config (from here or through API),
+	// This value is ignored.
+	DefaultSchedulerConfig structs.SchedulerConfiguration `hcl:"default_scheduler_config"`
+
 	// PluginLoader is used to load plugins.
 	PluginLoader loader.PluginCatalog
 
@@ -379,6 +384,13 @@ func DefaultConfig() *Config {
 		},
 		ServerHealthInterval: 2 * time.Second,
 		AutopilotInterval:    10 * time.Second,
+		DefaultSchedulerConfig: structs.SchedulerConfiguration{
+			PreemptionConfig: structs.PreemptionConfig{
+				SystemSchedulerEnabled:  true,
+				BatchSchedulerEnabled:   false,
+				ServiceSchedulerEnabled: false,
+			},
+		},
 	}
 
 	// Enable all known schedulers by default

nomad/leader.go

@@ -44,15 +44,6 @@ var minAutopilotVersion = version.Must(version.NewVersion("0.8.0"))
 
 var minSchedulerConfigVersion = version.Must(version.NewVersion("0.9.0"))
 
-// Default configuration for scheduler with preemption enabled for system jobs
-var defaultSchedulerConfig = &structs.SchedulerConfiguration{
-	PreemptionConfig: structs.PreemptionConfig{
-		SystemSchedulerEnabled:  true,
-		BatchSchedulerEnabled:   false,
-		ServiceSchedulerEnabled: false,
-	},
-}
-
 // monitorLeadership is used to monitor if we acquire or lose our role
 // as the leader in the Raft cluster. There is some work the leader is
 // expected to do, so we must react to changes
@@ -1319,7 +1310,7 @@ func (s *Server) getOrCreateSchedulerConfig() *structs.SchedulerConfiguration {
 		return nil
 	}
 
-	req := structs.SchedulerSetConfigRequest{Config: *defaultSchedulerConfig}
+	req := structs.SchedulerSetConfigRequest{Config: s.config.DefaultSchedulerConfig}
 	if _, _, err = s.raftApply(structs.SchedulerConfigRequestType, req); err != nil {
 		s.logger.Named("core").Error("failed to initialize config", "error", err)
 		return nil

nomad/structs/operator.go

@@ -124,7 +124,7 @@ type AutopilotConfig struct {
 type SchedulerConfiguration struct {
 	// PreemptionConfig specifies whether to enable eviction of lower
 	// priority jobs to place higher priority jobs.
-	PreemptionConfig PreemptionConfig
+	PreemptionConfig PreemptionConfig `hcl:"preemption_config"`
 
 	// CreateIndex/ModifyIndex store the create/modify indexes of this configuration.
 	CreateIndex uint64
@@ -152,13 +152,13 @@ type SchedulerSetConfigurationResponse struct {
 // PreemptionConfig specifies whether preemption is enabled based on scheduler type
 type PreemptionConfig struct {
 	// SystemSchedulerEnabled specifies if preemption is enabled for system jobs
-	SystemSchedulerEnabled bool
+	SystemSchedulerEnabled bool `hcl:"system_scheduler_enabled"`
 
 	// BatchSchedulerEnabled specifies if preemption is enabled for batch jobs
-	BatchSchedulerEnabled bool
+	BatchSchedulerEnabled bool `hcl:"batch_scheduler_enabled"`
 
 	// ServiceSchedulerEnabled specifies if preemption is enabled for service jobs
-	ServiceSchedulerEnabled bool
+	ServiceSchedulerEnabled bool `hcl:"service_Scheduler_enabled"`
 }
 
 // SchedulerSetConfigRequest is used by the Operator endpoint to update the