From c9edc456617697803160e18c20d6d96a08baf7a6 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Mon, 29 Apr 2019 18:48:07 -0500 Subject: [PATCH 1/2] Add config to disable preemption for batch/service jobs --- api/operator.go | 2 ++ command/agent/operator_endpoint.go | 5 ++++- command/agent/operator_endpoint_test.go | 17 +++++++++++++---- nomad/fsm_test.go | 7 +++++-- nomad/leader.go | 2 ++ nomad/structs/operator.go | 6 ++++++ 6 files changed, 32 insertions(+), 7 deletions(-) diff --git a/api/operator.go b/api/operator.go index db6dffdd6b2a..2e70bfa94af8 100644 --- a/api/operator.go +++ b/api/operator.go @@ -140,6 +140,8 @@ type SchedulerSetConfigurationResponse struct { // PreemptionConfig specifies whether preemption is enabled based on scheduler type type PreemptionConfig struct { SystemSchedulerEnabled bool + BatchEnabled bool + ServiceEnabled bool } // SchedulerGetConfiguration is used to query the current Scheduler configuration. diff --git a/command/agent/operator_endpoint.go b/command/agent/operator_endpoint.go index 5e391ac722c2..0cfe9f741dd0 100644 --- a/command/agent/operator_endpoint.go +++ b/command/agent/operator_endpoint.go @@ -250,7 +250,10 @@ func (s *HTTPServer) schedulerUpdateConfig(resp http.ResponseWriter, req *http.R } args.Config = structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{SystemSchedulerEnabled: conf.PreemptionConfig.SystemSchedulerEnabled}, + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: conf.PreemptionConfig.SystemSchedulerEnabled, + BatchEnabled: conf.PreemptionConfig.BatchEnabled, + ServiceEnabled: conf.PreemptionConfig.ServiceEnabled}, } // Check for cas value diff --git a/command/agent/operator_endpoint_test.go b/command/agent/operator_endpoint_test.go index 1f1278c839a6..21d8aba2033b 100644 --- a/command/agent/operator_endpoint_test.go +++ b/command/agent/operator_endpoint_test.go @@ -272,6 +272,8 @@ func TestOperator_SchedulerGetConfiguration(t *testing.T) { out, ok := obj.(structs.SchedulerConfigurationResponse) require.True(ok) require.True(out.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) + require.True(out.SchedulerConfig.PreemptionConfig.BatchEnabled) + require.True(out.SchedulerConfig.PreemptionConfig.ServiceEnabled) }) } @@ -280,7 +282,8 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { require := require.New(t) body := bytes.NewBuffer([]byte(`{"PreemptionConfig": { - "SystemSchedulerEnabled": true + "SystemSchedulerEnabled": true, + "ServiceEnabled": true }}`)) req, _ := http.NewRequest("PUT", "/v1/operator/scheduler/configuration", body) resp := httptest.NewRecorder() @@ -301,6 +304,7 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) { err = s.RPC("Operator.SchedulerGetConfiguration", &args, &reply) require.Nil(err) require.True(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) + require.True(reply.SchedulerConfig.PreemptionConfig.ServiceEnabled) }) } @@ -309,7 +313,8 @@ func TestOperator_SchedulerCASConfiguration(t *testing.T) { httpTest(t, nil, func(s *TestAgent) { require := require.New(t) body := bytes.NewBuffer([]byte(`{"PreemptionConfig": { - "SystemSchedulerEnabled": true + "SystemSchedulerEnabled": true, + "BatchEnabled":true }}`)) req, _ := http.NewRequest("PUT", "/v1/operator/scheduler/configuration", body) resp := httptest.NewRecorder() @@ -331,11 +336,13 @@ func TestOperator_SchedulerCASConfiguration(t *testing.T) { t.Fatalf("err: %v", err) } require.True(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) + require.True(reply.SchedulerConfig.PreemptionConfig.BatchEnabled) // Create a CAS request, bad index { buf := bytes.NewBuffer([]byte(`{"PreemptionConfig": { - "SystemSchedulerEnabled": false + "SystemSchedulerEnabled": false, + "BatchEnabled":true }}`)) req, _ := http.NewRequest("PUT", fmt.Sprintf("/v1/operator/scheduler/configuration?cas=%d", reply.QueryMeta.Index-1), buf) resp := httptest.NewRecorder() @@ -351,7 +358,8 @@ func TestOperator_SchedulerCASConfiguration(t *testing.T) { // Create a CAS request, good index { buf := bytes.NewBuffer([]byte(`{"PreemptionConfig": { - "SystemSchedulerEnabled": false + "SystemSchedulerEnabled": false, + "BatchEnabled":false }}`)) req, _ := http.NewRequest("PUT", fmt.Sprintf("/v1/operator/scheduler/configuration?cas=%d", reply.QueryMeta.Index), buf) resp := httptest.NewRecorder() @@ -369,5 +377,6 @@ func TestOperator_SchedulerCASConfiguration(t *testing.T) { t.Fatalf("err: %v", err) } require.False(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) + require.False(reply.SchedulerConfig.PreemptionConfig.BatchEnabled) }) } diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 0c3a846a85fc..ff30ff31a6ac 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -2971,11 +2971,12 @@ func TestFSM_SchedulerConfig(t *testing.T) { require := require.New(t) - // Set the autopilot config using a request. + // Set the scheduler config using a request. req := structs.SchedulerSetConfigRequest{ Config: structs.SchedulerConfiguration{ PreemptionConfig: structs.PreemptionConfig{ SystemSchedulerEnabled: true, + BatchEnabled: true, }, }, } @@ -2992,10 +2993,11 @@ func TestFSM_SchedulerConfig(t *testing.T) { require.Nil(err) require.Equal(config.PreemptionConfig.SystemSchedulerEnabled, req.Config.PreemptionConfig.SystemSchedulerEnabled) + require.Equal(config.PreemptionConfig.BatchEnabled, req.Config.PreemptionConfig.BatchEnabled) // Now use CAS and provide an old index req.CAS = true - req.Config.PreemptionConfig = structs.PreemptionConfig{SystemSchedulerEnabled: false} + req.Config.PreemptionConfig = structs.PreemptionConfig{SystemSchedulerEnabled: false, BatchEnabled: false} req.Config.ModifyIndex = config.ModifyIndex - 1 buf, err = structs.Encode(structs.SchedulerConfigRequestType, req) require.Nil(err) @@ -3009,4 +3011,5 @@ func TestFSM_SchedulerConfig(t *testing.T) { require.Nil(err) // Verify that preemption is still enabled require.True(config.PreemptionConfig.SystemSchedulerEnabled) + require.True(config.PreemptionConfig.BatchEnabled) } diff --git a/nomad/leader.go b/nomad/leader.go index 17c8199962fc..d99e340bade6 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -48,6 +48,8 @@ var minSchedulerConfigVersion = version.Must(version.NewVersion("0.9.0")) var defaultSchedulerConfig = &structs.SchedulerConfiguration{ PreemptionConfig: structs.PreemptionConfig{ SystemSchedulerEnabled: true, + BatchEnabled: true, + ServiceEnabled: true, }, } diff --git a/nomad/structs/operator.go b/nomad/structs/operator.go index cad46de9a040..cb1f5841863f 100644 --- a/nomad/structs/operator.go +++ b/nomad/structs/operator.go @@ -153,6 +153,12 @@ type SchedulerSetConfigurationResponse struct { type PreemptionConfig struct { // SystemSchedulerEnabled specifies if preemption is enabled for system jobs SystemSchedulerEnabled bool + + // BatchEnabled specifies if preemption is enabled for batch jobs + BatchEnabled bool + + // ServiceEnabled specifies if preemption is enabled for service jobs + ServiceEnabled bool } // SchedulerSetConfigRequest is used by the Operator endpoint to update the From a9d45c5048675a710ba785fbaacb999589651219 Mon Sep 17 00:00:00 2001 From: Preetha Appan Date: Fri, 3 May 2019 14:06:12 -0500 Subject: [PATCH 2/2] Rename to match system scheduler config. Also added docs --- api/operator.go | 6 +++--- command/agent/operator_endpoint.go | 6 +++--- command/agent/operator_endpoint_test.go | 18 +++++++++--------- internal/testing/apitests/operator_test.go | 7 ++++--- nomad/fsm_test.go | 8 ++++---- nomad/leader.go | 6 +++--- nomad/structs/operator.go | 8 ++++---- website/source/api/operator.html.md | 16 ++++++++++++++-- 8 files changed, 44 insertions(+), 31 deletions(-) diff --git a/api/operator.go b/api/operator.go index 2e70bfa94af8..72d8fbdd2484 100644 --- a/api/operator.go +++ b/api/operator.go @@ -139,9 +139,9 @@ type SchedulerSetConfigurationResponse struct { // PreemptionConfig specifies whether preemption is enabled based on scheduler type type PreemptionConfig struct { - SystemSchedulerEnabled bool - BatchEnabled bool - ServiceEnabled bool + SystemSchedulerEnabled bool + BatchSchedulerEnabled bool + ServiceSchedulerEnabled bool } // SchedulerGetConfiguration is used to query the current Scheduler configuration. diff --git a/command/agent/operator_endpoint.go b/command/agent/operator_endpoint.go index 0cfe9f741dd0..31e86d21a42c 100644 --- a/command/agent/operator_endpoint.go +++ b/command/agent/operator_endpoint.go @@ -251,9 +251,9 @@ func (s *HTTPServer) schedulerUpdateConfig(resp http.ResponseWriter, req *http.R args.Config = structs.SchedulerConfiguration{ PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: conf.PreemptionConfig.SystemSchedulerEnabled, - BatchEnabled: conf.PreemptionConfig.BatchEnabled, - ServiceEnabled: conf.PreemptionConfig.ServiceEnabled}, + SystemSchedulerEnabled: conf.PreemptionConfig.SystemSchedulerEnabled, + BatchSchedulerEnabled: conf.PreemptionConfig.BatchSchedulerEnabled, + ServiceSchedulerEnabled: conf.PreemptionConfig.ServiceSchedulerEnabled}, } // Check for cas value diff --git a/command/agent/operator_endpoint_test.go b/command/agent/operator_endpoint_test.go index 21d8aba2033b..2c5222334e90 100644 --- a/command/agent/operator_endpoint_test.go +++ b/command/agent/operator_endpoint_test.go @@ -272,8 +272,8 @@ func TestOperator_SchedulerGetConfiguration(t *testing.T) { out, ok := obj.(structs.SchedulerConfigurationResponse) require.True(ok) require.True(out.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) - require.True(out.SchedulerConfig.PreemptionConfig.BatchEnabled) - require.True(out.SchedulerConfig.PreemptionConfig.ServiceEnabled) + require.True(out.SchedulerConfig.PreemptionConfig.BatchSchedulerEnabled) + require.True(out.SchedulerConfig.PreemptionConfig.ServiceSchedulerEnabled) }) } @@ -283,7 +283,7 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) { require := require.New(t) body := bytes.NewBuffer([]byte(`{"PreemptionConfig": { "SystemSchedulerEnabled": true, - "ServiceEnabled": true + "ServiceSchedulerEnabled": true }}`)) req, _ := http.NewRequest("PUT", "/v1/operator/scheduler/configuration", body) resp := httptest.NewRecorder() @@ -304,7 +304,7 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) { err = s.RPC("Operator.SchedulerGetConfiguration", &args, &reply) require.Nil(err) require.True(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) - require.True(reply.SchedulerConfig.PreemptionConfig.ServiceEnabled) + require.True(reply.SchedulerConfig.PreemptionConfig.ServiceSchedulerEnabled) }) } @@ -314,7 +314,7 @@ func TestOperator_SchedulerCASConfiguration(t *testing.T) { require := require.New(t) body := bytes.NewBuffer([]byte(`{"PreemptionConfig": { "SystemSchedulerEnabled": true, - "BatchEnabled":true + "BatchSchedulerEnabled":true }}`)) req, _ := http.NewRequest("PUT", "/v1/operator/scheduler/configuration", body) resp := httptest.NewRecorder() @@ -336,13 +336,13 @@ func TestOperator_SchedulerCASConfiguration(t *testing.T) { t.Fatalf("err: %v", err) } require.True(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) - require.True(reply.SchedulerConfig.PreemptionConfig.BatchEnabled) + require.True(reply.SchedulerConfig.PreemptionConfig.BatchSchedulerEnabled) // Create a CAS request, bad index { buf := bytes.NewBuffer([]byte(`{"PreemptionConfig": { "SystemSchedulerEnabled": false, - "BatchEnabled":true + "BatchSchedulerEnabled":true }}`)) req, _ := http.NewRequest("PUT", fmt.Sprintf("/v1/operator/scheduler/configuration?cas=%d", reply.QueryMeta.Index-1), buf) resp := httptest.NewRecorder() @@ -359,7 +359,7 @@ func TestOperator_SchedulerCASConfiguration(t *testing.T) { { buf := bytes.NewBuffer([]byte(`{"PreemptionConfig": { "SystemSchedulerEnabled": false, - "BatchEnabled":false + "BatchSchedulerEnabled":false }}`)) req, _ := http.NewRequest("PUT", fmt.Sprintf("/v1/operator/scheduler/configuration?cas=%d", reply.QueryMeta.Index), buf) resp := httptest.NewRecorder() @@ -377,6 +377,6 @@ func TestOperator_SchedulerCASConfiguration(t *testing.T) { t.Fatalf("err: %v", err) } require.False(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) - require.False(reply.SchedulerConfig.PreemptionConfig.BatchEnabled) + require.False(reply.SchedulerConfig.PreemptionConfig.BatchSchedulerEnabled) }) } diff --git a/internal/testing/apitests/operator_test.go b/internal/testing/apitests/operator_test.go index 636d8264bd90..95104b40efa4 100644 --- a/internal/testing/apitests/operator_test.go +++ b/internal/testing/apitests/operator_test.go @@ -24,7 +24,7 @@ func TestAPI_OperatorSchedulerGetSetConfiguration(t *testing.T) { require.True(config.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) // Change a config setting - newConf := &api.SchedulerConfiguration{PreemptionConfig: api.PreemptionConfig{SystemSchedulerEnabled: false}} + newConf := &api.SchedulerConfiguration{PreemptionConfig: api.PreemptionConfig{SystemSchedulerEnabled: false, BatchSchedulerEnabled: false}} resp, wm, err := operator.SchedulerSetConfiguration(newConf, nil) require.Nil(err) require.NotZero(wm.LastIndex) @@ -33,6 +33,7 @@ func TestAPI_OperatorSchedulerGetSetConfiguration(t *testing.T) { config, _, err = operator.SchedulerGetConfiguration(nil) require.Nil(err) require.False(config.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) + require.False(config.SchedulerConfig.PreemptionConfig.BatchSchedulerEnabled) } func TestAPI_OperatorSchedulerCASConfiguration(t *testing.T) { @@ -53,7 +54,7 @@ func TestAPI_OperatorSchedulerCASConfiguration(t *testing.T) { // Pass an invalid ModifyIndex { newConf := &api.SchedulerConfiguration{ - PreemptionConfig: api.PreemptionConfig{SystemSchedulerEnabled: false}, + PreemptionConfig: api.PreemptionConfig{SystemSchedulerEnabled: false, BatchSchedulerEnabled: false}, ModifyIndex: config.SchedulerConfig.ModifyIndex - 1, } resp, wm, err := operator.SchedulerCASConfiguration(newConf, nil) @@ -65,7 +66,7 @@ func TestAPI_OperatorSchedulerCASConfiguration(t *testing.T) { // Pass a valid ModifyIndex { newConf := &api.SchedulerConfiguration{ - PreemptionConfig: api.PreemptionConfig{SystemSchedulerEnabled: false}, + PreemptionConfig: api.PreemptionConfig{SystemSchedulerEnabled: false, BatchSchedulerEnabled: false}, ModifyIndex: config.SchedulerConfig.ModifyIndex, } resp, wm, err := operator.SchedulerCASConfiguration(newConf, nil) diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index ff30ff31a6ac..5d76a76343ca 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -2976,7 +2976,7 @@ func TestFSM_SchedulerConfig(t *testing.T) { Config: structs.SchedulerConfiguration{ PreemptionConfig: structs.PreemptionConfig{ SystemSchedulerEnabled: true, - BatchEnabled: true, + BatchSchedulerEnabled: true, }, }, } @@ -2993,11 +2993,11 @@ func TestFSM_SchedulerConfig(t *testing.T) { require.Nil(err) require.Equal(config.PreemptionConfig.SystemSchedulerEnabled, req.Config.PreemptionConfig.SystemSchedulerEnabled) - require.Equal(config.PreemptionConfig.BatchEnabled, req.Config.PreemptionConfig.BatchEnabled) + require.Equal(config.PreemptionConfig.BatchSchedulerEnabled, req.Config.PreemptionConfig.BatchSchedulerEnabled) // Now use CAS and provide an old index req.CAS = true - req.Config.PreemptionConfig = structs.PreemptionConfig{SystemSchedulerEnabled: false, BatchEnabled: false} + req.Config.PreemptionConfig = structs.PreemptionConfig{SystemSchedulerEnabled: false, BatchSchedulerEnabled: false} req.Config.ModifyIndex = config.ModifyIndex - 1 buf, err = structs.Encode(structs.SchedulerConfigRequestType, req) require.Nil(err) @@ -3011,5 +3011,5 @@ func TestFSM_SchedulerConfig(t *testing.T) { require.Nil(err) // Verify that preemption is still enabled require.True(config.PreemptionConfig.SystemSchedulerEnabled) - require.True(config.PreemptionConfig.BatchEnabled) + require.True(config.PreemptionConfig.BatchSchedulerEnabled) } diff --git a/nomad/leader.go b/nomad/leader.go index d99e340bade6..9ead8a2219ed 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -47,9 +47,9 @@ var minSchedulerConfigVersion = version.Must(version.NewVersion("0.9.0")) // Default configuration for scheduler with preemption enabled for system jobs var defaultSchedulerConfig = &structs.SchedulerConfiguration{ PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - BatchEnabled: true, - ServiceEnabled: true, + SystemSchedulerEnabled: true, + BatchSchedulerEnabled: true, + ServiceSchedulerEnabled: true, }, } diff --git a/nomad/structs/operator.go b/nomad/structs/operator.go index cb1f5841863f..9966a5e63927 100644 --- a/nomad/structs/operator.go +++ b/nomad/structs/operator.go @@ -154,11 +154,11 @@ type PreemptionConfig struct { // SystemSchedulerEnabled specifies if preemption is enabled for system jobs SystemSchedulerEnabled bool - // BatchEnabled specifies if preemption is enabled for batch jobs - BatchEnabled bool + // BatchSchedulerEnabled specifies if preemption is enabled for batch jobs + BatchSchedulerEnabled bool - // ServiceEnabled specifies if preemption is enabled for service jobs - ServiceEnabled bool + // ServiceSchedulerEnabled specifies if preemption is enabled for service jobs + ServiceSchedulerEnabled bool } // SchedulerSetConfigRequest is used by the Operator endpoint to update the diff --git a/website/source/api/operator.html.md b/website/source/api/operator.html.md index 5b032cc8ccaf..6af83d65d7ca 100644 --- a/website/source/api/operator.html.md +++ b/website/source/api/operator.html.md @@ -363,7 +363,9 @@ $ curl \ "CreateIndex": 5, "ModifyIndex": 5, "PreemptionConfig": { - "SystemSchedulerEnabled": true + "SystemSchedulerEnabled": true, + "BatchSchedulerEnabled": true, + "ServiceSchedulerEnabled": true, } } } @@ -379,6 +381,10 @@ $ curl \ - `PreemptionConfig` `(PreemptionConfig)` - Options to enable preemption for various schedulers. - `SystemSchedulerEnabled` `(bool: true)` - Specifies whether preemption for system jobs is enabled. Note that this defaults to true. + - `BatchSchedulerEnabled` `(bool: true)` (Enterprise Only) - Specifies whether preemption for batch jobs is enabled. Note that + this defaults to true. + - `ServiceSchedulerEnabled` `(bool: true)` (Enterprise Only) - Specifies whether preemption for service jobs is enabled. Note that + this defaults to true. - `CreateIndex` - The Raft index at which the config was created. - `ModifyIndex` - The Raft index at which the config was modified. @@ -409,7 +415,9 @@ The table below shows this endpoint's support for ```json { "PreemptionConfig": { - "EnablePreemption": false + "SystemSchedulerEnabled": false, + "BatchSchedulerEnabled": false, + "ServiceSchedulerEnabled": true, } } ``` @@ -417,3 +425,7 @@ The table below shows this endpoint's support for - `PreemptionConfig` `(PreemptionConfig)` - Options to enable preemption for various schedulers. - `SystemSchedulerEnabled` `(bool: true)` - Specifies whether preemption for system jobs is enabled. Note that if this is set to true, then system jobs can preempt any other jobs. + - `BatchSchedulerEnabled` `(bool: true)` (Enterprise Only) - Specifies whether preemption for batch jobs is enabled. Note that + if this is set to true, then batch jobs can preempt any other jobs. + - `ServiceSchedulerEnabled` `(bool: true)` (Enterprise Only) - Specifies whether preemption for service jobs is enabled. Note that + if this is set to true, then service jobs can preempt any other jobs.