From 161bec4ada6e4f28d8ce3e8c7dbcfdd70e728c1a Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Thu, 25 Jun 2020 09:30:16 -0700 Subject: [PATCH 1/3] Add cpuset_cpus to docker driver. --- drivers/docker/config.go | 2 ++ drivers/docker/driver.go | 7 ++++ drivers/docker/driver_test.go | 47 +++++++++++++++++++++++++++ website/pages/docs/drivers/docker.mdx | 9 +++++ 4 files changed, 65 insertions(+) diff --git a/drivers/docker/config.go b/drivers/docker/config.go index 134c9b75e6ad..5d63cbaac993 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -300,6 +300,7 @@ var ( "cap_add": hclspec.NewAttr("cap_add", "list(string)", false), "cap_drop": hclspec.NewAttr("cap_drop", "list(string)", false), "command": hclspec.NewAttr("command", "string", false), + "cpuset_cpus": hclspec.NewAttr("cpuset_cpus", "string", false), "cpu_hard_limit": hclspec.NewAttr("cpu_hard_limit", "bool", false), "cpu_cfs_period": hclspec.NewDefault( hclspec.NewAttr("cpu_cfs_period", "number", false), @@ -407,6 +408,7 @@ type TaskConfig struct { Command string `codec:"command"` CPUCFSPeriod int64 `codec:"cpu_cfs_period"` CPUHardLimit bool `codec:"cpu_hard_limit"` + CPUSetCPUs string `codec:"cpuset_cpus"` Devices []DockerDevice `codec:"devices"` DNSSearchDomains []string `codec:"dns_search_domains"` DNSOptions []string `codec:"dns_options"` diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 0cac7e460481..ec20edb69dff 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -823,6 +823,13 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T Runtime: containerRuntime, } + // This translates to docker create/run --cpuset-cpus option. + // --cpuset-cpus limit the specific CPUs or cores a container can use. + if driverConfig.CPUSetCPUs != "" { + logger.Debug(fmt.Sprintf("Setting CPUSetCPUs to %s", driverConfig.CPUSetCPUs)) + hostConfig.CPUSetCPUs = driverConfig.CPUSetCPUs + } + // Calculate CPU Quota // cfs_quota_us is the time per core, so we must // multiply the time by the number of cores available diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 400d86e21561..db6039127cd0 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -1381,6 +1381,53 @@ func TestDockerDriver_DNS(t *testing.T) { } +func TestDockerDriver_CPUSetCPUs(t *testing.T) { + if !tu.IsCI() { + t.Parallel() + } + testutil.DockerCompatible(t) + if runtime.GOOS == "windows" { + t.Skip("Windows does not support CPUSetCPUs.") + } + + testCases := []struct { + Name string + CPUSetCPUs string + }{ + { + Name: "Single CPU", + CPUSetCPUs: "0", + }, + { + Name: "Comma separated list of CPUs", + CPUSetCPUs: "0,1,2", + }, + { + Name: "Range of CPUs", + CPUSetCPUs: "0-3", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.Name, func(t *testing.T) { + task, cfg, ports := dockerTask(t) + defer freeport.Return(ports) + + cfg.CPUSetCPUs = testCase.CPUSetCPUs + require.NoError(t, task.EncodeConcreteDriverConfig(cfg)) + + client, d, handle, cleanup := dockerSetup(t, task, nil) + defer cleanup() + require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second)) + + container, err := client.InspectContainer(handle.containerID) + require.NoError(t, err) + + require.Equal(t, cfg.CPUSetCPUs, container.HostConfig.CPUSetCPUs) + }) + } +} + func TestDockerDriver_MemoryHardLimit(t *testing.T) { if !tu.IsCI() { t.Parallel() diff --git a/website/pages/docs/drivers/docker.mdx b/website/pages/docs/drivers/docker.mdx index 271fb5eb6650..07eb8b871d66 100644 --- a/website/pages/docs/drivers/docker.mdx +++ b/website/pages/docs/drivers/docker.mdx @@ -77,6 +77,15 @@ The `docker` driver supports the following configuration in the job spec. Only command = "my-command" } ``` +- `cpuset_cpus` - (Optional) CPUs in which to allow execution (0-3, 0,1). + +Limit the specific CPUs or cores a container can use. A comma-separated list or hyphen-separated range of CPUs a container can use, if you have more than one CPU. The first CPU is numbered 0. A valid value might be 0-3 (to use the first, second, third, and fourth CPU) or 1,3 (to use the second and fourth CPU). + + ```hcl + config { + cpuset_cpus = "0-3" + } + ``` - `dns_search_domains` - (Optional) A list of DNS search domains for the container to use. From 850694634facda14b4e87af0bc4fab091d0d2bc2 Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Thu, 25 Jun 2020 13:27:16 -0700 Subject: [PATCH 2/3] Fix circleci. --- drivers/docker/driver_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index db6039127cd0..5bfb8d6eec68 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -1400,11 +1400,11 @@ func TestDockerDriver_CPUSetCPUs(t *testing.T) { }, { Name: "Comma separated list of CPUs", - CPUSetCPUs: "0,1,2", + CPUSetCPUs: "0,1", }, { Name: "Range of CPUs", - CPUSetCPUs: "0-3", + CPUSetCPUs: "0-1", }, } From 8f27c08efd470d77d25a7c47f3ffdd0cdf95c9a3 Mon Sep 17 00:00:00 2001 From: Shishir Mahajan Date: Fri, 26 Jun 2020 11:14:53 -0700 Subject: [PATCH 3/3] Fix review comments. --- drivers/docker/driver.go | 1 - website/pages/docs/drivers/docker.mdx | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index ec20edb69dff..6a8a6833cc25 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -826,7 +826,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T // This translates to docker create/run --cpuset-cpus option. // --cpuset-cpus limit the specific CPUs or cores a container can use. if driverConfig.CPUSetCPUs != "" { - logger.Debug(fmt.Sprintf("Setting CPUSetCPUs to %s", driverConfig.CPUSetCPUs)) hostConfig.CPUSetCPUs = driverConfig.CPUSetCPUs } diff --git a/website/pages/docs/drivers/docker.mdx b/website/pages/docs/drivers/docker.mdx index 07eb8b871d66..9d75f9dcab1a 100644 --- a/website/pages/docs/drivers/docker.mdx +++ b/website/pages/docs/drivers/docker.mdx @@ -78,8 +78,13 @@ The `docker` driver supports the following configuration in the job spec. Only } ``` - `cpuset_cpus` - (Optional) CPUs in which to allow execution (0-3, 0,1). +Limit the specific CPUs or cores a container can use. A comma-separated list +or hyphen-separated range of CPUs a container can use, if you have more than +one CPU. The first CPU is numbered 0. A valid value might be 0-3 (to use the +first, second, third, and fourth CPU) or 1,3 (to use the second and fourth CPU). -Limit the specific CPUs or cores a container can use. A comma-separated list or hyphen-separated range of CPUs a container can use, if you have more than one CPU. The first CPU is numbered 0. A valid value might be 0-3 (to use the first, second, third, and fourth CPU) or 1,3 (to use the second and fourth CPU). +Note: `cpuset_cpus` pins the workload to the CPUs but doesn't give the workload +exclusive access to those CPUs. ```hcl config {