Skip to content

Commit

Permalink
drivers/docker: reuse capabilities plumbing in docker driver
Browse files Browse the repository at this point in the history 
This changeset does not introduce any functional change for the
docker driver, but rather cleans up the implementation around
computing configured capabilities by re-using code written for
the exec/java task drivers.
  • Loading branch information
@shoenig
shoenig committed May 15, 2021
1 parent c53a948 commit 7984182
Show file tree
Hide file tree
Showing 8 changed files with 353 additions and 325 deletions.
122 changes: 4 additions & 118 deletions drivers/docker/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
Expand All @@ -23,10 +22,9 @@ import (
plugin "github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/drivers/docker/docklog"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/nomad/drivers/shared/eventer"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/drivers/shared/resolvconf"
"github.com/hashicorp/nomad/helper"
nstructs "github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/base"
"github.com/hashicorp/nomad/plugins/drivers"
Expand Down Expand Up @@ -913,8 +911,9 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
hostConfig.Privileged = driverConfig.Privileged

// set add/drop capabilities
hostConfig.CapAdd, hostConfig.CapDrop, err = d.getCaps(driverConfig)
if err != nil {
if hostConfig.CapAdd, hostConfig.CapDrop, err = capabilities.Delta(
capabilities.DockerDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop,
); err != nil {
return c, err
}

Expand Down Expand Up @@ -1184,119 +1183,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
}, nil
}

// getCaps computes the capabilities to supply to the --add-cap and --drop-cap
// options to the docker driver, which override the default capabilities enabled
// by docker itself.
func (d *Driver) getCaps(taskConfig *TaskConfig) ([]string, []string, error) {

// capabilities allowable by client docker plugin configuration
allowCaps := expandAllowCaps(d.config.AllowCaps)

// capabilities the task docker config is asking for based on the default
// capabilities allowable by nomad
desiredCaps, err := tweakCapabilities(nomadDefaultCaps(), taskConfig.CapAdd, taskConfig.CapDrop)
if err != nil {
return nil, nil, err
}

// capabilities the task is requesting that are NOT allowed by the docker plugin
if missing := missingCaps(allowCaps, desiredCaps); len(missing) > 0 {
return nil, nil, fmt.Errorf("Docker driver does not have the following caps allow-listed on this Nomad agent: %s", missing)
}

// capabilities that should be dropped relative to the docker default capabilities
dropCaps := capDrops(taskConfig.CapDrop, allowCaps)

return taskConfig.CapAdd, dropCaps, nil
}

// capDrops will compute the total dropped capabilities set
//
// {task cap_drop} U ({docker defaults} \ {driver allow caps})
func capDrops(dropCaps []string, allowCaps []string) []string {
dropSet := make(map[string]struct{})

for _, c := range normalizeCaps(dropCaps) {
dropSet[c] = struct{}{}
}

// if dropCaps includes ALL, no need to iterate every capability
if _, exists := dropSet["ALL"]; exists {
return []string{"ALL"}
}

dockerDefaults := helper.SliceStringToSet(normalizeCaps(dockerDefaultCaps()))
allowedCaps := helper.SliceStringToSet(normalizeCaps(allowCaps))

// find the docker default caps not in allowed caps
for dCap := range dockerDefaults {
if _, exists := allowedCaps[dCap]; !exists {
dropSet[dCap] = struct{}{}
}
}

drops := make([]string, 0, len(dropSet))
for c := range dropSet {
drops = append(drops, c)
}
sort.Strings(drops)
return drops
}

// expandAllowCaps returns the normalized set of allowable capabilities set
// for the docker plugin configuration.
func expandAllowCaps(allowCaps []string) []string {
if len(allowCaps) == 0 {
return nil
}

set := make(map[string]struct{}, len(allowCaps))

for _, rawCap := range allowCaps {
capability := strings.ToUpper(rawCap)
if capability == "ALL" {
for _, defCap := range normalizeCaps(executor.SupportedCaps(true)) {
set[defCap] = struct{}{}
}
} else {
set[capability] = struct{}{}
}
}

result := make([]string, 0, len(set))
for capability := range set {
result = append(result, capability)
}
sort.Strings(result)
return result
}

// missingCaps returns the set of elements in desired that are not present in
// allowed. The elements in desired are first upper-cased before comparison.
// The elements in allowed are assumed to be upper-cased.
func missingCaps(allowed, desired []string) []string {
_, missing := helper.SliceStringIsSubset(allowed, normalizeCaps(desired))
sort.Strings(missing)
return missing
}

// normalizeCaps returns a copy of caps with duplicate elements removed and all
// elements upper-cased.
func normalizeCaps(caps []string) []string {
set := make(map[string]struct{}, len(caps))
for _, c := range caps {
normal := strings.TrimPrefix(strings.ToUpper(c), "CAP_")
set[strings.ToUpper(normal)] = struct{}{}
}

result := make([]string, 0, len(set))
for c := range set {
result = append(result, c)
}
sort.Strings(result)
return result
}

func (d *Driver) toDockerMount(m *DockerMount, task *drivers.TaskConfig) (*docker.HostMount, error) {
hm, err := m.toDockerHostMount()
if err != nil {
Expand Down
191 changes: 12 additions & 179 deletions drivers/docker/driver_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
hclog "github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/client/testutil"
"github.com/hashicorp/nomad/drivers/shared/executor"
"github.com/hashicorp/nomad/helper/freeport"
"github.com/hashicorp/nomad/helper/pluginutils/hclspecutils"
"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
Expand Down Expand Up @@ -1387,44 +1386,44 @@ func TestDockerDriver_Capabilities(t *testing.T) {
{
Name: "default-allowlist-add-allowed",
CapAdd: []string{"fowner", "mknod"},
CapDrop: []string{"ALL"},
CapDrop: []string{"all"},
},
{
Name: "default-allowlist-add-forbidden",
CapAdd: []string{"net_admin"},
StartError: "NET_ADMIN",
StartError: "net_admin",
},
{
Name: "default-allowlist-drop-existing",
CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"},
CapDrop: []string{"fowner", "mknod", "net_raw"},
},
{
Name: "restrictive-allowlist-drop-all",
CapDrop: []string{"ALL"},
Allowlist: "FOWNER,MKNOD",
CapDrop: []string{"all"},
Allowlist: "fowner,mknod",
},
{
Name: "restrictive-allowlist-add-allowed",
CapAdd: []string{"fowner", "mknod"},
CapDrop: []string{"ALL"},
Allowlist: "fowner,mknod",
CapDrop: []string{"all"},
Allowlist: "mknod,fowner",
},
{
Name: "restrictive-allowlist-add-forbidden",
CapAdd: []string{"net_admin", "mknod"},
CapDrop: []string{"ALL"},
CapDrop: []string{"all"},
Allowlist: "fowner,mknod",
StartError: "NET_ADMIN",
StartError: "net_admin",
},
{
Name: "permissive-allowlist",
CapAdd: []string{"net_admin", "mknod"},
Allowlist: "ALL",
CapAdd: []string{"mknod", "net_admin"},
Allowlist: "all",
},
{
Name: "permissive-allowlist-add-all",
CapAdd: []string{"all"},
Allowlist: "ALL",
Allowlist: "all",
},
}

Expand Down Expand Up @@ -3064,169 +3063,3 @@ func TestDockerDriver_StopSignal(t *testing.T) {
})
}
}

func TestDockerCaps_normalizeCaps(t *testing.T) {
t.Run("empty", func(t *testing.T) {
result := normalizeCaps(nil)
require.Len(t, result, 0)
})

t.Run("mixed", func(t *testing.T) {
result := normalizeCaps([]string{
"DAC_OVERRIDE", "sys_chroot", "kill", "KILL",
})
require.Equal(t, []string{
"DAC_OVERRIDE", "KILL", "SYS_CHROOT",
}, result)
})
}

func TestDockerCaps_missingCaps(t *testing.T) {
allowed := []string{
"DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN",
}

t.Run("none missing", func(t *testing.T) {
result := missingCaps(allowed, []string{
"SYS_CHROOT", "chown", "KILL",
})
require.Equal(t, []string(nil), result)
})

t.Run("some missing", func(t *testing.T) {
result := missingCaps(allowed, []string{
"chown", "audit_write", "SETPCAP", "dac_override",
})
require.Equal(t, []string{"AUDIT_WRITE", "SETPCAP"}, result)
})
}

func TestDockerCaps_expandAllowCaps(t *testing.T) {
t.Run("empty", func(t *testing.T) {
result := expandAllowCaps(nil)
require.Empty(t, result)
})

t.Run("manual", func(t *testing.T) {
result := expandAllowCaps([]string{
"DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN",
})
require.Equal(t, []string{
"CHOWN", "DAC_OVERRIDE", "KILL", "SYS_CHROOT",
}, result)
})

t.Run("all", func(t *testing.T) {
result := expandAllowCaps([]string{"all"})
exp := normalizeCaps(executor.SupportedCaps(true))
sort.Strings(exp)
require.Equal(t, exp, result)
})
}

func TestDockerCaps_capDrops(t *testing.T) {
// docker default caps is always the same, task configured drop_caps and
// plugin config allow_caps may be altered

// This is the 90% use case, where NET_RAW is dropped, as Nomad's default
// capability allow-list is a subset of the docker default cap list.
t.Run("defaults", func(t *testing.T) {
result := capDrops(nil, nomadDefaultCaps())
require.Equal(t, []string{"NET_RAW"}, result)
})

// Users want to use ICMP (ping).
t.Run("enable net_raw", func(t *testing.T) {
result := capDrops(nil, append(nomadDefaultCaps(), "net_raw"))
require.Empty(t, result)
})

// The plugin is reduced in ability.
t.Run("enable minimal", func(t *testing.T) {
allow := []string{"setgid", "setuid", "chown", "kill"}
exp := []string{"AUDIT_WRITE", "DAC_OVERRIDE", "FOWNER", "FSETID", "MKNOD", "NET_BIND_SERVICE", "NET_RAW", "SETFCAP", "SETPCAP", "SYS_CHROOT"}
result := capDrops(nil, allow)
require.Equal(t, exp, result)
})

// The task drops abilities.
t.Run("task drops", func(t *testing.T) {
drops := []string{"audit_write", "fowner", "kill", "chown"}
exp := []string{"AUDIT_WRITE", "CHOWN", "FOWNER", "KILL", "NET_RAW"}
result := capDrops(drops, nomadDefaultCaps())
require.Equal(t, exp, result)
})

// Drop all mixed with others.
t.Run("task drops mix", func(t *testing.T) {
drops := []string{"audit_write", "all", "chown"}
exp := []string{"ALL"} // minimized
result := capDrops(drops, nomadDefaultCaps())
require.Equal(t, exp, result)
})
}

func TestDockerCaps_getCaps(t *testing.T) {
testutil.ExecCompatible(t) // tests require linux

t.Run("defaults", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: nomadDefaultCaps(),
}}
add, drop, err := d.getCaps(&TaskConfig{
CapAdd: nil, CapDrop: nil,
})
require.NoError(t, err)
require.Empty(t, add)
require.Equal(t, []string{"NET_RAW"}, drop)
})

t.Run("enable net_raw", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: append(nomadDefaultCaps(), "net_raw"),
}}
add, drop, err := d.getCaps(&TaskConfig{
CapAdd: nil, CapDrop: nil,
})
require.NoError(t, err)
require.Empty(t, add)
require.Empty(t, drop)
})

t.Run("block sys_time", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: nomadDefaultCaps(),
}}
_, _, err := d.getCaps(&TaskConfig{
CapAdd: []string{"SYS_TIME"},
CapDrop: nil,
})
require.EqualError(t, err, `Docker driver does not have the following caps allow-listed on this Nomad agent: [SYS_TIME]`)
})

t.Run("enable sys_time", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: append(nomadDefaultCaps(), "sys_time"),
}}
add, drop, err := d.getCaps(&TaskConfig{
CapAdd: []string{"SYS_TIME"},
CapDrop: nil,
})
require.NoError(t, err)
require.Equal(t, []string{"SYS_TIME"}, add)
require.Equal(t, []string{"NET_RAW"}, drop)
})

t.Run("task drops chown", func(t *testing.T) {
d := Driver{config: &DriverConfig{
AllowCaps: nomadDefaultCaps(),
}}
add, drop, err := d.getCaps(&TaskConfig{
CapAdd: nil,
CapDrop: []string{"chown"},
})
require.NoError(t, err)
require.Empty(t, add)
require.Equal(t, []string{"CHOWN", "NET_RAW"}, drop)
})
}
4 changes: 3 additions & 1 deletion drivers/exec/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,9 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
cfg.Mounts = append(cfg.Mounts, dnsMount)
}

caps, err := capabilities.Calculate(d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop)
caps, err := capabilities.Calculate(
capabilities.NomadDefaults(), d.config.AllowCaps, driverConfig.CapAdd, driverConfig.CapDrop,
)
if err != nil {
return nil, nil, err
}
Expand Down

0 comments on commit 7984182

Please sign in to comment.