diff --git a/.changelog/13343.txt b/.changelog/13343.txt new file mode 100644 index 00000000000..0de61d8b840 --- /dev/null +++ b/.changelog/13343.txt @@ -0,0 +1,3 @@ +```release-note:improvement +vault: Add new configuration `disable_file` to prevent access to the Vault token by tasks that use `image` filesystem isolation +``` diff --git a/api/tasks.go b/api/tasks.go index 65344290510..188fa8649bb 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -919,6 +919,7 @@ type Vault struct { Policies []string `hcl:"policies,optional"` Namespace *string `mapstructure:"namespace" hcl:"namespace,optional"` Env *bool `hcl:"env,optional"` + DisableFile *bool `mapstructure:"disable_file" hcl:"disable_file,optional"` ChangeMode *string `mapstructure:"change_mode" hcl:"change_mode,optional"` ChangeSignal *string `mapstructure:"change_signal" hcl:"change_signal,optional"` } @@ -927,6 +928,9 @@ func (v *Vault) Canonicalize() { if v.Env == nil { v.Env = pointerOf(true) } + if v.DisableFile == nil { + v.DisableFile = pointerOf(false) + } if v.Namespace == nil { v.Namespace = pointerOf("") } diff --git a/api/tasks_test.go b/api/tasks_test.go index 10b917b43fe..231993906fd 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -460,6 +460,7 @@ func TestTask_Canonicalize_Vault(t *testing.T) { input: &Vault{}, expected: &Vault{ Env: pointerOf(true), + DisableFile: pointerOf(false), Namespace: pointerOf(""), ChangeMode: pointerOf("restart"), ChangeSignal: pointerOf("SIGHUP"), diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 63752476178..8ef9c6c658e 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -60,6 +60,10 @@ var ( // directory TaskSecrets = "secrets" + // TaskPrivate is the name of the private directory inside each task + // directory + TaskPrivate = "private" + // TaskDirs is the set of directories created in each tasks directory. TaskDirs = map[string]os.FileMode{TmpDirName: os.ModeSticky | 0777} @@ -306,6 +310,13 @@ func (d *AllocDir) UnmountAll() error { } } + if pathExists(dir.PrivateDir) { + if err := removeSecretDir(dir.PrivateDir); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("failed to remove the private dir %q: %v", dir.PrivateDir, err)) + } + } + // Unmount dev/ and proc/ have been mounted. if err := dir.unmountSpecialDirs(); err != nil { mErr.Errors = append(mErr.Errors, err) @@ -447,6 +458,10 @@ func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) { d.mu.RUnlock() return nil, fmt.Errorf("Reading secret file prohibited: %s", path) } + if filepath.HasPrefix(p, dir.PrivateDir) { + d.mu.RUnlock() + return nil, fmt.Errorf("Reading private file prohibited: %s", path) + } } d.mu.RUnlock() diff --git a/client/allocdir/task_dir.go b/client/allocdir/task_dir.go index 120212bf233..eb6afc6227f 100644 --- a/client/allocdir/task_dir.go +++ b/client/allocdir/task_dir.go @@ -41,6 +41,10 @@ type TaskDir struct { // /secrets/ SecretsDir string + // PrivateDir is the path to private/ directory on the host + // /private/ + PrivateDir string + // skip embedding these paths in chroots. Used for avoiding embedding // client.alloc_dir recursively. skip map[string]struct{} @@ -68,6 +72,7 @@ func newTaskDir(logger hclog.Logger, clientAllocDir, allocDir, taskName string) SharedTaskDir: filepath.Join(taskDir, SharedAllocName), LocalDir: filepath.Join(taskDir, TaskLocal), SecretsDir: filepath.Join(taskDir, TaskSecrets), + PrivateDir: filepath.Join(taskDir, TaskPrivate), skip: skip, logger: logger, } @@ -130,6 +135,15 @@ func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error { return err } + // Create the private directory + if err := createSecretDir(t.PrivateDir); err != nil { + return err + } + + if err := dropDirPermissions(t.PrivateDir, os.ModePerm); err != nil { + return err + } + // Build chroot if chroot filesystem isolation is going to be used if createChroot { if err := t.buildChroot(chroot); err != nil { diff --git a/client/allocrunner/taskrunner/task_runner_linux_test.go b/client/allocrunner/taskrunner/task_runner_linux_test.go new file mode 100644 index 00000000000..05de054a6b9 --- /dev/null +++ b/client/allocrunner/taskrunner/task_runner_linux_test.go @@ -0,0 +1,85 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package taskrunner + +import ( + "context" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/client/vaultclient" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/shoenig/test/must" +) + +func TestTaskRunner_DisableFileForVaultToken_UpgradePath(t *testing.T) { + ci.Parallel(t) + ci.SkipTestWithoutRootAccess(t) + + // Create test allocation with a Vault block. + alloc := mock.BatchAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Config = map[string]any{ + "run_for": "0s", + } + task.Vault = &structs.Vault{ + Policies: []string{"default"}, + } + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + // Remove private dir and write the Vault token to the secrets dir to + // simulate an old task. + err := conf.TaskDir.Build(false, nil) + must.NoError(t, err) + + err = syscall.Unmount(conf.TaskDir.PrivateDir, 0) + must.NoError(t, err) + err = os.Remove(conf.TaskDir.PrivateDir) + must.NoError(t, err) + + token := "1234" + tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + err = os.WriteFile(tokenPath, []byte(token), 0666) + must.NoError(t, err) + + // Setup a test Vault client. + handler := func(*structs.Allocation, []string) (map[string]string, error) { + return map[string]string{task.Name: token}, nil + } + vaultClient := conf.Vault.(*vaultclient.MockVaultClient) + vaultClient.DeriveTokenFn = handler + + // Start task runner and wait for task to finish. + tr, err := NewTaskRunner(conf) + must.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + go tr.Run() + time.Sleep(500 * time.Millisecond) + + testWaitForTaskToDie(t, tr) + + // Verify task exited successfully. + finalState := tr.TaskState() + must.Eq(t, structs.TaskStateDead, finalState.State) + must.False(t, finalState.Failed) + + // Verfiry token is in secrets dir. + tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + data, err := os.ReadFile(tokenPath) + must.NoError(t, err) + must.Eq(t, token, string(data)) + + // Varify token is not in private dir since the allocation doesn't have + // this path. + tokenPath = filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) + _, err = os.Stat(tokenPath) + must.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index b25a261f3f9..24757b6ce92 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1644,11 +1644,16 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) { require.False(t, finalState.Failed) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := os.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) + tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + data, err = os.ReadFile(tokenPath) + require.NoError(t, err) + require.Equal(t, token, string(data)) + // Kill task runner to trigger stop hooks tr.Kill(context.Background(), structs.NewTaskEvent("kill")) select { @@ -1672,6 +1677,57 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) { }) } +func TestTaskRunner_DisableFileForVaultToken(t *testing.T) { + ci.Parallel(t) + + // Create test allocation with a Vault block disabling the token file in + // the secrets dir. + alloc := mock.BatchAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Config = map[string]any{ + "run_for": "0s", + } + task.Vault = &structs.Vault{ + Policies: []string{"default"}, + DisableFile: true, + } + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + // Setup a test Vault client + token := "1234" + handler := func(*structs.Allocation, []string) (map[string]string, error) { + return map[string]string{task.Name: token}, nil + } + vaultClient := conf.Vault.(*vaultclient.MockVaultClient) + vaultClient.DeriveTokenFn = handler + + // Start task runner and wait for it to complete. + tr, err := NewTaskRunner(conf) + must.NoError(t, err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + go tr.Run() + + testWaitForTaskToDie(t, tr) + + // Verify task exited successfully. + finalState := tr.TaskState() + must.Eq(t, structs.TaskStateDead, finalState.State) + must.False(t, finalState.Failed) + + // Verify token is in the private dir. + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) + data, err := os.ReadFile(tokenPath) + must.NoError(t, err) + must.Eq(t, token, string(data)) + + // Verify token is not in secrets dir. + tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + _, err = os.Stat(tokenPath) + must.ErrorIs(t, err, os.ErrNotExist) +} + // TestTaskRunner_DeriveToken_Retry asserts that if a recoverable error is // returned when deriving a vault token a task will continue to block while // it's retried. @@ -1721,7 +1777,7 @@ func TestTaskRunner_DeriveToken_Retry(t *testing.T) { require.Equal(t, 1, count) // Check that the token is on disk - tokenPath := filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) + tokenPath := filepath.Join(conf.TaskDir.PrivateDir, vaultTokenFile) data, err := os.ReadFile(tokenPath) require.NoError(t, err) require.Equal(t, token, string(data)) diff --git a/client/allocrunner/taskrunner/vault_hook.go b/client/allocrunner/taskrunner/vault_hook.go index 2a2ce87af9c..b979e055c0c 100644 --- a/client/allocrunner/taskrunner/vault_hook.go +++ b/client/allocrunner/taskrunner/vault_hook.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "path" "path/filepath" "sync" "time" @@ -80,8 +81,13 @@ type vaultHook struct { ctx context.Context cancel context.CancelFunc - // tokenPath is the path in which to read and write the token - tokenPath string + // privateDirTokenPath is the path inside the task's private directory where + // the Vault token is read and written. + privateDirTokenPath string + + // secretsDirTokenPath is the path inside the task's secret directory where the + // Vault token is written unless disabled by the task. + secretsDirTokenPath string // alloc is the allocation alloc *structs.Allocation @@ -131,17 +137,24 @@ func (h *vaultHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRe // Try to recover a token if it was previously written in the secrets // directory recoveredToken := "" - h.tokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) - data, err := os.ReadFile(h.tokenPath) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("failed to recover vault token: %v", err) - } + h.privateDirTokenPath = filepath.Join(req.TaskDir.PrivateDir, vaultTokenFile) + h.secretsDirTokenPath = filepath.Join(req.TaskDir.SecretsDir, vaultTokenFile) + + // Handle upgrade path by searching for the previous token in all possible + // paths where the token may be. + for _, path := range []string{h.privateDirTokenPath, h.secretsDirTokenPath} { + data, err := os.ReadFile(path) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to recover vault token from %s: %v", path, err) + } - // Token file doesn't exist - } else { - // Store the recovered token - recoveredToken = string(data) + // Token file doesn't exist in this path. + } else { + // Store the recovered token + recoveredToken = string(data) + break + } } // Launch the token manager @@ -345,9 +358,25 @@ func (h *vaultHook) deriveVaultToken() (token string, exit bool) { // writeToken writes the given token to disk func (h *vaultHook) writeToken(token string) error { - if err := os.WriteFile(h.tokenPath, []byte(token), 0666); err != nil { + // Handle upgrade path by first checking if the tasks private directory + // exists. If it doesn't, this allocation probably existed before the + // private directory was introduced, so keep using the secret directory to + // prevent unnecessary errors during task recovery. + if _, err := os.Stat(path.Dir(h.privateDirTokenPath)); os.IsNotExist(err) { + if err := os.WriteFile(h.secretsDirTokenPath, []byte(token), 0666); err != nil { + return fmt.Errorf("failed to write vault token to secrets dir: %v", err) + } + return nil + } + + if err := os.WriteFile(h.privateDirTokenPath, []byte(token), 0600); err != nil { return fmt.Errorf("failed to write vault token: %v", err) } + if !h.vaultBlock.DisableFile { + if err := os.WriteFile(h.secretsDirTokenPath, []byte(token), 0666); err != nil { + return fmt.Errorf("failed to write vault token to secrets dir: %v", err) + } + } return nil } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index eeb114a4041..b12599141b6 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1265,6 +1265,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, Policies: apiTask.Vault.Policies, Namespace: *apiTask.Vault.Namespace, Env: *apiTask.Vault.Env, + DisableFile: *apiTask.Vault.DisableFile, ChangeMode: *apiTask.Vault.ChangeMode, ChangeSignal: *apiTask.Vault.ChangeSignal, } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 056f4a68677..9b6d9b0a831 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -2785,6 +2785,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Namespace: pointer.Of("ns1"), Policies: []string{"a", "b", "c"}, Env: pointer.Of(true), + DisableFile: pointer.Of(false), ChangeMode: pointer.Of("c"), ChangeSignal: pointer.Of("sighup"), }, @@ -3204,6 +3205,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Namespace: "ns1", Policies: []string{"a", "b", "c"}, Env: true, + DisableFile: false, ChangeMode: "c", ChangeSignal: "sighup", }, diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index 7172e1136dc..edd4e4b4cd7 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -244,6 +244,7 @@ etc/ lib/ lib64/ local/ +private/ proc/ secrets/ sys/ diff --git a/jobspec/parse.go b/jobspec/parse.go index b840a247f7f..7286b6e33a8 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -511,6 +511,7 @@ func parseVault(result *api.Vault, list *ast.ObjectList) error { "namespace", "policies", "env", + "disable_file", "change_mode", "change_signal", } diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index 7a4fdf74650..213e286a53f 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -213,8 +213,9 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error { // If we have a vault block, then parse that if o := listVal.Filter("vault"); len(o.Items) > 0 { tgVault := &api.Vault{ - Env: boolToPtr(true), - ChangeMode: stringToPtr("restart"), + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr("restart"), } if err := parseVault(tgVault, o); err != nil { diff --git a/jobspec/parse_job.go b/jobspec/parse_job.go index b51b256669b..46c3dd7573f 100644 --- a/jobspec/parse_job.go +++ b/jobspec/parse_job.go @@ -194,8 +194,9 @@ func parseJob(result *api.Job, list *ast.ObjectList) error { // If we have a vault block, then parse that if o := listVal.Filter("vault"); len(o.Items) > 0 { jobVault := &api.Vault{ - Env: boolToPtr(true), - ChangeMode: stringToPtr("restart"), + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr("restart"), } if err := parseVault(jobVault, o); err != nil { diff --git a/jobspec/parse_task.go b/jobspec/parse_task.go index a7a091c3227..5d596738d94 100644 --- a/jobspec/parse_task.go +++ b/jobspec/parse_task.go @@ -314,8 +314,9 @@ func parseTask(item *ast.ObjectItem, keys []string) (*api.Task, error) { // If we have a vault block, then parse that if o := listVal.Filter("vault"); len(o.Items) > 0 { v := &api.Vault{ - Env: boolToPtr(true), - ChangeMode: stringToPtr("restart"), + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr("restart"), } if err := parseVault(v, o); err != nil { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 6ccc7225497..3fe453c21a9 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -369,10 +369,11 @@ func TestParse(t *testing.T) { }, }, Vault: &api.Vault{ - Namespace: stringToPtr("ns1"), - Policies: []string{"foo", "bar"}, - Env: boolToPtr(true), - ChangeMode: stringToPtr(vaultChangeModeRestart), + Namespace: stringToPtr("ns1"), + Policies: []string{"foo", "bar"}, + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr(vaultChangeModeRestart), }, Templates: []*api.Template{ { @@ -435,6 +436,7 @@ func TestParse(t *testing.T) { Vault: &api.Vault{ Policies: []string{"foo", "bar"}, Env: boolToPtr(false), + DisableFile: boolToPtr(false), ChangeMode: stringToPtr(vaultChangeModeSignal), ChangeSignal: stringToPtr("SIGUSR1"), }, @@ -801,17 +803,19 @@ func TestParse(t *testing.T) { { Name: "redis", Vault: &api.Vault{ - Policies: []string{"group"}, - Env: boolToPtr(true), - ChangeMode: stringToPtr(vaultChangeModeRestart), + Policies: []string{"group"}, + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr(vaultChangeModeRestart), }, }, { Name: "redis2", Vault: &api.Vault{ - Policies: []string{"task"}, - Env: boolToPtr(false), - ChangeMode: stringToPtr(vaultChangeModeRestart), + Policies: []string{"task"}, + Env: boolToPtr(false), + DisableFile: boolToPtr(true), + ChangeMode: stringToPtr(vaultChangeModeRestart), }, }, }, @@ -822,9 +826,10 @@ func TestParse(t *testing.T) { { Name: "redis", Vault: &api.Vault{ - Policies: []string{"job"}, - Env: boolToPtr(true), - ChangeMode: stringToPtr(vaultChangeModeRestart), + Policies: []string{"job"}, + Env: boolToPtr(true), + DisableFile: boolToPtr(false), + ChangeMode: stringToPtr(vaultChangeModeRestart), }, }, }, diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index 17bd0a0fff5..66695c1c326 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -362,6 +362,7 @@ job "binstore-storagelocker" { vault { policies = ["foo", "bar"] env = false + disable_file = false change_mode = "signal" change_signal = "SIGUSR1" } diff --git a/jobspec/test-fixtures/vault_inheritance.hcl b/jobspec/test-fixtures/vault_inheritance.hcl index c844f8d1b7a..4ee5921874f 100644 --- a/jobspec/test-fixtures/vault_inheritance.hcl +++ b/jobspec/test-fixtures/vault_inheritance.hcl @@ -15,8 +15,9 @@ job "example" { task "redis2" { vault { - policies = ["task"] - env = false + policies = ["task"] + env = false + disable_file = true } } } diff --git a/jobspec2/parse_job.go b/jobspec2/parse_job.go index e1cd81cb400..001d23b0c2b 100644 --- a/jobspec2/parse_job.go +++ b/jobspec2/parse_job.go @@ -65,6 +65,9 @@ func normalizeVault(v *api.Vault) { if v.Env == nil { v.Env = pointer.Of(true) } + if v.DisableFile == nil { + v.DisableFile = pointer.Of(false) + } if v.ChangeMode == nil { v.ChangeMode = pointer.Of("restart") } diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index a93fa98a521..7d03c1fd48a 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -6887,6 +6887,7 @@ func TestTaskDiff(t *testing.T) { Vault: &Vault{ Policies: []string{"foo", "bar"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -6910,6 +6911,12 @@ func TestTaskDiff(t *testing.T) { Old: "", New: "SIGUSR1", }, + { + Type: DiffTypeAdded, + Name: "DisableFile", + Old: "", + New: "true", + }, { Type: DiffTypeAdded, Name: "Env", @@ -6947,6 +6954,7 @@ func TestTaskDiff(t *testing.T) { Vault: &Vault{ Policies: []string{"foo", "bar"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -6971,6 +6979,12 @@ func TestTaskDiff(t *testing.T) { Old: "SIGUSR1", New: "", }, + { + Type: DiffTypeDeleted, + Name: "DisableFile", + Old: "true", + New: "", + }, { Type: DiffTypeDeleted, Name: "Env", @@ -7009,6 +7023,7 @@ func TestTaskDiff(t *testing.T) { Namespace: "ns1", Policies: []string{"foo", "bar"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -7018,6 +7033,7 @@ func TestTaskDiff(t *testing.T) { Namespace: "ns2", Policies: []string{"bar", "baz"}, Env: false, + DisableFile: false, ChangeMode: "restart", ChangeSignal: "foo", }, @@ -7041,6 +7057,12 @@ func TestTaskDiff(t *testing.T) { Old: "SIGUSR1", New: "foo", }, + { + Type: DiffTypeEdited, + Name: "DisableFile", + Old: "true", + New: "false", + }, { Type: DiffTypeEdited, Name: "Env", @@ -7086,6 +7108,7 @@ func TestTaskDiff(t *testing.T) { Namespace: "ns1", Policies: []string{"foo", "bar"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -7095,6 +7118,7 @@ func TestTaskDiff(t *testing.T) { Namespace: "ns1", Policies: []string{"bar", "baz"}, Env: true, + DisableFile: true, ChangeMode: "signal", ChangeSignal: "SIGUSR1", }, @@ -7118,6 +7142,12 @@ func TestTaskDiff(t *testing.T) { Old: "SIGUSR1", New: "SIGUSR1", }, + { + Type: DiffTypeNone, + Name: "DisableFile", + Old: "true", + New: "true", + }, { Type: DiffTypeNone, Name: "Env", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 83ef6ce8861..af9ea2a3b9a 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -9760,6 +9760,10 @@ type Vault struct { // variable Env bool + // DisableFile marks whether the Vault Token should be exposed in the file + // vault_token in the task's secrets directory. + DisableFile bool + // ChangeMode is used to configure the task's behavior when the Vault // token changes because the original token could not be renewed in time. ChangeMode string @@ -9769,13 +9773,6 @@ type Vault struct { ChangeSignal string } -func DefaultVaultBlock() *Vault { - return &Vault{ - Env: true, - ChangeMode: VaultChangeModeRestart, - } -} - func (v *Vault) Equal(o *Vault) bool { if v == nil || o == nil { return v == o @@ -9787,6 +9784,8 @@ func (v *Vault) Equal(o *Vault) bool { return false case v.Env != o.Env: return false + case v.DisableFile != o.DisableFile: + return false case v.ChangeMode != o.ChangeMode: return false case v.ChangeSignal != o.ChangeSignal: diff --git a/website/content/docs/concepts/filesystem.mdx b/website/content/docs/concepts/filesystem.mdx index c0514750e9a..74dff6d50a0 100644 --- a/website/content/docs/concepts/filesystem.mdx +++ b/website/content/docs/concepts/filesystem.mdx @@ -29,10 +29,12 @@ allocation directory like the one below. │ └── tmp ├── task1 │ ├── local +│ ├── private │ ├── secrets │ └── tmp └── task2 ├── local + ├── private ├── secrets └── tmp ``` @@ -68,6 +70,17 @@ allocation directory like the one below. `NOMAD_TASK_DIR`. Note this is not the same as the "task working directory". This directory is private to the task. + - **«taskname»/private/**: This directory is used by Nomad to store private files + related to the allocation, such as Vault tokens, that are not shared with tasks + when using [`image` isolation](#image-isolation). The contents of files in this + directory cannot be read by the `nomad alloc fs` command or the via Nomad's + API. + + While not shared with tasks that use image isolation, this + path is still accessible by tasks using + chroot or none isolation + + - **«taskname»/secrets/**: This directory is the location provided to the task as `NOMAD_SECRETS_DIR`. The contents of files in this directory cannot be read by the `nomad alloc fs` command. It can be used to store secret data that @@ -97,6 +110,7 @@ drwxrwxrwx 4.0 KiB 2020-10-27T18:00:32Z tmp/ $ nomad alloc fs c0b2245f task1/ Mode Size Modified Time Name drwxrwxrwx 4.0 KiB 2020-10-27T18:00:33Z local/ +drwxrwxrwx 60 B 2020-10-27T18:00:32Z private/ drwxrwxrwx 60 B 2020-10-27T18:00:32Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T18:00:32Z tmp/ ``` @@ -150,6 +164,7 @@ minimal filesystem tree: │ └── tmp └── task1 ├── local + ├── private ├── secrets └── tmp ``` @@ -165,6 +180,7 @@ drwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z task1/ $ nomad alloc fs b0686b27 task1 Mode Size Modified Time Name drwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z local/ +drwxrwxrwx 60 B 2020-10-27T18:51:54Z private/ drwxrwxrwx 60 B 2020-10-27T18:51:54Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T18:51:54Z tmp/ @@ -287,6 +303,7 @@ contents], in addition to the `NOMAD_ALLOC_DIR`, `NOMAD_TASK_DIR`, and ├── lib32 ├── lib64 ├── local + ├── private ├── proc ├── run ├── sbin @@ -315,6 +332,7 @@ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib32/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z lib64/ drwxrwxrwx 4.0 KiB 2020-10-27T19:05:22Z local/ +drwxrwxrwx 60 B 2020-10-27T19:05:22Z private/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:24Z proc/ drwxr-xr-x 4.0 KiB 2020-10-27T19:05:22Z run/ drwxr-xr-x 12 KiB 2020-10-27T19:05:22Z sbin/ @@ -334,6 +352,7 @@ $ nomad alloc exec eebd13a7 /bin/sh $ mount ... /dev/mapper/root on /alloc type ext4 (rw,relatime,errors=remount-ro,data=ordered) +tmpfs on /private type tmpfs (rw,noexec,relatime,size=1024k) tmpfs on /secrets type tmpfs (rw,noexec,relatime,size=1024k) ... ``` @@ -377,6 +396,7 @@ minimal filesystem tree: └── task3 ├── executor.out ├── local + ├── private ├── secrets └── tmp ``` @@ -388,6 +408,7 @@ $ nomad alloc fs 87ec7d12 task3 Mode Size Modified Time Name -rw-r--r-- 140 B 2020-10-27T19:15:33Z executor.out drwxrwxrwx 4.0 KiB 2020-10-27T19:15:33Z local/ +drwxrwxrwx 60 B 2020-10-27T19:15:33Z private/ drwxrwxrwx 60 B 2020-10-27T19:15:33Z secrets/ dtrwxrwxrwx 4.0 KiB 2020-10-27T19:15:33Z tmp/ ``` diff --git a/website/content/docs/job-specification/vault.mdx b/website/content/docs/job-specification/vault.mdx index 20fe3ab27cf..cfe5a1893c6 100644 --- a/website/content/docs/job-specification/vault.mdx +++ b/website/content/docs/job-specification/vault.mdx @@ -45,6 +45,7 @@ to the secret directory at `secrets/vault_token` and by injecting a `VAULT_TOKEN environment variable. If the Nomad cluster is [configured](/nomad/docs/configuration/vault#namespace) to use [Vault Namespaces](/vault/docs/enterprise/namespaces), a `VAULT_NAMESPACE` environment variable will be injected whenever `VAULT_TOKEN` is set. +This behavior can be altered using the `env` and `file` parameters. If Nomad is unable to renew the Vault token (perhaps due to a Vault outage or network error), the client will attempt to retrieve a new Vault token. If successful, the @@ -70,6 +71,19 @@ with Vault as well. - `env` `(bool: true)` - Specifies if the `VAULT_TOKEN` and `VAULT_NAMESPACE` environment variables should be set when starting the task. +- `disable_file` `(bool: false)` - Specifies if the Vault token should be + written to `secrets/vault_token`. + + While the secrets path is not shared with tasks that + use + image + filesystem isolation, it is still accessible by tasks using + chroot + or none + isolation. + + + - `namespace` `(string: "")` - Specifies the Vault Namespace to use for the task. The Nomad client will retrieve a Vault token that is scoped to this particular namespace. @@ -109,6 +123,58 @@ vault { } ``` +### Private Token and Change Modes + +This example retrieves a Vault token that is not shared with the task when using +a driver that provides `image` isolation like [Docker][docker]. + +This allows Nomad to use a powerful Vault token that interacts with the task's +[`template`][template] stanzas to issue all kinds of secrets (e.g., database +secrets, other vault tokens, etc.), without sharing that issuing power with +the task itself: + +```hcl +vault { + policies = ["tls-policy", "nomad-job-policy"] + change_mode = "noop" + env = false + file = false +} + +template { + data = <<-EOH +{{with secret "auth/token/create/nomad-job" "policies=examplepolicy"}}{{.Auth.ClientToken}}{{ end }} +EOH + + destination = "${NOMAD_SECRETS_DIR}/examplepolicy.token" + change_mode = "noop" + perms = "600" +} + +template { + data = <<-EOH +{{ with secret "pki_int/issue/nomad-task" + "common_name=example.service.consul" "ttl=72h" + "alt_names=localhost" "ip_sans=127.0.0.1"}} +{{ .Data.certificate }} +{{ .Data.private_key }} +{{ end }} +EOH + + destination = "${NOMAD_SECRETS_DIR}/client.crt" + change_mode = "restart" + perms = "600" +} +``` + +The example above uses `change_mode = "noop"` in the `template` stanza for +`examplepolicy.token`, which means that the task's workload is responsible for +detecting and handling changes to that file. In contrast, the `template` stanza +for `client.crt` is configured so that Nomad will restart the task whenever +the certificate is reissued, as indicated by `change_mode = "restart"` +(which is the default value for `change_mode`). + + ### Vault Namespace This example shows specifying a particular Vault namespace for a given task. @@ -125,8 +191,7 @@ vault { } ``` +[docker]: /nomad/docs/drivers/docker "Docker Driver" [restart]: /nomad/docs/job-specification/restart "Nomad restart Job Specification" - [template]: /nomad/docs/job-specification/template "Nomad template Job Specification" - [vault]: https://www.vaultproject.io/ "Vault by HashiCorp"