Merge pull request #8453 from hashicorp/oss-multi-vault-ns

oss compoments for multi-vault namespaces

api/jobs.go

@@ -787,6 +787,7 @@ type Job struct {
 	Meta              map[string]string
 	ConsulToken       *string `mapstructure:"consul_token"`
 	VaultToken        *string `mapstructure:"vault_token"`
+	VaultNamespace    *string `mapstructure:"vault_namespace"`
 	NomadTokenID      *string `mapstructure:"nomad_token_id"`
 	Status            *string
 	StatusDescription *string
@@ -850,6 +851,9 @@ func (j *Job) Canonicalize() {
 	if j.VaultToken == nil {
 		j.VaultToken = stringToPtr("")
 	}
+	if j.VaultNamespace == nil {
+		j.VaultNamespace = stringToPtr("")
+	}
 	if j.NomadTokenID == nil {
 		j.NomadTokenID = stringToPtr("")
 	}

api/jobs_test.go

@@ -244,6 +244,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				AllAtOnce:         boolToPtr(false),
 				ConsulToken:       stringToPtr(""),
 				VaultToken:        stringToPtr(""),
+				VaultNamespace:    stringToPtr(""),
 				NomadTokenID:      stringToPtr(""),
 				Status:            stringToPtr(""),
 				StatusDescription: stringToPtr(""),
@@ -334,6 +335,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				AllAtOnce:         boolToPtr(false),
 				ConsulToken:       stringToPtr(""),
 				VaultToken:        stringToPtr(""),
+				VaultNamespace:    stringToPtr(""),
 				NomadTokenID:      stringToPtr(""),
 				Status:            stringToPtr(""),
 				StatusDescription: stringToPtr(""),
@@ -407,6 +409,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				AllAtOnce:         boolToPtr(false),
 				ConsulToken:       stringToPtr(""),
 				VaultToken:        stringToPtr(""),
+				VaultNamespace:    stringToPtr(""),
 				NomadTokenID:      stringToPtr(""),
 				Stop:              boolToPtr(false),
 				Stable:            boolToPtr(false),
@@ -573,6 +576,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				AllAtOnce:         boolToPtr(false),
 				ConsulToken:       stringToPtr(""),
 				VaultToken:        stringToPtr(""),
+				VaultNamespace:    stringToPtr(""),
 				NomadTokenID:      stringToPtr(""),
 				Stop:              boolToPtr(false),
 				Stable:            boolToPtr(false),
@@ -731,6 +735,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				AllAtOnce:         boolToPtr(false),
 				ConsulToken:       stringToPtr(""),
 				VaultToken:        stringToPtr(""),
+				VaultNamespace:    stringToPtr(""),
 				NomadTokenID:      stringToPtr(""),
 				Stop:              boolToPtr(false),
 				Stable:            boolToPtr(false),
@@ -817,6 +822,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				AllAtOnce:         boolToPtr(false),
 				ConsulToken:       stringToPtr(""),
 				VaultToken:        stringToPtr(""),
+				VaultNamespace:    stringToPtr(""),
 				NomadTokenID:      stringToPtr(""),
 				Stop:              boolToPtr(false),
 				Stable:            boolToPtr(false),
@@ -982,6 +988,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				AllAtOnce:         boolToPtr(false),
 				ConsulToken:       stringToPtr(""),
 				VaultToken:        stringToPtr(""),
+				VaultNamespace:    stringToPtr(""),
 				NomadTokenID:      stringToPtr(""),
 				Stop:              boolToPtr(false),
 				Stable:            boolToPtr(false),
@@ -1145,6 +1152,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 				AllAtOnce:         boolToPtr(false),
 				ConsulToken:       stringToPtr(""),
 				VaultToken:        stringToPtr(""),
+				VaultNamespace:    stringToPtr(""),
 				NomadTokenID:      stringToPtr(""),
 				Stop:              boolToPtr(false),
 				Stable:            boolToPtr(false),

api/tasks.go

@@ -812,6 +812,7 @@ func (tmpl *Template) Canonicalize() {
 
 type Vault struct {
 	Policies     []string
+	Namespace    *string `mapstructure:"namespace"`
 	Env          *bool
 	ChangeMode   *string `mapstructure:"change_mode"`
 	ChangeSignal *string `mapstructure:"change_signal"`
@@ -821,6 +822,9 @@ func (v *Vault) Canonicalize() {
 	if v.Env == nil {
 		v.Env = boolToPtr(true)
 	}
+	if v.Namespace == nil {
+		v.Namespace = stringToPtr("")
+	}
 	if v.ChangeMode == nil {
 		v.ChangeMode = stringToPtr("restart")
 	}

client/allocrunner/taskrunner/task_runner_getters.go

@@ -57,7 +57,13 @@ func (tr *TaskRunner) setVaultToken(token string) {
 	tr.vaultToken = token
 
 	// Update the task's environment
-	tr.envBuilder.SetVaultToken(token, tr.clientConfig.VaultConfig.Namespace, tr.task.Vault.Env)
+	taskNamespace := tr.task.Vault.Namespace
+
+	ns := tr.clientConfig.VaultConfig.Namespace
+	if taskNamespace != "" {
+		ns = taskNamespace
+	}
+	tr.envBuilder.SetVaultToken(token, ns, tr.task.Vault.Env)
 }
 
 // getDriverHandle returns a driver handle.

client/allocrunner/taskrunner/template/template.go

@@ -87,6 +87,9 @@ type TaskTemplateManagerConfig struct {
 	// VaultToken is the Vault token for the task.
 	VaultToken string
 
+	// VaultNamespace is the Vault namespace for the task
+	VaultNamespace string
+
 	// TaskDir is the task's directory
 	TaskDir string
 
@@ -655,9 +658,15 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 	if cc.VaultConfig != nil && cc.VaultConfig.IsEnabled() {
 		conf.Vault.Address = &cc.VaultConfig.Addr
 		conf.Vault.Token = &config.VaultToken
+
+		// Set the Vault Namespace. Passed in Task config has
+		// highest precedence.
 		if config.ClientConfig.VaultConfig.Namespace != "" {
 			conf.Vault.Namespace = &config.ClientConfig.VaultConfig.Namespace
 		}
+		if config.VaultNamespace != "" {
+			conf.Vault.Namespace = &config.VaultNamespace
+		}
 
 		if strings.HasPrefix(cc.VaultConfig.Addr, "https") || cc.VaultConfig.TLSCertFile != "" {
 			skipVerify := cc.VaultConfig.TLSSkipVerify != nil && *cc.VaultConfig.TLSSkipVerify

client/allocrunner/taskrunner/template/template_test.go

@@ -1377,6 +1377,41 @@ func TestTaskTemplateManager_Config_VaultNamespace(t *testing.T) {
 	assert.Equal(testNS, *ctconf.Vault.Namespace, "Vault Namespace Value")
 }
 
+// TestTaskTemplateManager_Config_VaultNamespace asserts the Vault namespace setting is
+// propagated to consul-template's configuration.
+func TestTaskTemplateManager_Config_VaultNamespace_TaskOverride(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+
+	testNS := "test-namespace"
+	c := config.DefaultConfig()
+	c.Node = mock.Node()
+	c.VaultConfig = &sconfig.VaultConfig{
+		Enabled:       helper.BoolToPtr(true),
+		Addr:          "https://localhost/",
+		TLSServerName: "notlocalhost",
+		Namespace:     testNS,
+	}
+
+	alloc := mock.Alloc()
+	overriddenNS := "new-namespace"
+
+	// Set the template manager config vault namespace
+	config := &TaskTemplateManagerConfig{
+		ClientConfig:   c,
+		VaultToken:     "token",
+		VaultNamespace: overriddenNS,
+		EnvBuilder:     taskenv.NewBuilder(c.Node, alloc, alloc.Job.TaskGroups[0].Tasks[0], c.Region),
+	}
+
+	ctmplMapping, err := parseTemplateConfigs(config)
+	assert.Nil(err, "Parsing Templates")
+
+	ctconf, err := newRunnerConfig(config, ctmplMapping)
+	assert.Nil(err, "Building Runner Config")
+	assert.Equal(overriddenNS, *ctconf.Vault.Namespace, "Vault Namespace Value")
+}
+
 func TestTaskTemplateManager_BlockedEvents(t *testing.T) {
 	// The tests sets a template that need keys 0, 1, 2, 3, 4,
 	// then subsequently sets 0, 1, 2 keys

client/allocrunner/taskrunner/template_hook.go

@@ -47,6 +47,9 @@ type templateHook struct {
 	// vaultToken is the current Vault token
 	vaultToken string
 
+	// vaultNamespace is the current Vault namespace
+	vaultNamespace string
+
 	// taskDir is the task directory
 	taskDir string
 }
@@ -75,6 +78,12 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
 	// Store the current Vault token and the task directory
 	h.taskDir = req.TaskDir.Dir
 	h.vaultToken = req.VaultToken
+
+	// Set vault namespace if specified
+	if req.Task.Vault != nil {
+		h.vaultNamespace = req.Task.Vault.Namespace
+	}
+
 	unblockCh, err := h.newManager()
 	if err != nil {
 		return err
@@ -98,6 +107,7 @@ func (h *templateHook) newManager() (unblock chan struct{}, err error) {
 		Templates:            h.config.templates,
 		ClientConfig:         h.config.clientConfig,
 		VaultToken:           h.vaultToken,
+		VaultNamespace:       h.vaultNamespace,
 		TaskDir:              h.taskDir,
 		EnvBuilder:           h.config.envBuilder,
 		MaxTemplateEventRate: template.DefaultMaxTemplateEventRate,

client/client.go

@@ -2457,6 +2457,7 @@ func (c *Client) deriveToken(alloc *structs.Allocation, taskNames []string, vcli
 	}
 
 	// Derive the tokens
+	// namespace is handled via nomad/vault
 	var resp structs.DeriveVaultTokenResponse
 	if err := c.RPC("Node.DeriveVaultToken", &req, &resp); err != nil {
 		vlogger.Error("error making derive token RPC", "error", err)

command/agent/job_endpoint.go

@@ -773,22 +773,23 @@ func ApiJobToStructJob(job *api.Job) *structs.Job {
 	job.Canonicalize()
 
 	j := &structs.Job{
-		Stop:        *job.Stop,
-		Region:      *job.Region,
-		Namespace:   *job.Namespace,
-		ID:          *job.ID,
-		ParentID:    *job.ParentID,
-		Name:        *job.Name,
-		Type:        *job.Type,
-		Priority:    *job.Priority,
-		AllAtOnce:   *job.AllAtOnce,
-		Datacenters: job.Datacenters,
-		Payload:     job.Payload,
-		Meta:        job.Meta,
-		ConsulToken: *job.ConsulToken,
-		VaultToken:  *job.VaultToken,
-		Constraints: ApiConstraintsToStructs(job.Constraints),
-		Affinities:  ApiAffinitiesToStructs(job.Affinities),
+		Stop:           *job.Stop,
+		Region:         *job.Region,
+		Namespace:      *job.Namespace,
+		ID:             *job.ID,
+		ParentID:       *job.ParentID,
+		Name:           *job.Name,
+		Type:           *job.Type,
+		Priority:       *job.Priority,
+		AllAtOnce:      *job.AllAtOnce,
+		Datacenters:    job.Datacenters,
+		Payload:        job.Payload,
+		Meta:           job.Meta,
+		ConsulToken:    *job.ConsulToken,
+		VaultToken:     *job.VaultToken,
+		VaultNamespace: *job.VaultNamespace,
+		Constraints:    ApiConstraintsToStructs(job.Constraints),
+		Affinities:     ApiAffinitiesToStructs(job.Affinities),
 	}
 
 	// Update has been pushed into the task groups. stagger and max_parallel are
@@ -976,6 +977,12 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta
 		for l, task := range taskGroup.Tasks {
 			t := &structs.Task{}
 			ApiTaskToStructsTask(task, t)
+
+			// Set the tasks vault namespace from Job if it was not
+			// specified by the task or group
+			if t.Vault != nil && t.Vault.Namespace == "" && job.VaultNamespace != "" {
+				t.Vault.Namespace = job.VaultNamespace
+			}
 			tg.Tasks[l] = t
 		}
 	}
@@ -1089,6 +1096,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) {
 	if apiTask.Vault != nil {
 		structsTask.Vault = &structs.Vault{
 			Policies:     apiTask.Vault.Policies,
+			Namespace:    *apiTask.Vault.Namespace,
 			Env:          *apiTask.Vault.Env,
 			ChangeMode:   *apiTask.Vault.ChangeMode,
 			ChangeSignal: *apiTask.Vault.ChangeSignal,

command/agent/job_endpoint_test.go

@@ -2121,6 +2121,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 							},
 						},
 						Vault: &api.Vault{
+							Namespace:    helper.StringToPtr("ns1"),
 							Policies:     []string{"a", "b", "c"},
 							Env:          helper.BoolToPtr(true),
 							ChangeMode:   helper.StringToPtr("c"),
@@ -2149,6 +2150,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 		},
 		ConsulToken:       helper.StringToPtr("abc123"),
 		VaultToken:        helper.StringToPtr("def456"),
+		VaultNamespace:    helper.StringToPtr("ghi789"),
 		Status:            helper.StringToPtr("status"),
 		StatusDescription: helper.StringToPtr("status_desc"),
 		Version:           helper.Uint64ToPtr(10),
@@ -2158,16 +2160,17 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 	}
 
 	expected := &structs.Job{
-		Stop:        true,
-		Region:      "global",
-		Namespace:   "foo",
-		ID:          "foo",
-		ParentID:    "lol",
-		Name:        "name",
-		Type:        "service",
-		Priority:    50,
-		AllAtOnce:   true,
-		Datacenters: []string{"dc1", "dc2"},
+		Stop:           true,
+		Region:         "global",
+		Namespace:      "foo",
+		VaultNamespace: "ghi789",
+		ID:             "foo",
+		ParentID:       "lol",
+		Name:           "name",
+		Type:           "service",
+		Priority:       50,
+		AllAtOnce:      true,
+		Datacenters:    []string{"dc1", "dc2"},
 		Constraints: []*structs.Constraint{
 			{
 				LTarget: "a",
@@ -2488,6 +2491,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 							},
 						},
 						Vault: &structs.Vault{
+							Namespace:    "ns1",
 							Policies:     []string{"a", "b", "c"},
 							Env:          true,
 							ChangeMode:   "c",

command/job_run.go

@@ -101,6 +101,11 @@ Run Options:
     the job file. This overrides the token found in $VAULT_TOKEN environment
     variable and that found in the job.
 
+  -vault-namespace
+    If set, the passed Vault namespace is stored in the job before sending to the
+    Nomad servers. This overrides the namespace found in $VAULT_NAMESPACE environment
+    variable and that found in the job.
+
   -verbose
     Display full information.
 `
@@ -119,6 +124,7 @@ func (c *JobRunCommand) AutocompleteFlags() complete.Flags {
 			"-verbose":         complete.PredictNothing,
 			"-consul-token":    complete.PredictNothing,
 			"-vault-token":     complete.PredictAnything,
+			"-vault-namespace": complete.PredictAnything,
 			"-output":          complete.PredictNothing,
 			"-policy-override": complete.PredictNothing,
 			"-preserve-counts": complete.PredictNothing,
@@ -133,7 +139,7 @@ func (c *JobRunCommand) Name() string { return "job run" }
 
 func (c *JobRunCommand) Run(args []string) int {
 	var detach, verbose, output, override, preserveCounts bool
-	var checkIndexStr, consulToken, vaultToken string
+	var checkIndexStr, consulToken, vaultToken, vaultNamespace string
 
 	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
 	flags.Usage = func() { c.Ui.Output(c.Help()) }
@@ -145,6 +151,7 @@ func (c *JobRunCommand) Run(args []string) int {
 	flags.StringVar(&checkIndexStr, "check-index", "", "")
 	flags.StringVar(&consulToken, "consul-token", "", "")
 	flags.StringVar(&vaultToken, "vault-token", "", "")
+	flags.StringVar(&vaultNamespace, "vault-namespace", "", "")
 
 	if err := flags.Parse(args); err != nil {
 		return 1
@@ -213,6 +220,15 @@ func (c *JobRunCommand) Run(args []string) int {
 		job.VaultToken = helper.StringToPtr(vaultToken)
 	}
 
+	// Parse the Vault namespace
+	if vaultNamespace == "" {
+		vaultNamespace = os.Getenv("VAULT_NAMESPACE")
+	}
+
+	if vaultNamespace != "" {
+		job.VaultNamespace = helper.StringToPtr(vaultNamespace)
+	}
+
 	if output {
 		req := struct {
 			Job *api.Job