Skip to content

Commit

Permalink
oss compoments for multi-vault namespaces
Browse files Browse the repository at this point in the history
adds in oss components to support enterprise multi-vault namespace feature

upgrade specific doc on vault multi-namespaces

vault docs

update test to reflect new error
  • Loading branch information
drewbailey committed Jul 24, 2020
1 parent f4eae8a commit 1981036
Show file tree
Hide file tree
Showing 30 changed files with 467 additions and 83 deletions.
4 changes: 4 additions & 0 deletions api/jobs.go
Original file line number Diff line number Diff line change
Expand Up @@ -787,6 +787,7 @@ type Job struct {
Meta map[string]string
ConsulToken *string `mapstructure:"consul_token"`
VaultToken *string `mapstructure:"vault_token"`
VaultNamespace *string `mapstructure:"vault_namespace"`
NomadTokenID *string `mapstructure:"nomad_token_id"`
Status *string
StatusDescription *string
Expand Down Expand Up @@ -850,6 +851,9 @@ func (j *Job) Canonicalize() {
if j.VaultToken == nil {
j.VaultToken = stringToPtr("")
}
if j.VaultNamespace == nil {
j.VaultNamespace = stringToPtr("")
}
if j.NomadTokenID == nil {
j.NomadTokenID = stringToPtr("")
}
Expand Down
8 changes: 8 additions & 0 deletions api/jobs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@ func TestJobs_Canonicalize(t *testing.T) {
AllAtOnce: boolToPtr(false),
ConsulToken: stringToPtr(""),
VaultToken: stringToPtr(""),
VaultNamespace: stringToPtr(""),
NomadTokenID: stringToPtr(""),
Status: stringToPtr(""),
StatusDescription: stringToPtr(""),
Expand Down Expand Up @@ -333,6 +334,7 @@ func TestJobs_Canonicalize(t *testing.T) {
AllAtOnce: boolToPtr(false),
ConsulToken: stringToPtr(""),
VaultToken: stringToPtr(""),
VaultNamespace: stringToPtr(""),
NomadTokenID: stringToPtr(""),
Status: stringToPtr(""),
StatusDescription: stringToPtr(""),
Expand Down Expand Up @@ -406,6 +408,7 @@ func TestJobs_Canonicalize(t *testing.T) {
AllAtOnce: boolToPtr(false),
ConsulToken: stringToPtr(""),
VaultToken: stringToPtr(""),
VaultNamespace: stringToPtr(""),
NomadTokenID: stringToPtr(""),
Stop: boolToPtr(false),
Stable: boolToPtr(false),
Expand Down Expand Up @@ -572,6 +575,7 @@ func TestJobs_Canonicalize(t *testing.T) {
AllAtOnce: boolToPtr(false),
ConsulToken: stringToPtr(""),
VaultToken: stringToPtr(""),
VaultNamespace: stringToPtr(""),
NomadTokenID: stringToPtr(""),
Stop: boolToPtr(false),
Stable: boolToPtr(false),
Expand Down Expand Up @@ -730,6 +734,7 @@ func TestJobs_Canonicalize(t *testing.T) {
AllAtOnce: boolToPtr(false),
ConsulToken: stringToPtr(""),
VaultToken: stringToPtr(""),
VaultNamespace: stringToPtr(""),
NomadTokenID: stringToPtr(""),
Stop: boolToPtr(false),
Stable: boolToPtr(false),
Expand Down Expand Up @@ -816,6 +821,7 @@ func TestJobs_Canonicalize(t *testing.T) {
AllAtOnce: boolToPtr(false),
ConsulToken: stringToPtr(""),
VaultToken: stringToPtr(""),
VaultNamespace: stringToPtr(""),
NomadTokenID: stringToPtr(""),
Stop: boolToPtr(false),
Stable: boolToPtr(false),
Expand Down Expand Up @@ -981,6 +987,7 @@ func TestJobs_Canonicalize(t *testing.T) {
AllAtOnce: boolToPtr(false),
ConsulToken: stringToPtr(""),
VaultToken: stringToPtr(""),
VaultNamespace: stringToPtr(""),
NomadTokenID: stringToPtr(""),
Stop: boolToPtr(false),
Stable: boolToPtr(false),
Expand Down Expand Up @@ -1144,6 +1151,7 @@ func TestJobs_Canonicalize(t *testing.T) {
AllAtOnce: boolToPtr(false),
ConsulToken: stringToPtr(""),
VaultToken: stringToPtr(""),
VaultNamespace: stringToPtr(""),
NomadTokenID: stringToPtr(""),
Stop: boolToPtr(false),
Stable: boolToPtr(false),
Expand Down
4 changes: 4 additions & 0 deletions api/tasks.go
Original file line number Diff line number Diff line change
Expand Up @@ -812,6 +812,7 @@ func (tmpl *Template) Canonicalize() {

type Vault struct {
Policies []string
Namespace *string `mapstructure:"namespace"`
Env *bool
ChangeMode *string `mapstructure:"change_mode"`
ChangeSignal *string `mapstructure:"change_signal"`
Expand All @@ -821,6 +822,9 @@ func (v *Vault) Canonicalize() {
if v.Env == nil {
v.Env = boolToPtr(true)
}
if v.Namespace == nil {
v.Namespace = stringToPtr("")
}
if v.ChangeMode == nil {
v.ChangeMode = stringToPtr("restart")
}
Expand Down
8 changes: 7 additions & 1 deletion client/allocrunner/taskrunner/task_runner_getters.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,13 @@ func (tr *TaskRunner) setVaultToken(token string) {
tr.vaultToken = token

// Update the task's environment
tr.envBuilder.SetVaultToken(token, tr.clientConfig.VaultConfig.Namespace, tr.task.Vault.Env)
taskNamespace := tr.task.Vault.Namespace

ns := tr.clientConfig.VaultConfig.Namespace
if taskNamespace != "" {
ns = taskNamespace
}
tr.envBuilder.SetVaultToken(token, ns, tr.task.Vault.Env)
}

// getDriverHandle returns a driver handle.
Expand Down
9 changes: 9 additions & 0 deletions client/allocrunner/taskrunner/template/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ type TaskTemplateManagerConfig struct {
// VaultToken is the Vault token for the task.
VaultToken string

// VaultNamespace is the Vault namespace for the task
VaultNamespace string

// TaskDir is the task's directory
TaskDir string

Expand Down Expand Up @@ -655,9 +658,15 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
if cc.VaultConfig != nil && cc.VaultConfig.IsEnabled() {
conf.Vault.Address = &cc.VaultConfig.Addr
conf.Vault.Token = &config.VaultToken

// Set the Vault Namespace. Passed in Task config has
// highest precedence.
if config.ClientConfig.VaultConfig.Namespace != "" {
conf.Vault.Namespace = &config.ClientConfig.VaultConfig.Namespace
}
if config.VaultNamespace != "" {
conf.Vault.Namespace = &config.VaultNamespace
}

if strings.HasPrefix(cc.VaultConfig.Addr, "https") || cc.VaultConfig.TLSCertFile != "" {
skipVerify := cc.VaultConfig.TLSSkipVerify != nil && *cc.VaultConfig.TLSSkipVerify
Expand Down
35 changes: 35 additions & 0 deletions client/allocrunner/taskrunner/template/template_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1377,6 +1377,41 @@ func TestTaskTemplateManager_Config_VaultNamespace(t *testing.T) {
assert.Equal(testNS, *ctconf.Vault.Namespace, "Vault Namespace Value")
}

// TestTaskTemplateManager_Config_VaultNamespace asserts the Vault namespace setting is
// propagated to consul-template's configuration.
func TestTaskTemplateManager_Config_VaultNamespace_TaskOverride(t *testing.T) {
t.Parallel()
assert := assert.New(t)

testNS := "test-namespace"
c := config.DefaultConfig()
c.Node = mock.Node()
c.VaultConfig = &sconfig.VaultConfig{
Enabled: helper.BoolToPtr(true),
Addr: "https://localhost/",
TLSServerName: "notlocalhost",
Namespace: testNS,
}

alloc := mock.Alloc()
overriddenNS := "new-namespace"

// Set the template manager config vault namespace
config := &TaskTemplateManagerConfig{
ClientConfig: c,
VaultToken: "token",
VaultNamespace: overriddenNS,
EnvBuilder: taskenv.NewBuilder(c.Node, alloc, alloc.Job.TaskGroups[0].Tasks[0], c.Region),
}

ctmplMapping, err := parseTemplateConfigs(config)
assert.Nil(err, "Parsing Templates")

ctconf, err := newRunnerConfig(config, ctmplMapping)
assert.Nil(err, "Building Runner Config")
assert.Equal(overriddenNS, *ctconf.Vault.Namespace, "Vault Namespace Value")
}

func TestTaskTemplateManager_BlockedEvents(t *testing.T) {
// The tests sets a template that need keys 0, 1, 2, 3, 4,
// then subsequently sets 0, 1, 2 keys
Expand Down
10 changes: 10 additions & 0 deletions client/allocrunner/taskrunner/template_hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ type templateHook struct {
// vaultToken is the current Vault token
vaultToken string

// vaultNamespace is the current Vault namespace
vaultNamespace string

// taskDir is the task directory
taskDir string
}
Expand Down Expand Up @@ -75,6 +78,12 @@ func (h *templateHook) Prestart(ctx context.Context, req *interfaces.TaskPrestar
// Store the current Vault token and the task directory
h.taskDir = req.TaskDir.Dir
h.vaultToken = req.VaultToken

// Set vault namespace if specified
if req.Task.Vault != nil {
h.vaultNamespace = req.Task.Vault.Namespace
}

unblockCh, err := h.newManager()
if err != nil {
return err
Expand All @@ -98,6 +107,7 @@ func (h *templateHook) newManager() (unblock chan struct{}, err error) {
Templates: h.config.templates,
ClientConfig: h.config.clientConfig,
VaultToken: h.vaultToken,
VaultNamespace: h.vaultNamespace,
TaskDir: h.taskDir,
EnvBuilder: h.config.envBuilder,
MaxTemplateEventRate: template.DefaultMaxTemplateEventRate,
Expand Down
1 change: 1 addition & 0 deletions client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -2457,6 +2457,7 @@ func (c *Client) deriveToken(alloc *structs.Allocation, taskNames []string, vcli
}

// Derive the tokens
// namespace is handled via nomad/vault
var resp structs.DeriveVaultTokenResponse
if err := c.RPC("Node.DeriveVaultToken", &req, &resp); err != nil {
vlogger.Error("error making derive token RPC", "error", err)
Expand Down
40 changes: 24 additions & 16 deletions command/agent/job_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -773,22 +773,23 @@ func ApiJobToStructJob(job *api.Job) *structs.Job {
job.Canonicalize()

j := &structs.Job{
Stop: *job.Stop,
Region: *job.Region,
Namespace: *job.Namespace,
ID: *job.ID,
ParentID: *job.ParentID,
Name: *job.Name,
Type: *job.Type,
Priority: *job.Priority,
AllAtOnce: *job.AllAtOnce,
Datacenters: job.Datacenters,
Payload: job.Payload,
Meta: job.Meta,
ConsulToken: *job.ConsulToken,
VaultToken: *job.VaultToken,
Constraints: ApiConstraintsToStructs(job.Constraints),
Affinities: ApiAffinitiesToStructs(job.Affinities),
Stop: *job.Stop,
Region: *job.Region,
Namespace: *job.Namespace,
ID: *job.ID,
ParentID: *job.ParentID,
Name: *job.Name,
Type: *job.Type,
Priority: *job.Priority,
AllAtOnce: *job.AllAtOnce,
Datacenters: job.Datacenters,
Payload: job.Payload,
Meta: job.Meta,
ConsulToken: *job.ConsulToken,
VaultToken: *job.VaultToken,
VaultNamespace: *job.VaultNamespace,
Constraints: ApiConstraintsToStructs(job.Constraints),
Affinities: ApiAffinitiesToStructs(job.Affinities),
}

// Update has been pushed into the task groups. stagger and max_parallel are
Expand Down Expand Up @@ -976,6 +977,12 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta
for l, task := range taskGroup.Tasks {
t := &structs.Task{}
ApiTaskToStructsTask(task, t)

// Set the tasks vault namespace from Job if it was not
// specified by the task or group
if t.Vault != nil && t.Vault.Namespace == "" && job.VaultNamespace != "" {
t.Vault.Namespace = job.VaultNamespace
}
tg.Tasks[l] = t
}
}
Expand Down Expand Up @@ -1089,6 +1096,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) {
if apiTask.Vault != nil {
structsTask.Vault = &structs.Vault{
Policies: apiTask.Vault.Policies,
Namespace: *apiTask.Vault.Namespace,
Env: *apiTask.Vault.Env,
ChangeMode: *apiTask.Vault.ChangeMode,
ChangeSignal: *apiTask.Vault.ChangeSignal,
Expand Down
24 changes: 14 additions & 10 deletions command/agent/job_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2121,6 +2121,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
},
},
Vault: &api.Vault{
Namespace: helper.StringToPtr("ns1"),
Policies: []string{"a", "b", "c"},
Env: helper.BoolToPtr(true),
ChangeMode: helper.StringToPtr("c"),
Expand Down Expand Up @@ -2149,6 +2150,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
},
ConsulToken: helper.StringToPtr("abc123"),
VaultToken: helper.StringToPtr("def456"),
VaultNamespace: helper.StringToPtr("ghi789"),
Status: helper.StringToPtr("status"),
StatusDescription: helper.StringToPtr("status_desc"),
Version: helper.Uint64ToPtr(10),
Expand All @@ -2158,16 +2160,17 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
}

expected := &structs.Job{
Stop: true,
Region: "global",
Namespace: "foo",
ID: "foo",
ParentID: "lol",
Name: "name",
Type: "service",
Priority: 50,
AllAtOnce: true,
Datacenters: []string{"dc1", "dc2"},
Stop: true,
Region: "global",
Namespace: "foo",
VaultNamespace: "ghi789",
ID: "foo",
ParentID: "lol",
Name: "name",
Type: "service",
Priority: 50,
AllAtOnce: true,
Datacenters: []string{"dc1", "dc2"},
Constraints: []*structs.Constraint{
{
LTarget: "a",
Expand Down Expand Up @@ -2488,6 +2491,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
},
},
Vault: &structs.Vault{
Namespace: "ns1",
Policies: []string{"a", "b", "c"},
Env: true,
ChangeMode: "c",
Expand Down
18 changes: 17 additions & 1 deletion command/job_run.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ Run Options:
the job file. This overrides the token found in $VAULT_TOKEN environment
variable and that found in the job.
-vault-namespace
If set, the passed Vault namespace is stored in the job before sending to the
Nomad servers. This overrides the namespace found in $VAULT_NAMESPACE environment
variable and that found in the job.
-verbose
Display full information.
`
Expand All @@ -119,6 +124,7 @@ func (c *JobRunCommand) AutocompleteFlags() complete.Flags {
"-verbose": complete.PredictNothing,
"-consul-token": complete.PredictNothing,
"-vault-token": complete.PredictAnything,
"-vault-namespace": complete.PredictAnything,
"-output": complete.PredictNothing,
"-policy-override": complete.PredictNothing,
"-preserve-counts": complete.PredictNothing,
Expand All @@ -133,7 +139,7 @@ func (c *JobRunCommand) Name() string { return "job run" }

func (c *JobRunCommand) Run(args []string) int {
var detach, verbose, output, override, preserveCounts bool
var checkIndexStr, consulToken, vaultToken string
var checkIndexStr, consulToken, vaultToken, vaultNamespace string

flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
flags.Usage = func() { c.Ui.Output(c.Help()) }
Expand All @@ -145,6 +151,7 @@ func (c *JobRunCommand) Run(args []string) int {
flags.StringVar(&checkIndexStr, "check-index", "", "")
flags.StringVar(&consulToken, "consul-token", "", "")
flags.StringVar(&vaultToken, "vault-token", "", "")
flags.StringVar(&vaultNamespace, "vault-namespace", "", "")

if err := flags.Parse(args); err != nil {
return 1
Expand Down Expand Up @@ -213,6 +220,15 @@ func (c *JobRunCommand) Run(args []string) int {
job.VaultToken = helper.StringToPtr(vaultToken)
}

// Parse the Vault namespace
if vaultNamespace == "" {
vaultNamespace = os.Getenv("VAULT_NAMESPACE")
}

if vaultNamespace != "" {
job.VaultNamespace = helper.StringToPtr(vaultNamespace)
}

if output {
req := struct {
Job *api.Job
Expand Down
Loading

0 comments on commit 1981036

Please sign in to comment.