-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
consul/connect: add support for running connect native tasks
This PR adds the capability of running Connect Native Tasks on Nomad, particularly when TLS and ACLs are enabled on Consul. The `connect` stanza now includes a `native` parameter, which can be set to the name of task that backs the Connect Native Consul service. There is a new Client configuration parameter for the `consul` stanza called `share_ssl`. Like `allow_unauthenticated` the default value is true, but recommended to be disabled in production environments. When enabled, the Nomad Client's Consul TLS information is shared with Connect Native tasks through the normal Consul environment variables. This does NOT include auth or token information. If Consul ACLs are enabled, Service Identity Tokens are automatically and injected into the Connect Native task through the CONSUL_HTTP_TOKEN environment variable. Any of the automatically set environment variables can be overridden by the Connect Native task using the `env` stanza. Fixes #6083
- Loading branch information
Showing
21 changed files
with
894 additions
and
90 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,218 @@ | ||
package taskrunner | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"io" | ||
"io/ioutil" | ||
"os" | ||
"path/filepath" | ||
|
||
hclog "github.com/hashicorp/go-hclog" | ||
ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces" | ||
"github.com/hashicorp/nomad/nomad/structs" | ||
"github.com/hashicorp/nomad/nomad/structs/config" | ||
"github.com/pkg/errors" | ||
) | ||
|
||
const ( | ||
connectNativeHookName = "connect_native" | ||
) | ||
|
||
type connectNativeHookConfig struct { | ||
consulShareTLS bool | ||
consul consulTransportConfig | ||
alloc *structs.Allocation | ||
logger hclog.Logger | ||
} | ||
|
||
func newConnectNativeHookConfig(alloc *structs.Allocation, consul *config.ConsulConfig, logger hclog.Logger) *connectNativeHookConfig { | ||
return &connectNativeHookConfig{ | ||
alloc: alloc, | ||
logger: logger, | ||
consulShareTLS: consul.ShareSSL == nil || *consul.ShareSSL, // default enabled | ||
consul: newConsulTransportConfig(consul), | ||
} | ||
} | ||
|
||
// connectNativeHook manages additional automagic configuration for a connect | ||
// native task. | ||
// | ||
// If nomad client is configured to talk to Consul using TLS (or other special | ||
// auth), the native task will inherit that configuration EXCEPT for the consul | ||
// token. | ||
// | ||
// If consul is configured with ACLs enabled, a Service Identity token will be | ||
// generated on behalf of the native service and supplied to the task. | ||
type connectNativeHook struct { | ||
// alloc is the allocation with the connect native task being run | ||
alloc *structs.Allocation | ||
|
||
// consulShareTLS is used to toggle whether the TLS configuration of the | ||
// Nomad Client may be shared with Connect Native applications. | ||
consulShareTLS bool | ||
|
||
// consulConfig is used to enable the connect native enabled task to | ||
// communicate with consul directly, as is necessary for the task to request | ||
// its connect mTLS certificates. | ||
consulConfig consulTransportConfig | ||
|
||
// logger is used to log things | ||
logger hclog.Logger | ||
} | ||
|
||
func newConnectNativeHook(c *connectNativeHookConfig) *connectNativeHook { | ||
return &connectNativeHook{ | ||
alloc: c.alloc, | ||
consulShareTLS: c.consulShareTLS, | ||
consulConfig: c.consul, | ||
logger: c.logger.Named(connectNativeHookName), | ||
} | ||
} | ||
|
||
func (connectNativeHook) Name() string { | ||
return connectNativeHookName | ||
} | ||
|
||
func (h *connectNativeHook) Prestart( | ||
ctx context.Context, | ||
request *ifs.TaskPrestartRequest, | ||
response *ifs.TaskPrestartResponse) error { | ||
|
||
if !request.Task.Kind.IsConnectNative() { | ||
response.Done = true | ||
return nil | ||
} | ||
|
||
h.logger.Debug("share consul TLS configuration for connect native", "enabled", h.consulShareTLS, "task", request.Task.Name) | ||
if h.consulShareTLS { | ||
// copy TLS certificates | ||
if err := h.copyCertificates(h.consulConfig, request.TaskDir.SecretsDir); err != nil { | ||
h.logger.Error("failed to copy Consul TLS certificates", "error", err) | ||
return err | ||
} | ||
|
||
// set environment variables for communicating with Consul agent, but | ||
// only if those environment variables are not already set | ||
response.Env = h.tlsEnv(request.TaskEnv.EnvMap) | ||
|
||
} | ||
|
||
if err := h.maybeSetSITokenEnv(request.TaskDir.SecretsDir, request.Task.Name, response.Env); err != nil { | ||
h.logger.Error("failed to load Consul Service Identity Token", "error", err, "task", request.Task.Name) | ||
return err | ||
} | ||
|
||
// tls/acl setup for native task done | ||
response.Done = true | ||
return nil | ||
} | ||
|
||
const ( | ||
secretCAFilename = "consul_ca_file" | ||
secretCertfileFilename = "consul_cert_file" | ||
secretKeyfileFilename = "consul_key_file" | ||
) | ||
|
||
func (h *connectNativeHook) copyCertificates(consulConfig consulTransportConfig, dir string) error { | ||
if err := h.copyCertificate(consulConfig.CAFile, dir, secretCAFilename); err != nil { | ||
return err | ||
} | ||
if err := h.copyCertificate(consulConfig.CertFile, dir, secretCertfileFilename); err != nil { | ||
return err | ||
} | ||
if err := h.copyCertificate(consulConfig.KeyFile, dir, secretKeyfileFilename); err != nil { | ||
return err | ||
} | ||
return nil | ||
} | ||
|
||
func (connectNativeHook) copyCertificate(source, dir, name string) error { | ||
if source == "" { | ||
return nil | ||
} | ||
|
||
original, err := os.Open(source) | ||
if err != nil { | ||
return errors.Wrap(err, "failed to open consul TLS certificate") | ||
} | ||
defer original.Close() | ||
|
||
destination := filepath.Join(dir, name) | ||
fd, err := os.Create(destination) | ||
if err != nil { | ||
return errors.Wrapf(err, "failed to create secrets/%s", name) | ||
} | ||
defer fd.Close() | ||
|
||
if _, err := io.Copy(fd, original); err != nil { | ||
return errors.Wrapf(err, "failed to copy certificate secrets/%s", name) | ||
} | ||
|
||
if err := fd.Sync(); err != nil { | ||
return errors.Wrapf(err, "failed to write secrets/%s", name) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
// tlsEnv creates a set of additional of environment variables to be used when launching | ||
// the connect native task. This will enable the task to communicate with Consul | ||
// if Consul has transport security turned on. | ||
// | ||
// We do NOT set CONSUL_HTTP_TOKEN from the nomad agent's consul config, as that | ||
// is a separate security concern addressed by the service identity hook. | ||
func (h *connectNativeHook) tlsEnv(env map[string]string) map[string]string { | ||
m := make(map[string]string) | ||
|
||
if _, exists := env["CONSUL_CACERT"]; !exists && h.consulConfig.CAFile != "" { | ||
m["CONSUL_CACERT"] = filepath.Join("/secrets", secretCAFilename) | ||
} | ||
|
||
if _, exists := env["CONSUL_CLIENT_CERT"]; !exists && h.consulConfig.CertFile != "" { | ||
m["CONSUL_CLIENT_CERT"] = filepath.Join("/secrets", secretCertfileFilename) | ||
} | ||
|
||
if _, exists := env["CONSUL_CLIENT_KEY"]; !exists && h.consulConfig.KeyFile != "" { | ||
m["CONSUL_CLIENT_KEY"] = filepath.Join("/secrets", secretKeyfileFilename) | ||
} | ||
|
||
if _, exists := env["CONSUL_HTTP_SSL"]; !exists { | ||
if v := h.consulConfig.SSL; v != "" { | ||
m["CONSUL_HTTP_SSL"] = v | ||
} | ||
} | ||
|
||
if _, exists := env["CONSUL_HTTP_SSL_VERIFY"]; !exists { | ||
if v := h.consulConfig.VerifySSL; v != "" { | ||
m["CONSUL_HTTP_SSL_VERIFY"] = v | ||
} | ||
} | ||
|
||
return m | ||
} | ||
|
||
// maybeSetSITokenEnv will set the CONSUL_HTTP_TOKEN environment variable in | ||
// the given env map, if the token is found to exist in the task's secrets | ||
// directory. | ||
// | ||
// Following the pattern of the envoy_bootstrap_hook, the Consul Service Identity | ||
// ACL Token is generated prior to this hook, if Consul ACLs are enabled. This is | ||
// done in the sids_hook, which places the token at secrets/si_token in the task | ||
// workspace. The content of that file is the SI token specific to this task | ||
// instance. | ||
func (h *connectNativeHook) maybeSetSITokenEnv(dir, task string, env map[string]string) error { | ||
fmt.Printf("maybeSetSITokenEnv, dir: %s, task: %s, env: %v\n", dir, task, env) | ||
|
||
token, err := ioutil.ReadFile(filepath.Join(dir, sidsTokenFile)) | ||
if err != nil { | ||
if !os.IsNotExist(err) { | ||
return errors.Wrapf(err, "failed to load SI token for native task %s", task) | ||
} | ||
h.logger.Trace("no SI token to load for native task", "task", task) | ||
return nil // token file DNE; acls not enabled | ||
} | ||
h.logger.Trace("recovered pre-existing SI token for native task", "task", task) | ||
env["CONSUL_HTTP_TOKEN"] = string(token) | ||
return nil | ||
} |
Oops, something went wrong.