diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b6589fc12e2..c53bbb18d274 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,10 +58,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x @@ -176,10 +176,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x @@ -294,10 +294,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x @@ -412,10 +412,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x @@ -635,10 +635,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x @@ -846,10 +846,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x @@ -938,10 +938,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x @@ -1118,10 +1118,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x @@ -1298,10 +1298,10 @@ jobs: name: install protoc - run: command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.4/consul_1.6.4_linux_amd64.zip + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip sudo unzip -d /usr/local/bin /tmp/consul.zip rm -rf /tmp/consul* - name: Install Consul 1.6.4 + name: Install Consul 1.8.3 - run: command: | set -x diff --git a/.circleci/config/commands/install-consul.yml b/.circleci/config/commands/install-consul.yml index cad97da0cea5..c3eaf3b49684 100644 --- a/.circleci/config/commands/install-consul.yml +++ b/.circleci/config/commands/install-consul.yml @@ -1,7 +1,7 @@ parameters: version: type: string - default: 1.6.4 + default: 1.8.3 steps: - run: name: Install Consul << parameters.version >> diff --git a/api/services.go b/api/services.go index ce6bcd9928fd..9640119d64bb 100644 --- a/api/services.go +++ b/api/services.go @@ -152,6 +152,7 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) { // ConsulConnect represents a Consul Connect jobspec stanza. type ConsulConnect struct { Native bool + Gateway *ConsulGateway SidecarService *ConsulSidecarService `mapstructure:"sidecar_service"` SidecarTask *SidecarTask `mapstructure:"sidecar_task"` } @@ -163,6 +164,7 @@ func (cc *ConsulConnect) Canonicalize() { cc.SidecarService.Canonicalize() cc.SidecarTask.Canonicalize() + cc.Gateway.Canonicalize() } // ConsulSidecarService represents a Consul Connect SidecarService jobspec @@ -290,3 +292,263 @@ type ConsulExposePath struct { LocalPathPort int `mapstructure:"local_path_port"` ListenerPort string `mapstructure:"listener_port"` } + +// ConsulGateway is used to configure one of the Consul Connect Gateway types. +type ConsulGateway struct { + // Proxy is used to configure the Envoy instance acting as the gateway. + Proxy *ConsulGatewayProxy + + // Ingress represents the Consul Configuration Entry for an Ingress Gateway. + Ingress *ConsulIngressConfigEntry + + // Terminating is not yet supported. + // Terminating *ConsulTerminatingConfigEntry + + // Mesh is not yet supported. + // Mesh *ConsulMeshConfigEntry +} + +func (g *ConsulGateway) Canonicalize() { + if g == nil { + return + } + g.Proxy.Canonicalize() + g.Ingress.Canonicalize() +} + +func (g *ConsulGateway) Copy() *ConsulGateway { + if g == nil { + return nil + } + + return &ConsulGateway{ + Proxy: g.Proxy.Copy(), + Ingress: g.Ingress.Copy(), + } +} + +type ConsulGatewayBindAddress struct { + Address string `mapstructure:"address"` + Port int `mapstructure:"port"` +} + +var ( + defaultGatewayConnectTimeout = 5 * time.Second +) + +// ConsulGatewayProxy is used to tune parameters of the proxy instance acting as +// one of the forms of Connect gateways that Consul supports. +// +// https://www.consul.io/docs/connect/proxies/envoy#gateway-options +type ConsulGatewayProxy struct { + ConnectTimeout *time.Duration `mapstructure:"connect_timeout"` + EnvoyGatewayBindTaggedAddresses bool `mapstructure:"envoy_gateway_bind_tagged_addresses"` + EnvoyGatewayBindAddresses map[string]*ConsulGatewayBindAddress `mapstructure:"envoy_gateway_bind_addresses"` + EnvoyGatewayNoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind"` + Config map[string]interface{} // escape hatch envoy config +} + +func (p *ConsulGatewayProxy) Canonicalize() { + if p == nil { + return + } + + if p.ConnectTimeout == nil { + // same as the default from consul + p.ConnectTimeout = timeToPtr(defaultGatewayConnectTimeout) + } + + if len(p.EnvoyGatewayBindAddresses) == 0 { + p.EnvoyGatewayBindAddresses = nil + } + + if len(p.Config) == 0 { + p.Config = nil + } +} + +func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy { + if p == nil { + return nil + } + + var binds map[string]*ConsulGatewayBindAddress = nil + if p.EnvoyGatewayBindAddresses != nil { + binds = make(map[string]*ConsulGatewayBindAddress, len(p.EnvoyGatewayBindAddresses)) + for k, v := range p.EnvoyGatewayBindAddresses { + binds[k] = v + } + } + + var config map[string]interface{} = nil + if p.Config != nil { + config = make(map[string]interface{}, len(p.Config)) + for k, v := range p.Config { + config[k] = v + } + } + + return &ConsulGatewayProxy{ + ConnectTimeout: timeToPtr(*p.ConnectTimeout), + EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses, + EnvoyGatewayBindAddresses: binds, + EnvoyGatewayNoDefaultBind: p.EnvoyGatewayNoDefaultBind, + Config: config, + } +} + +// ConsulGatewayTLSConfig is used to configure TLS for a gateway. +type ConsulGatewayTLSConfig struct { + Enabled bool +} + +func (tc *ConsulGatewayTLSConfig) Canonicalize() { +} + +func (tc *ConsulGatewayTLSConfig) Copy() *ConsulGatewayTLSConfig { + if tc == nil { + return nil + } + + return &ConsulGatewayTLSConfig{ + Enabled: tc.Enabled, + } +} + +// ConsulIngressService is used to configure a service fronted by the ingress gateway. +type ConsulIngressService struct { + // Namespace is not yet supported. + // Namespace string + Name string + + Hosts []string +} + +func (s *ConsulIngressService) Canonicalize() { + if s == nil { + return + } + + if len(s.Hosts) == 0 { + s.Hosts = nil + } +} + +func (s *ConsulIngressService) Copy() *ConsulIngressService { + if s == nil { + return nil + } + + var hosts []string = nil + if n := len(s.Hosts); n > 0 { + hosts = make([]string, n) + copy(hosts, s.Hosts) + } + + return &ConsulIngressService{ + Name: s.Name, + Hosts: hosts, + } +} + +const ( + defaultIngressListenerProtocol = "tcp" +) + +// ConsulIngressListener is used to configure a listener on a Consul Ingress +// Gateway. +type ConsulIngressListener struct { + Port int + Protocol string + Services []*ConsulIngressService +} + +func (l *ConsulIngressListener) Canonicalize() { + if l == nil { + return + } + + if l.Protocol == "" { + // same as default from consul + l.Protocol = defaultIngressListenerProtocol + } + + if len(l.Services) == 0 { + l.Services = nil + } +} + +func (l *ConsulIngressListener) Copy() *ConsulIngressListener { + if l == nil { + return nil + } + + var services []*ConsulIngressService = nil + if n := len(l.Services); n > 0 { + services = make([]*ConsulIngressService, n) + for i := 0; i < n; i++ { + services[i] = l.Services[i].Copy() + } + } + + return &ConsulIngressListener{ + Port: l.Port, + Protocol: l.Protocol, + Services: services, + } +} + +// ConsulIngressConfigEntry represents the Consul Configuration Entry type for +// an Ingress Gateway. +// +// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields +type ConsulIngressConfigEntry struct { + // Namespace is not yet supported. + // Namespace string + + TLS *ConsulGatewayTLSConfig + Listeners []*ConsulIngressListener +} + +func (e *ConsulIngressConfigEntry) Canonicalize() { + if e == nil { + return + } + + e.TLS.Canonicalize() + + if len(e.Listeners) == 0 { + e.Listeners = nil + } + + for _, listener := range e.Listeners { + listener.Canonicalize() + } +} + +func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry { + if e == nil { + return nil + } + + var listeners []*ConsulIngressListener = nil + if n := len(e.Listeners); n > 0 { + listeners = make([]*ConsulIngressListener, n) + for i := 0; i < n; i++ { + listeners[i] = e.Listeners[i].Copy() + } + } + + return &ConsulIngressConfigEntry{ + TLS: e.TLS.Copy(), + Listeners: listeners, + } +} + +// ConsulTerminatingConfigEntry is not yet supported. +// type ConsulTerminatingConfigEntry struct { +// } + +// ConsulMeshConfigEntry is not yet supported. +// type ConsulMeshConfigEntry struct { +// } diff --git a/api/services_test.go b/api/services_test.go index f1c9d0556228..2e5336855779 100644 --- a/api/services_test.go +++ b/api/services_test.go @@ -261,3 +261,158 @@ func TestService_Connect_SidecarTask_Canonicalize(t *testing.T) { require.Equal(t, exp, st.Resources) }) } + +func TestService_ConsulGateway_Canonicalize(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + cg := (*ConsulGateway)(nil) + cg.Canonicalize() + require.Nil(t, cg) + }) + + t.Run("set defaults", func(t *testing.T) { + cg := &ConsulGateway{ + Proxy: &ConsulGatewayProxy{ + ConnectTimeout: nil, + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: make(map[string]*ConsulGatewayBindAddress, 0), + EnvoyGatewayNoDefaultBind: true, + Config: make(map[string]interface{}, 0), + }, + Ingress: &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: false, + }, + Listeners: make([]*ConsulIngressListener, 0), + }, + } + cg.Canonicalize() + require.Equal(t, timeToPtr(5*time.Second), cg.Proxy.ConnectTimeout) + require.Nil(t, cg.Proxy.EnvoyGatewayBindAddresses) + require.Nil(t, cg.Proxy.Config) + require.Nil(t, cg.Ingress.Listeners) + }) +} + +func TestService_ConsulGateway_Copy(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + result := (*ConsulGateway)(nil).Copy() + require.Nil(t, result) + }) + + gateway := &ConsulGateway{ + Proxy: &ConsulGatewayProxy{ + ConnectTimeout: timeToPtr(3 * time.Second), + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{ + "listener1": {Address: "10.0.0.1", Port: 2000}, + "listener2": {Address: "10.0.0.1", Port: 2001}, + }, + EnvoyGatewayNoDefaultBind: true, + Config: map[string]interface{}{ + "foo": "bar", + "baz": 3, + }, + }, + Ingress: &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*ConsulIngressListener{{ + Port: 3333, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{ + "127.0.0.1", "127.0.0.1:3333", + }}, + }}, + }, + }, + } + + t.Run("complete", func(t *testing.T) { + result := gateway.Copy() + require.Equal(t, gateway, result) + }) +} + +func TestService_ConsulIngressConfigEntry_Canonicalize(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + c := (*ConsulIngressConfigEntry)(nil) + c.Canonicalize() + require.Nil(t, c) + }) + + t.Run("empty fields", func(t *testing.T) { + c := &ConsulIngressConfigEntry{ + TLS: nil, + Listeners: []*ConsulIngressListener{}, + } + c.Canonicalize() + require.Nil(t, c.TLS) + require.Nil(t, c.Listeners) + }) + + t.Run("complete", func(t *testing.T) { + c := &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{Enabled: true}, + Listeners: []*ConsulIngressListener{{ + Port: 9090, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"1.1.1.1"}, + }}, + }}, + } + c.Canonicalize() + require.Equal(t, &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{Enabled: true}, + Listeners: []*ConsulIngressListener{{ + Port: 9090, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"1.1.1.1"}, + }}, + }}, + }, c) + }) +} + +func TestService_ConsulIngressConfigEntry_Copy(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + result := (*ConsulIngressConfigEntry)(nil).Copy() + require.Nil(t, result) + }) + + entry := &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*ConsulIngressListener{{ + Port: 1111, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"1.1.1.1", "1.1.1.1:9000"}, + }, { + Name: "service2", + Hosts: []string{"2.2.2.2"}, + }}, + }}, + } + + t.Run("complete", func(t *testing.T) { + result := entry.Copy() + require.Equal(t, entry, result) + }) +} diff --git a/client/allocrunner/consul_grpc_sock_hook.go b/client/allocrunner/consul_grpc_sock_hook.go index 1c406f4aa6c3..91014fedc0e9 100644 --- a/client/allocrunner/consul_grpc_sock_hook.go +++ b/client/allocrunner/consul_grpc_sock_hook.go @@ -68,7 +68,7 @@ func (h *consulGRPCSocketHook) shouldRun() bool { } for _, s := range tg.Services { - if s.Connect.HasSidecar() { + if s.Connect.HasSidecar() || s.Connect.IsGateway() { return true } } diff --git a/client/allocrunner/taskrunner/envoybootstrap_hook.go b/client/allocrunner/taskrunner/envoybootstrap_hook.go index 297d3daa8071..ea3010638d02 100644 --- a/client/allocrunner/taskrunner/envoybootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoybootstrap_hook.go @@ -75,6 +75,11 @@ const ( envoyAdminBindEnvPrefix = "NOMAD_ENVOY_ADMIN_ADDR_" ) +const ( + grpcConsulVariable = "CONSUL_GRPC_ADDR" + grpcDefaultAddress = "127.0.0.1:8502" +) + // envoyBootstrapHook writes the bootstrap config for the Connect Envoy proxy // sidecar. type envoyBootstrapHook struct { @@ -103,41 +108,73 @@ func (envoyBootstrapHook) Name() string { return envoyBootstrapHookName } -func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error { - if !req.Task.Kind.IsConnectProxy() { - // Not a Connect proxy sidecar - resp.Done = true - return nil +func (_ *envoyBootstrapHook) extractNameAndKind(kind structs.TaskKind) (string, string, error) { + serviceKind := kind.Name() + serviceName := kind.Value() + + switch serviceKind { + case structs.ConnectProxyPrefix, structs.ConnectIngressPrefix: + default: + return "", "", errors.New("envoy must be used as connect sidecar or gateway") } - serviceName := req.Task.Kind.Value() if serviceName == "" { - return errors.New("connect proxy sidecar does not specify service name") + return "", "", errors.New("envoy must be configured with a service name") } + return serviceKind, serviceName, nil +} + +func (h *envoyBootstrapHook) lookupService(svcKind, svcName, tgName string) (*structs.Service, error) { tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup) var service *structs.Service for _, s := range tg.Services { - if s.Name == serviceName { + if s.Name == svcName { service = s break } } if service == nil { - return errors.New("connect proxy sidecar task exists but no services configured with a sidecar") + if svcKind == structs.ConnectProxyPrefix { + return nil, errors.New("connect proxy sidecar task exists but no services configured with a sidecar") + } else { + return nil, errors.New("connect gateway task exists but no service associated") + } } - h.logger.Debug("bootstrapping Connect proxy sidecar", "task", req.Task.Name, "service", serviceName) + return service, nil +} - //TODO Should connect directly to Consul if the sidecar is running on the host netns. - grpcAddr := "unix://" + allocdir.AllocGRPCSocket +// Prestart creates an envoy bootstrap config file. +// +// Must be aware of both launching envoy as a sidecar proxy, as well as a connect gateway. +func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error { + if !req.Task.Kind.IsConnectProxy() && !req.Task.Kind.IsAnyConnectGateway() { + // Not a Connect proxy sidecar + resp.Done = true + return nil + } + + serviceKind, serviceName, err := h.extractNameAndKind(req.Task.Kind) + if err != nil { + return err + } + + service, err := h.lookupService(serviceKind, serviceName, h.alloc.TaskGroup) + if err != nil { + return err + } + + grpcAddr := h.grpcAddress(req.TaskEnv.EnvMap) - // Envoy runs an administrative API on the loopback interface. If multiple sidecars - // are running, the bind addresses need to have unique ports. - // TODO: support running in host netns, using freeport to find available port - envoyAdminBind := buildEnvoyAdminBind(h.alloc, req.Task.Name) + h.logger.Debug("bootstrapping Consul "+serviceKind, "task", req.Task.Name, "service", serviceName) + + // Envoy runs an administrative API on the loopback interface. There is no + // way to turn this feature off. + // https://github.com/envoyproxy/envoy/issues/1297 + envoyAdminBind := buildEnvoyAdminBind(h.alloc, serviceName, req.Task.Name) resp.Env = map[string]string{ helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_'): envoyAdminBind, } @@ -146,10 +183,6 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP // it to the secrets directory like Vault tokens. bootstrapFilePath := filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json") - id := agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+tg.Name, service) - - h.logger.Debug("bootstrapping envoy", "sidecar_for", service.Name, "bootstrap_file", bootstrapFilePath, "sidecar_for_id", id, "grpc_addr", grpcAddr, "admin_bind", envoyAdminBind) - siToken, err := h.maybeLoadSIToken(req.Task.Name, req.TaskDir.SecretsDir) if err != nil { h.logger.Error("failed to generate envoy bootstrap config", "sidecar_for", service.Name) @@ -157,16 +190,9 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP } h.logger.Debug("check for SI token for task", "task", req.Task.Name, "exists", siToken != "") - bootstrapBuilder := envoyBootstrapArgs{ - consulConfig: h.consulConfig, - sidecarFor: id, - grpcAddr: grpcAddr, - envoyAdminBind: envoyAdminBind, - siToken: siToken, - } - - bootstrapArgs := bootstrapBuilder.args() - bootstrapEnv := bootstrapBuilder.env(os.Environ()) + bootstrap := h.newEnvoyBootstrapArgs(h.alloc.TaskGroup, service, grpcAddr, envoyAdminBind, siToken, bootstrapFilePath) + bootstrapArgs := bootstrap.args() + bootstrapEnv := bootstrap.env(os.Environ()) // Since Consul services are registered asynchronously with this task // hook running, retry a small number of times with backoff. @@ -231,12 +257,32 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP return nil } -func buildEnvoyAdminBind(alloc *structs.Allocation, taskName string) string { +// buildEnvoyAdminBind determines a unique port for use by the envoy admin +// listener. +// +// In bridge mode, if multiple sidecars are running, the bind addresses need +// to be unique within the namespace, so we simply start at 19000 and increment +// by the index of the task. +// +// In host mode, use the port provided through the service definition, which can +// be a port chosen by Nomad. +func buildEnvoyAdminBind(alloc *structs.Allocation, serviceName, taskName string) string { + tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) port := envoyBaseAdminPort - for idx, task := range alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks { - if task.Name == taskName { - port += idx - break + switch tg.Networks[0].Mode { + case "host": + for _, service := range tg.Services { + if service.Name == serviceName { + _, port = tg.Networks.Port(service.PortLabel) + break + } + } + default: + for idx, task := range tg.Tasks { + if task.Name == taskName { + port += idx + break + } } } return fmt.Sprintf("localhost:%d", port) @@ -269,15 +315,69 @@ func (h *envoyBootstrapHook) execute(cmd *exec.Cmd) (string, error) { return stdout.String(), nil } +// grpcAddress determines the Consul gRPC endpoint address to use. +// +// In host networking this will default to 127.0.0.1:8502. +// In bridge/cni networking this will default to unix://. +// In either case, CONSUL_GRPC_ADDR will override the default. +func (h *envoyBootstrapHook) grpcAddress(env map[string]string) string { + if address := env[grpcConsulVariable]; address != "" { + return address + } + + tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup) + switch tg.Networks[0].Mode { + case "host": + return grpcDefaultAddress + default: + return "unix://" + allocdir.AllocGRPCSocket + } +} + +func (h *envoyBootstrapHook) newEnvoyBootstrapArgs( + tgName string, + service *structs.Service, + grpcAddr, envoyAdminBind, siToken, filepath string, +) envoyBootstrapArgs { + var ( + sidecarForID string // sidecar only + gateway string // gateway only + ) + + if service.Connect.HasSidecar() { + sidecarForID = agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+tgName, service) + } + + if service.Connect.IsGateway() { + gateway = "ingress" // more types in the future + } + + h.logger.Debug("bootstrapping envoy", + "sidecar_for", service.Name, "bootstrap_file", filepath, + "sidecar_for_id", sidecarForID, "grpc_addr", grpcAddr, + "admin_bind", envoyAdminBind, "gateway", gateway, + ) + + return envoyBootstrapArgs{ + consulConfig: h.consulConfig, + sidecarFor: sidecarForID, + grpcAddr: grpcAddr, + envoyAdminBind: envoyAdminBind, + siToken: siToken, + gateway: gateway, + } +} + // envoyBootstrapArgs is used to accumulate CLI arguments that will be passed // along to the exec invocation of consul which will then generate the bootstrap // configuration file for envoy. type envoyBootstrapArgs struct { consulConfig consulTransportConfig - sidecarFor string + sidecarFor string // sidecars only grpcAddr string envoyAdminBind string siToken string + gateway string // gateways only } // args returns the CLI arguments consul needs in the correct order, with the @@ -290,7 +390,14 @@ func (e envoyBootstrapArgs) args() []string { "-http-addr", e.consulConfig.HTTPAddr, "-admin-bind", e.envoyAdminBind, "-bootstrap", - "-sidecar-for", e.sidecarFor, + } + + if v := e.sidecarFor; v != "" { + arguments = append(arguments, "-sidecar-for", e.sidecarFor) + } + + if v := e.gateway; v != "" { + arguments = append(arguments, "-gateway", e.gateway) } if v := e.siToken; v != "" { diff --git a/client/allocrunner/taskrunner/envoybootstrap_hook_test.go b/client/allocrunner/taskrunner/envoybootstrap_hook_test.go index de3fa2ec861d..66b308d3cf3f 100644 --- a/client/allocrunner/taskrunner/envoybootstrap_hook_test.go +++ b/client/allocrunner/taskrunner/envoybootstrap_hook_test.go @@ -166,6 +166,23 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) { "-client-key", "/etc/tls/key-file", }, result) }) + + t.Run("ingress gateway", func(t *testing.T) { + ebArgs := envoyBootstrapArgs{ + consulConfig: consulPlainConfig, + grpcAddr: "1.1.1.1", + envoyAdminBind: "localhost:3333", + gateway: "my-ingress-gateway", + } + result := ebArgs.args() + require.Equal(t, []string{"connect", "envoy", + "-grpc-addr", "1.1.1.1", + "-http-addr", "2.2.2.2", + "-admin-bind", "localhost:3333", + "-bootstrap", + "-gateway", "my-ingress-gateway", + }, result) + }) } func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) { @@ -199,8 +216,24 @@ func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) { }) } -// dig through envoy config to look for consul token +// envoyConfig is used to unmarshal an envoy bootstrap configuration file, so that +// we can inspect the contents in tests. type envoyConfig struct { + Admin struct { + Address struct { + SocketAddress struct { + Address string `json:"address"` + Port int `json:"port_value"` + } `json:"socket_address"` + } `json:"address"` + } `json:"admin"` + Node struct { + Cluster string `json:"cluster"` + Metadata struct { + Namespace string `json:"namespace"` + Version string `json:"envoy_version"` + } + } DynamicResources struct { ADSConfig struct { GRPCServices struct { @@ -219,10 +252,10 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) { t.Parallel() testutil.RequireConsul(t) - testconsul := getTestConsul(t) - defer testconsul.Stop() + testConsul := getTestConsul(t) + defer testConsul.Stop() - alloc := mock.Alloc() + alloc := mock.ConnectAlloc() alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{ { Mode: "bridge", @@ -259,7 +292,7 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) { // Register Group Services consulConfig := consulapi.DefaultConfig() - consulConfig.Address = testconsul.HTTPAddr + consulConfig.Address = testConsul.HTTPAddr consulAPIClient, err := consulapi.NewClient(consulConfig) require.NoError(t, err) @@ -275,6 +308,7 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) { req := &interfaces.TaskPrestartRequest{ Task: sidecarTask, TaskDir: allocDir.NewTaskDir(sidecarTask.Name), + TaskEnv: taskenv.NewEmptyTaskEnv(), } require.NoError(t, req.TaskDir.Build(false, nil)) @@ -311,17 +345,17 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) { require.Equal(t, token, value) } -// TestTaskRunner_EnvoyBootstrapHook_Prestart asserts the EnvoyBootstrapHook +// TestTaskRunner_EnvoyBootstrapHook_sidecar_ok asserts the EnvoyBootstrapHook // creates Envoy's bootstrap.json configuration based on Connect proxy sidecars // registered for the task. -func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) { +func TestTaskRunner_EnvoyBootstrapHook_sidecar_ok(t *testing.T) { t.Parallel() testutil.RequireConsul(t) - testconsul := getTestConsul(t) - defer testconsul.Stop() + testConsul := getTestConsul(t) + defer testConsul.Stop() - alloc := mock.Alloc() + alloc := mock.ConnectAlloc() alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{ { Mode: "bridge", @@ -347,7 +381,7 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) { } sidecarTask := &structs.Task{ Name: "sidecar", - Kind: "connect-proxy:foo", + Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"), } tg.Tasks = append(tg.Tasks, sidecarTask) @@ -358,7 +392,7 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) { // Register Group Services consulConfig := consulapi.DefaultConfig() - consulConfig.Address = testconsul.HTTPAddr + consulConfig.Address = testConsul.HTTPAddr consulAPIClient, err := consulapi.NewClient(consulConfig) require.NoError(t, err) @@ -374,6 +408,7 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) { req := &interfaces.TaskPrestartRequest{ Task: sidecarTask, TaskDir: allocDir.NewTaskDir(sidecarTask.Name), + TaskEnv: taskenv.NewEmptyTaskEnv(), } require.NoError(t, req.TaskDir.Build(false, nil)) @@ -407,8 +442,85 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) { require.Equal(t, "", value) } +func TestTaskRunner_EnvoyBootstrapHook_gateway_ok(t *testing.T) { + t.Parallel() + logger := testlog.HCLogger(t) + + testConsul := getTestConsul(t) + defer testConsul.Stop() + + // Setup an Allocation + alloc := mock.ConnectIngressGatewayAlloc("bridge") + allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyBootstrapIngressGateway") + defer cleanupDir() + + // Get a Consul client + consulConfig := consulapi.DefaultConfig() + consulConfig.Address = testConsul.HTTPAddr + consulAPIClient, err := consulapi.NewClient(consulConfig) + require.NoError(t, err) + + // Register Group Services + serviceClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true) + go serviceClient.Run() + defer serviceClient.Shutdown() + require.NoError(t, serviceClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter()))) + + // Register Configuration Entry + ceClient := consulAPIClient.ConfigEntries() + set, _, err := ceClient.Set(&consulapi.IngressGatewayConfigEntry{ + Kind: consulapi.IngressGateway, + Name: "gateway-service", // matches job + Listeners: []consulapi.IngressListener{{ + Port: 2000, + Protocol: "tcp", + Services: []consulapi.IngressService{{ + Name: "service1", + }}, + }}, + }, nil) + require.NoError(t, err) + require.True(t, set) + + // Run Connect bootstrap hook + h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{ + Addr: consulConfig.Address, + }, logger)) + + req := &interfaces.TaskPrestartRequest{ + Task: alloc.Job.TaskGroups[0].Tasks[0], + TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name), + TaskEnv: taskenv.NewEmptyTaskEnv(), + } + require.NoError(t, req.TaskDir.Build(false, nil)) + + var resp interfaces.TaskPrestartResponse + + // Run the hook + require.NoError(t, h.Prestart(context.Background(), req, &resp)) + + // Assert the hook is done + require.True(t, resp.Done) + require.NotNil(t, resp.Env) + + // Read the Envoy Config file + env := map[string]string{ + taskenv.SecretsDir: req.TaskDir.SecretsDir, + } + f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env)) + require.NoError(t, err) + defer f.Close() + + var out envoyConfig + require.NoError(t, json.NewDecoder(f).Decode(&out)) + + // the only interesting thing on bootstrap is the presence of the cluster, + // everything is configured at runtime through xDS + require.Equal(t, "my-ingress-service", out.Node.Cluster) +} + // TestTaskRunner_EnvoyBootstrapHook_Noop asserts that the Envoy bootstrap hook -// is a noop for non-Connect proxy sidecar tasks. +// is a noop for non-Connect proxy sidecar / gateway tasks. func TestTaskRunner_EnvoyBootstrapHook_Noop(t *testing.T) { t.Parallel() logger := testlog.HCLogger(t) @@ -451,10 +563,10 @@ func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) { t.Parallel() testutil.RequireConsul(t) - testconsul := getTestConsul(t) - defer testconsul.Stop() + testConsul := getTestConsul(t) + defer testConsul.Stop() - alloc := mock.Alloc() + alloc := mock.ConnectAlloc() alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{ { Mode: "bridge", @@ -495,11 +607,12 @@ func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) { // Run Connect bootstrap Hook h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{ - Addr: testconsul.HTTPAddr, + Addr: testConsul.HTTPAddr, }, logger)) req := &interfaces.TaskPrestartRequest{ Task: sidecarTask, TaskDir: allocDir.NewTaskDir(sidecarTask.Name), + TaskEnv: taskenv.NewEmptyTaskEnv(), } require.NoError(t, req.TaskDir.Build(false, nil)) @@ -518,3 +631,64 @@ func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) { require.Error(t, err) require.True(t, os.IsNotExist(err)) } + +func TestTaskRunner_EnvoyBootstrapHook_extractNameAndKind(t *testing.T) { + t.Run("connect sidecar", func(t *testing.T) { + kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind( + structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"), + ) + require.Nil(t, err) + require.Equal(t, "connect-proxy", kind) + require.Equal(t, "foo", name) + }) + + t.Run("connect gateway", func(t *testing.T) { + kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind( + structs.NewTaskKind(structs.ConnectIngressPrefix, "foo"), + ) + require.Nil(t, err) + require.Equal(t, "connect-ingress", kind) + require.Equal(t, "foo", name) + }) + + t.Run("connect native", func(t *testing.T) { + _, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind( + structs.NewTaskKind(structs.ConnectNativePrefix, "foo"), + ) + require.EqualError(t, err, "envoy must be used as connect sidecar or gateway") + }) + + t.Run("normal task", func(t *testing.T) { + _, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind( + structs.TaskKind(""), + ) + require.EqualError(t, err, "envoy must be used as connect sidecar or gateway") + }) +} + +func TestTaskRunner_EnvoyBootstrapHook_grpcAddress(t *testing.T) { + bridgeH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig( + mock.ConnectIngressGatewayAlloc("bridge"), + new(config.ConsulConfig), + testlog.HCLogger(t), + )) + + hostH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig( + mock.ConnectIngressGatewayAlloc("host"), + new(config.ConsulConfig), + testlog.HCLogger(t), + )) + + t.Run("environment", func(t *testing.T) { + env := map[string]string{ + grpcConsulVariable: "1.2.3.4:9000", + } + require.Equal(t, "1.2.3.4:9000", bridgeH.grpcAddress(env)) + require.Equal(t, "1.2.3.4:9000", hostH.grpcAddress(env)) + }) + + t.Run("defaults", func(t *testing.T) { + require.Equal(t, "unix://alloc/tmp/consul_grpc.sock", bridgeH.grpcAddress(nil)) + require.Equal(t, "127.0.0.1:8502", hostH.grpcAddress(nil)) + }) +} diff --git a/client/allocrunner/taskrunner/task_runner_hooks.go b/client/allocrunner/taskrunner/task_runner_hooks.go index 07325c012038..c9c2f752ec10 100644 --- a/client/allocrunner/taskrunner/task_runner_hooks.go +++ b/client/allocrunner/taskrunner/task_runner_hooks.go @@ -127,7 +127,7 @@ func (tr *TaskRunner) initHooks() { })) } - if task.Kind.IsConnectProxy() { + if task.Kind.IsConnectProxy() || task.Kind.IsAnyConnectGateway() { tr.runnerHooks = append(tr.runnerHooks, newEnvoyBootstrapHook( newEnvoyBootstrapHookConfig(alloc, tr.clientConfig.ConsulConfig, hookLogger), )) diff --git a/client/allocrunner/taskrunner/template/template_test.go b/client/allocrunner/taskrunner/template/template_test.go index 3489b19489fe..7c9db1368563 100644 --- a/client/allocrunner/taskrunner/template/template_test.go +++ b/client/allocrunner/taskrunner/template/template_test.go @@ -154,7 +154,9 @@ func newTestHarness(t *testing.T, templates []*structs.Template, consul, vault b harness.taskDir = d if consul { - harness.consul, err = ctestutil.NewTestServerConfigT(t, nil) + harness.consul, err = ctestutil.NewTestServerConfigT(t, func(c *ctestutil.TestServerConfig) { + // defaults + }) if err != nil { t.Fatalf("error starting test Consul server: %v", err) } diff --git a/client/client.go b/client/client.go index 31a62c8ad68e..16339d25eb0f 100644 --- a/client/client.go +++ b/client/client.go @@ -101,6 +101,10 @@ const ( // Update sidecar_task.html when updating this. defaultConnectSidecarImage = "envoyproxy/envoy:v1.11.2@sha256:a7769160c9c1a55bb8d07a3b71ce5d64f72b1f665f10d81aa1581bc3cf850d09" + // defaultConnectGatewayImage is the image set in the node meta by default + // to be used by Consul Connect Gateway tasks. + defaultConnectGatewayImage = defaultConnectSidecarImage + // defaultConnectLogLevel is the log level set in the node meta by default // to be used by Consul Connect sidecar tasks defaultConnectLogLevel = "info" @@ -1386,6 +1390,9 @@ func (c *Client) setupNode() error { if _, ok := node.Meta["connect.sidecar_image"]; !ok { node.Meta["connect.sidecar_image"] = defaultConnectSidecarImage } + if _, ok := node.Meta["connect.gateway_image"]; !ok { + node.Meta["connect.gateway_image"] = defaultConnectGatewayImage + } if _, ok := node.Meta["connect.log_level"]; !ok { node.Meta["connect.log_level"] = defaultConnectLogLevel } diff --git a/command/agent/agent.go b/command/agent/agent.go index 4c1a700c90d1..909b7c9f73ba 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -77,6 +77,9 @@ type Agent struct { // consulCatalog is the subset of Consul's Catalog API Nomad uses. consulCatalog consul.CatalogAPI + // consulConfigEntries is the subset of Consul's Configuration Entires API Nomad uses. + consulConfigEntries consul.ConfigAPI + // consulACLs is Nomad's subset of Consul's ACL API Nomad uses. consulACLs consul.ACLsAPI @@ -669,7 +672,7 @@ func (a *Agent) setupServer() error { } // Create the server - server, err := nomad.NewServer(conf, a.consulCatalog, a.consulACLs) + server, err := nomad.NewServer(conf, a.consulCatalog, a.consulConfigEntries, a.consulACLs) if err != nil { return fmt.Errorf("server setup failed: %v", err) } @@ -1124,6 +1127,9 @@ func (a *Agent) setupConsul(consulConfig *config.ConsulConfig) error { // Create Consul Catalog client for service discovery. a.consulCatalog = client.Catalog() + // Create Consul ConfigEntries client for managing Config Entries. + a.consulConfigEntries = client.ConfigEntries() + // Create Consul ACL client for managing tokens. a.consulACLs = client.ACL() diff --git a/command/agent/consul/client.go b/command/agent/consul/client.go index fae7307074ba..645e907fd85c 100644 --- a/command/agent/consul/client.go +++ b/command/agent/consul/client.go @@ -105,6 +105,15 @@ type AgentAPI interface { UpdateTTL(id, output, status string) error } +// ConfigAPI is the consul/api.ConfigEntries API subset used by Nomad Server. +// +// ACL requirements +// - operator:write (server only) +type ConfigAPI interface { + Set(entry api.ConfigEntry, w *api.WriteOptions) (bool, *api.WriteMeta, error) + // Delete(kind, name string, w *api.WriteOptions) (*api.WriteMeta, error) (not used) +} + // ACLsAPI is the consul/api.ACL API subset used by Nomad Server. // // ACL requirements @@ -835,6 +844,9 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w return nil, fmt.Errorf("invalid Consul Connect configuration for service %q: %v", service.Name, err) } + // newConnectGateway returns nil if there's no Connect gateway. + gateway := newConnectGateway(service.Name, service.Connect) + // Determine whether to use meta or canary_meta var meta map[string]string if workload.Canary && len(service.CanaryMeta) > 0 { @@ -852,8 +864,15 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w // This enables the consul UI to show that Nomad registered this service meta["external-source"] = "nomad" + // Explicitly set the service kind in case this service represents a Connect gateway. + kind := api.ServiceKindTypical + if service.Connect.IsGateway() { + kind = api.ServiceKindIngressGateway + } + // Build the Consul Service registration request serviceReg := &api.AgentServiceRegistration{ + Kind: kind, ID: id, Name: service.Name, Tags: tags, @@ -862,6 +881,7 @@ func (c *ServiceClient) serviceRegs(ops *operations, service *structs.Service, w Port: port, Meta: meta, Connect: connect, // will be nil if no Connect stanza + Proxy: gateway, // will be nil if no Connect Gateway stanza } ops.regServices = append(ops.regServices, serviceReg) diff --git a/command/agent/consul/config_entries_testing.go b/command/agent/consul/config_entries_testing.go new file mode 100644 index 000000000000..9b3166e8190f --- /dev/null +++ b/command/agent/consul/config_entries_testing.go @@ -0,0 +1,55 @@ +package consul + +import ( + "sync" + + "github.com/hashicorp/consul/api" + "github.com/hashicorp/go-hclog" +) + +var _ ConfigAPI = (*MockConfigsAPI)(nil) + +type MockConfigsAPI struct { + logger hclog.Logger + + lock sync.Mutex + state struct { + error error + entries map[string]api.ConfigEntry + } +} + +func NewMockConfigsAPI(l hclog.Logger) *MockConfigsAPI { + return &MockConfigsAPI{ + logger: l.Named("mock_consul"), + state: struct { + error error + entries map[string]api.ConfigEntry + }{entries: make(map[string]api.ConfigEntry)}, + } +} + +// Set is a mock of ConfigAPI.Set +func (m *MockConfigsAPI) Set(entry api.ConfigEntry, w *api.WriteOptions) (bool, *api.WriteMeta, error) { + m.lock.Lock() + defer m.lock.Unlock() + + if m.state.error != nil { + return false, nil, m.state.error + } + + m.state.entries[entry.GetName()] = entry + + return true, &api.WriteMeta{ + RequestTime: 1, + }, nil +} + +// SetError is a helper method for configuring an error that will be returned +// on future calls to mocked methods. +func (m *MockConfigsAPI) SetError(err error) { + m.lock.Lock() + defer m.lock.Unlock() + + m.state.error = err +} diff --git a/command/agent/consul/connect.go b/command/agent/consul/connect.go index ef2cd602f439..c2cdc4ef1537 100644 --- a/command/agent/consul/connect.go +++ b/command/agent/consul/connect.go @@ -12,24 +12,66 @@ import ( // Connect struct. If the nomad Connect struct is nil, nil will be returned to // disable Connect for this service. func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.Networks) (*api.AgentServiceConnect, error) { - if nc == nil { + switch { + case nc == nil: // no connect stanza means there is no connect service to register return nil, nil - } - if nc.IsNative() { + case nc.IsGateway(): + // gateway settings are configured on the service block on the consul side + return nil, nil + + case nc.IsNative(): + // the service is connect native return &api.AgentServiceConnect{Native: true}, nil + + case nc.HasSidecar(): + sidecarReg, err := connectSidecarRegistration(serviceName, nc.SidecarService, networks) + if err != nil { + return nil, err + } + return &api.AgentServiceConnect{SidecarService: sidecarReg}, nil + + default: + return nil, fmt.Errorf("Connect configuration empty for service %s", serviceName) } +} - sidecarReg, err := connectSidecarRegistration(serviceName, nc.SidecarService, networks) - if err != nil { - return nil, err +// newConnectGateway creates a new Consul AgentServiceConnectProxyConfig struct based on +// a Nomad Connect struct. If the Nomad Connect struct does not contain a gateway, nil +// will be returned as this service is not a gateway. +func newConnectGateway(serviceName string, connect *structs.ConsulConnect) *api.AgentServiceConnectProxyConfig { + if !connect.IsGateway() { + return nil } - return &api.AgentServiceConnect{ - Native: false, - SidecarService: sidecarReg, - }, nil + proxy := connect.Gateway.Proxy + + envoyConfig := make(map[string]interface{}) + + if len(proxy.EnvoyGatewayBindAddresses) > 0 { + envoyConfig["envoy_gateway_bind_addresses"] = proxy.EnvoyGatewayBindAddresses + } + + if proxy.EnvoyGatewayNoDefaultBind { + envoyConfig["envoy_gateway_no_default_bind"] = true + } + + if proxy.EnvoyGatewayBindTaggedAddresses { + envoyConfig["envoy_gateway_bind_tagged_addresses"] = true + } + + if proxy.ConnectTimeout != nil { + envoyConfig["connect_timeout_ms"] = proxy.ConnectTimeout.Milliseconds() + } + + if len(proxy.Config) > 0 { + for k, v := range proxy.Config { + envoyConfig[k] = v + } + } + + return &api.AgentServiceConnectProxyConfig{Config: envoyConfig} } func connectSidecarRegistration(serviceName string, css *structs.ConsulSidecarService, networks structs.Networks) (*api.AgentServiceRegistration, error) { diff --git a/command/agent/consul/connect_test.go b/command/agent/consul/connect_test.go index 8ce74c1df7b3..d5628b91e48a 100644 --- a/command/agent/consul/connect_test.go +++ b/command/agent/consul/connect_test.go @@ -2,8 +2,10 @@ package consul import ( "testing" + "time" "github.com/hashicorp/consul/api" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" "github.com/stretchr/testify/require" ) @@ -397,3 +399,66 @@ func TestConnect_getExposePathPort(t *testing.T) { require.EqualError(t, err, "Connect only supported with exactly 1 network (found 2)") }) } + +func TestConnect_newConnectGateway(t *testing.T) { + t.Parallel() + + t.Run("not a gateway", func(t *testing.T) { + result := newConnectGateway("s1", &structs.ConsulConnect{Native: true}) + require.Nil(t, result) + }) + + t.Run("canonical empty", func(t *testing.T) { + result := newConnectGateway("s1", &structs.ConsulConnect{ + Gateway: &structs.ConsulGateway{ + Proxy: &structs.ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(1 * time.Second), + EnvoyGatewayBindTaggedAddresses: false, + EnvoyGatewayBindAddresses: nil, + EnvoyGatewayNoDefaultBind: false, + Config: nil, + }, + }, + }) + require.Equal(t, &api.AgentServiceConnectProxyConfig{ + Config: map[string]interface{}{ + "connect_timeout_ms": int64(1000), + }, + }, result) + }) + + t.Run("full", func(t *testing.T) { + result := newConnectGateway("s1", &structs.ConsulConnect{ + Gateway: &structs.ConsulGateway{ + Proxy: &structs.ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(1 * time.Second), + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{ + "service1": &structs.ConsulGatewayBindAddress{ + Address: "10.0.0.1", + Port: 2000, + }, + }, + EnvoyGatewayNoDefaultBind: true, + Config: map[string]interface{}{ + "foo": 1, + }, + }, + }, + }) + require.Equal(t, &api.AgentServiceConnectProxyConfig{ + Config: map[string]interface{}{ + "connect_timeout_ms": int64(1000), + "envoy_gateway_bind_tagged_addresses": true, + "envoy_gateway_bind_addresses": map[string]*structs.ConsulGatewayBindAddress{ + "service1": &structs.ConsulGatewayBindAddress{ + Address: "10.0.0.1", + Port: 2000, + }, + }, + "envoy_gateway_no_default_bind": true, + "foo": 1, + }, + }, result) + }) +} diff --git a/command/agent/consul/int_test.go b/command/agent/consul/int_test.go index 48191c162e73..b4760a768f8d 100644 --- a/command/agent/consul/int_test.go +++ b/command/agent/consul/int_test.go @@ -39,7 +39,7 @@ func TestConsul_Integration(t *testing.T) { if testing.Short() { t.Skip("-short set; skipping") } - require := require.New(t) + r := require.New(t) // Create an embedded Consul server testconsul, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { @@ -133,7 +133,7 @@ func TestConsul_Integration(t *testing.T) { taskDir := allocDir.NewTaskDir(task.Name) vclient := vaultclient.NewMockVaultClient() consulClient, err := consulapi.NewClient(consulConfig) - require.Nil(err) + r.Nil(err) serviceClient := consul.NewServiceClient(consulClient.Agent(), testlog.HCLogger(t), true) defer serviceClient.Shutdown() // just-in-case cleanup @@ -165,7 +165,7 @@ func TestConsul_Integration(t *testing.T) { } tr, err := taskrunner.NewTaskRunner(config) - require.NoError(err) + r.NoError(err) go tr.Run() defer func() { // Make sure we always shutdown task runner when the test exits @@ -180,7 +180,7 @@ func TestConsul_Integration(t *testing.T) { // Block waiting for the service to appear catalog := consulClient.Catalog() res, meta, err := catalog.Service("httpd2", "test", nil) - require.Nil(err) + r.Nil(err) for i := 0; len(res) == 0 && i < 10; i++ { //Expected initial request to fail, do a blocking query @@ -189,7 +189,7 @@ func TestConsul_Integration(t *testing.T) { t.Fatalf("error querying for service: %v", err) } } - require.Len(res, 1) + r.Len(res, 1) // Truncate results res = res[:] @@ -197,16 +197,16 @@ func TestConsul_Integration(t *testing.T) { // Assert the service with the checks exists for i := 0; len(res) == 0 && i < 10; i++ { res, meta, err = catalog.Service("httpd", "http", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second}) - require.Nil(err) + r.Nil(err) } - require.Len(res, 1) + r.Len(res, 1) // Assert the script check passes (mock_driver script checks always // pass) after having time to run once time.Sleep(2 * time.Second) checks, _, err := consulClient.Health().Checks("httpd", nil) - require.Nil(err) - require.Len(checks, 2) + r.Nil(err) + r.Len(checks, 2) for _, check := range checks { if expected := "httpd"; check.ServiceName != expected { @@ -261,7 +261,7 @@ func TestConsul_Integration(t *testing.T) { // Ensure Consul is clean services, _, err := catalog.Services(nil) - require.Nil(err) - require.Len(services, 1) - require.Contains(services, "consul") + r.Nil(err) + r.Len(services, 1) + r.Contains(services, "consul") } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 0a5c99a22c6c..7931180cfe80 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1290,6 +1290,111 @@ func ApiConsulConnectToStructs(in *api.ConsulConnect) *structs.ConsulConnect { Native: in.Native, SidecarService: apiConnectSidecarServiceToStructs(in.SidecarService), SidecarTask: apiConnectSidecarTaskToStructs(in.SidecarTask), + Gateway: apiConnectGatewayToStructs(in.Gateway), + } +} + +func apiConnectGatewayToStructs(in *api.ConsulGateway) *structs.ConsulGateway { + if in == nil { + return nil + } + + return &structs.ConsulGateway{ + Proxy: apiConnectGatewayProxyToStructs(in.Proxy), + Ingress: apiConnectIngressGatewayToStructs(in.Ingress), + } +} + +func apiConnectGatewayProxyToStructs(in *api.ConsulGatewayProxy) *structs.ConsulGatewayProxy { + if in == nil { + return nil + } + + var bindAddresses map[string]*structs.ConsulGatewayBindAddress + if in.EnvoyGatewayBindAddresses != nil { + bindAddresses = make(map[string]*structs.ConsulGatewayBindAddress) + for k, v := range in.EnvoyGatewayBindAddresses { + bindAddresses[k] = &structs.ConsulGatewayBindAddress{ + Address: v.Address, + Port: v.Port, + } + } + } + + return &structs.ConsulGatewayProxy{ + ConnectTimeout: in.ConnectTimeout, + EnvoyGatewayBindTaggedAddresses: in.EnvoyGatewayBindTaggedAddresses, + EnvoyGatewayBindAddresses: bindAddresses, + EnvoyGatewayNoDefaultBind: in.EnvoyGatewayNoDefaultBind, + Config: helper.CopyMapStringInterface(in.Config), + } +} + +func apiConnectIngressGatewayToStructs(in *api.ConsulIngressConfigEntry) *structs.ConsulIngressConfigEntry { + if in == nil { + return nil + } + + return &structs.ConsulIngressConfigEntry{ + TLS: apiConnectGatewayTLSConfig(in.TLS), + Listeners: apiConnectIngressListenersToStructs(in.Listeners), + } +} + +func apiConnectGatewayTLSConfig(in *api.ConsulGatewayTLSConfig) *structs.ConsulGatewayTLSConfig { + if in == nil { + return nil + } + + return &structs.ConsulGatewayTLSConfig{ + Enabled: in.Enabled, + } +} + +func apiConnectIngressListenersToStructs(in []*api.ConsulIngressListener) []*structs.ConsulIngressListener { + if len(in) == 0 { + return nil + } + + listeners := make([]*structs.ConsulIngressListener, len(in)) + for i, listener := range in { + listeners[i] = apiConnectIngressListenerToStructs(listener) + } + return listeners +} + +func apiConnectIngressListenerToStructs(in *api.ConsulIngressListener) *structs.ConsulIngressListener { + if in == nil { + return nil + } + + return &structs.ConsulIngressListener{ + Port: in.Port, + Protocol: in.Protocol, + Services: apiConnectIngressServicesToStructs(in.Services), + } +} + +func apiConnectIngressServicesToStructs(in []*api.ConsulIngressService) []*structs.ConsulIngressService { + if len(in) == 0 { + return nil + } + + services := make([]*structs.ConsulIngressService, len(in)) + for i, service := range in { + services[i] = apiConnectIngressServiceToStructs(service) + } + return services +} + +func apiConnectIngressServiceToStructs(in *api.ConsulIngressService) *structs.ConsulIngressService { + if in == nil { + return nil + } + + return &structs.ConsulIngressService{ + Name: in.Name, + Hosts: helper.CopySliceString(in.Hosts), } } @@ -1313,7 +1418,7 @@ func apiConnectSidecarServiceProxyToStructs(in *api.ConsulProxy) *structs.Consul LocalServicePort: in.LocalServicePort, Upstreams: apiUpstreamsToStructs(in.Upstreams), Expose: apiConsulExposeConfigToStructs(in.ExposeConfig), - Config: in.Config, + Config: helper.CopyMapStringInterface(in.Config), } } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 9bc2f5ac6e4c..29bf1b31fe0e 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -3004,7 +3004,7 @@ func TestConversion_apiConnectSidecarServiceProxyToStructs(t *testing.T) { require.Equal(t, &structs.ConsulProxy{ LocalServiceAddress: "192.168.30.1", LocalServicePort: 9000, - Config: config, + Config: nil, Upstreams: []structs.ConsulUpstream{{ DestinationName: "upstream", }}, diff --git a/go.sum b/go.sum index 7af177f7af12..c1a9951251e5 100644 --- a/go.sum +++ b/go.sum @@ -141,7 +141,6 @@ github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533 h1:8wZizuKuZVu5COB7Es github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/container-storage-interface/spec v1.2.0-rc1.0.20191021210849-a33ece0a8a9f h1:m2LYF3fo9IPapVt5FGRVw5bJPmlWqWIezB0jkQh03Zo= github.com/container-storage-interface/spec v1.2.0-rc1.0.20191021210849-a33ece0a8a9f/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v1.0.0 h1:fU3UuQapBs+zLJu82NhR11Rif1ny2zfMMAyPJzSN5tQ= @@ -175,7 +174,6 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= @@ -210,7 +208,6 @@ github.com/docker/docker-credential-helpers v0.6.2-0.20180719074751-73e5f5dbfea3 github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916 h1:yWHOI+vFjEsAakUTSrtqc/SAHrhSkmn48pqjidZX3QA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= @@ -231,9 +228,13 @@ github.com/endocrimes/go-winio v0.4.13-0.20190628114223-fb47a8b41948 h1:PgcXIRC4 github.com/endocrimes/go-winio v0.4.13-0.20190628114223-fb47a8b41948/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +<<<<<<< HEAD +github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s= +======= github.com/envoyproxy/go-control-plane v0.9.5 h1:lRJIqDD8yjV1YyPRqecMdytjDLs2fTXq363aCib5xPU= github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s= github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= +>>>>>>> master github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= @@ -269,7 +270,10 @@ github.com/godbus/dbus v5.0.1+incompatible h1:fsDsnr/6MFSIm3kl6JJpq5pH+vO/rA5jUu github.com/godbus/dbus v5.0.1+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +<<<<<<< HEAD +======= github.com/gogo/googleapis v1.2.0 h1:Z0v3OJDotX9ZBpdz2V+AI7F4fITSZhVE5mg6GQppwMM= +>>>>>>> master github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -335,12 +339,10 @@ github.com/hashicorp/consul v1.7.7/go.mod h1:urbfGaVZDmnXC6geg0LYPh/SRUk1E8nfmDH github.com/hashicorp/consul-template v0.24.1 h1:96zTJ5YOq4HMTgtehXRvzGoQNEG2Z4jBYY5ofhq8/Cc= github.com/hashicorp/consul-template v0.24.1/go.mod h1:KcTEopo2kCp7kww0d4oG7d3oX2Uou4hzb1Rs/wY9TVI= github.com/hashicorp/consul/api v1.2.0/go.mod h1:1SIkFYi2ZTXUE5Kgt179+4hH33djo11+0Eo2XgTAtkw= -github.com/hashicorp/consul/api v1.4.0 h1:jfESivXnO5uLdH650JU/6AnjRoHrLhULq0FnC3Kp9EY= github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU= github.com/hashicorp/consul/api v1.6.0 h1:SZB2hQW8AcTOpfDmiVblQbijxzsRuiyy0JpHfabvHio= github.com/hashicorp/consul/api v1.6.0/go.mod h1:1NSuaUUkFaJzMasbfq/11wKYWSR67Xn6r2DXKhuDNFg= github.com/hashicorp/consul/sdk v0.2.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.4.0 h1:zBtCfKJZcJDBvSCkQJch4ulp59m1rATFLKwNo/LYY30= github.com/hashicorp/consul/sdk v0.4.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= github.com/hashicorp/consul/sdk v0.6.0 h1:FfhMEkwvQl57CildXJyGHnwGGM4HMODGyfjGwNM1Vdw= github.com/hashicorp/consul/sdk v0.6.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= @@ -480,7 +482,6 @@ github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 h1:BvV6PYcRz0yGnW github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -535,7 +536,6 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0 h1:tEElEatulEHDeedTxwckzyYMA5c86fbmNIUL1hBIiTg= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= @@ -549,7 +549,6 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b h1:9+ke9YJ9KGWw5ANXK6ozjoK47uI3uNbXv4YVINBnGm8= github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.3 h1:gqwbsGvc0jbhAPW/26WfEoSiPANAVlR49AAVdvaTjI4= github.com/mitchellh/go-testing-interface v1.0.3/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -573,10 +572,8 @@ github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/moby/sys/mountinfo v0.1.3 h1:KIrhRO14+AkwKvG/g2yIpNMOUVZ02xNhOw8KY1WsLOI= github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -668,16 +665,21 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +<<<<<<< HEAD +======= github.com/rboyer/safeio v0.2.1 h1:05xhhdRNAdS3apYm7JRjOqngf4xruaW959jmRxGDuSU= +>>>>>>> master github.com/rboyer/safeio v0.2.1/go.mod h1:Cq/cEPK+YXFn622lsQ0K4KsPZSPtaptHHEldsy7Fmig= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03 h1:Wdi9nwnhFNAlseAOekn6B5G/+GMtks9UKbvRU/CMM/o= github.com/renier/xmlrpc v0.0.0-20170708154548-ce4a1a486c03/go.mod h1:gRAiPF5C5Nd0eyyRdqIu9qTiFSoZzpTq727b5B8fkkU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +<<<<<<< HEAD +======= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +>>>>>>> master github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -714,7 +716,6 @@ github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= @@ -750,7 +751,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -763,7 +763,6 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -1007,7 +1006,10 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +<<<<<<< HEAD +======= google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= +>>>>>>> master google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= diff --git a/helper/funcs.go b/helper/funcs.go index d9084e09b4e8..72b0ed69c269 100644 --- a/helper/funcs.go +++ b/helper/funcs.go @@ -93,11 +93,19 @@ func StringToPtr(str string) *string { return &str } -// TimeToPtr returns the pointer to a time stamp +// TimeToPtr returns the pointer to a time.Duration. func TimeToPtr(t time.Duration) *time.Duration { return &t } +// CompareTimePtrs return true if a is the same as b. +func CompareTimePtrs(a, b *time.Duration) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} + // Float64ToPtr returns the pointer to an float64 func Float64ToPtr(f float64) *float64 { return &f @@ -287,6 +295,19 @@ func CopyMapStringStruct(m map[string]struct{}) map[string]struct{} { return c } +func CopyMapStringInterface(m map[string]interface{}) map[string]interface{} { + l := len(m) + if l == 0 { + return nil + } + + c := make(map[string]interface{}, l) + for k, v := range m { + c[k] = v + } + return c +} + func CopyMapStringInt(m map[string]int) map[string]int { l := len(m) if l == 0 { diff --git a/helper/funcs_test.go b/helper/funcs_test.go index cd5e26d38abd..5c5fea94ea71 100644 --- a/helper/funcs_test.go +++ b/helper/funcs_test.go @@ -5,6 +5,7 @@ import ( "reflect" "sort" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -32,6 +33,25 @@ func TestSliceStringContains(t *testing.T) { require.False(t, SliceStringContains(list, "d")) } +func TestCompareTimePtrs(t *testing.T) { + t.Run("nil", func(t *testing.T) { + a := (*time.Duration)(nil) + b := (*time.Duration)(nil) + require.True(t, CompareTimePtrs(a, b)) + c := TimeToPtr(3 * time.Second) + require.False(t, CompareTimePtrs(a, c)) + require.False(t, CompareTimePtrs(c, a)) + }) + + t.Run("not nil", func(t *testing.T) { + a := TimeToPtr(1 * time.Second) + b := TimeToPtr(1 * time.Second) + c := TimeToPtr(2 * time.Second) + require.True(t, CompareTimePtrs(a, b)) + require.False(t, CompareTimePtrs(a, c)) + }) +} + func TestCompareSliceSetString(t *testing.T) { cases := []struct { A []string @@ -134,6 +154,19 @@ func TestCopyMapStringSliceString(t *testing.T) { } } +func TestCopyMapSliceInterface(t *testing.T) { + m := map[string]interface{}{ + "foo": "bar", + "baz": 2, + } + + c := CopyMapStringInterface(m) + require.True(t, reflect.DeepEqual(m, c)) + + m["foo"] = "zzz" + require.False(t, reflect.DeepEqual(m, c)) +} + func TestClearEnvVar(t *testing.T) { type testCase struct { input string diff --git a/jobspec/parse_service.go b/jobspec/parse_service.go index 0e36a3beec91..95f9b3ad75c0 100644 --- a/jobspec/parse_service.go +++ b/jobspec/parse_service.go @@ -146,6 +146,7 @@ func parseService(o *ast.ObjectItem) (*api.Service, error) { func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) { valid := []string{ "native", + "gateway", "sidecar_service", "sidecar_task", } @@ -160,6 +161,7 @@ func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) { return nil, err } + delete(m, "gateway") delete(m, "sidecar_service") delete(m, "sidecar_task") @@ -174,8 +176,20 @@ func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) { return nil, fmt.Errorf("connect should be an object") } + // Parse the gateway + o := connectList.Filter("gateway") + if len(o.Items) > 1 { + return nil, fmt.Errorf("only one 'gateway' block allowed per task") + } else if len(o.Items) == 1 { + g, err := parseGateway(o.Items[0]) + if err != nil { + return nil, fmt.Errorf("gateway, %v", err) + } + connect.Gateway = g + } + // Parse the sidecar_service - o := connectList.Filter("sidecar_service") + o = connectList.Filter("sidecar_service") if len(o.Items) == 0 { return &connect, nil } @@ -207,6 +221,329 @@ func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) { return &connect, nil } +func parseGateway(o *ast.ObjectItem) (*api.ConsulGateway, error) { + valid := []string{ + "proxy", + "ingress", + } + + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, multierror.Prefix(err, "gateway ->") + } + + var gateway api.ConsulGateway + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + delete(m, "proxy") + delete(m, "ingress") + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &gateway, + }) + if err != nil { + return nil, err + } + if err := dec.Decode(m); err != nil { + return nil, fmt.Errorf("gateway: %v", err) + } + + // list of parameters + var listVal *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("proxy: should be an object") + } + + // extract and parse the proxy block + po := listVal.Filter("proxy") + if len(po.Items) != 1 { + return nil, fmt.Errorf("must have one 'proxy' block") + } + proxy, err := parseGatewayProxy(po.Items[0]) + if err != nil { + return nil, fmt.Errorf("proxy, %v", err) + } + gateway.Proxy = proxy + + // extract and parse the ingress block + io := listVal.Filter("ingress") + if len(io.Items) != 1 { + // in the future, may be terminating or mesh block instead + return nil, fmt.Errorf("must have one 'ingress' block") + } + ingress, err := parseIngressConfigEntry(io.Items[0]) + if err != nil { + return nil, fmt.Errorf("ingress, %v", err) + } + gateway.Ingress = ingress + + return &gateway, nil +} + +// parseGatewayProxy parses envoy gateway proxy options supported by Consul. +// +// consul.io/docs/connect/proxies/envoy#gateway-options +func parseGatewayProxy(o *ast.ObjectItem) (*api.ConsulGatewayProxy, error) { + valid := []string{ + "connect_timeout", + "envoy_gateway_bind_tagged_addresses", + "envoy_gateway_bind_addresses", + "envoy_gateway_no_default_bind", + "envoy_dns_discovery_type", + "config", + } + + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, multierror.Prefix(err, "proxy ->") + } + + var proxy api.ConsulGatewayProxy + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + delete(m, "config") + delete(m, "envoy_gateway_bind_addresses") + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &proxy, + }) + if err != nil { + return nil, err + } + if err := dec.Decode(m); err != nil { + return nil, fmt.Errorf("proxy: %v", err) + } + + var listVal *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("proxy: should be an object") + } + + // need to parse envoy_gateway_bind_addresses if present + + if ebo := listVal.Filter("envoy_gateway_bind_addresses"); len(ebo.Items) > 0 { + proxy.EnvoyGatewayBindAddresses = make(map[string]*api.ConsulGatewayBindAddress) + for _, listenerM := range ebo.Items { // object item, each listener object + listenerName := listenerM.Keys[0].Token.Value().(string) + + var listenerListVal *ast.ObjectList + if ot, ok := listenerM.Val.(*ast.ObjectType); ok { + listenerListVal = ot.List + } else { + return nil, fmt.Errorf("listener: should be an object") + } + + var bind api.ConsulGatewayBindAddress + if err := hcl.DecodeObject(&bind, listenerListVal); err != nil { + panic(err) + } + proxy.EnvoyGatewayBindAddresses[listenerName] = &bind + } + } + + // need to parse the opaque config if present + + if co := listVal.Filter("config"); len(co.Items) > 1 { + return nil, fmt.Errorf("only 1 meta object supported") + } else if len(co.Items) == 1 { + var mSlice []map[string]interface{} + if err := hcl.DecodeObject(&mSlice, co.Items[0].Val); err != nil { + return nil, err + } + + if len(mSlice) > 1 { + return nil, fmt.Errorf("only 1 meta object supported") + } + + m := mSlice[0] + + if err := mapstructure.WeakDecode(m, &proxy.Config); err != nil { + return nil, err + } + + proxy.Config = flattenMapSlice(proxy.Config) + } + + return &proxy, nil +} + +func parseConsulIngressService(o *ast.ObjectItem) (*api.ConsulIngressService, error) { + valid := []string{ + "name", + "hosts", + } + + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, multierror.Prefix(err, "service ->") + } + + var service api.ConsulIngressService + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &service, + }) + if err != nil { + return nil, err + } + + if err := dec.Decode(m); err != nil { + return nil, err + } + + return &service, nil +} + +func parseConsulIngressListener(o *ast.ObjectItem) (*api.ConsulIngressListener, error) { + valid := []string{ + "port", + "protocol", + "service", + } + + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, multierror.Prefix(err, "listener ->") + } + + var listener api.ConsulIngressListener + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + delete(m, "service") + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &listener, + }) + if err != nil { + return nil, err + } + + if err := dec.Decode(m); err != nil { + return nil, err + } + + // Parse services + + var listVal *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("listener: should be an object") + } + + so := listVal.Filter("service") + if len(so.Items) > 0 { + listener.Services = make([]*api.ConsulIngressService, len(so.Items)) + for i := range so.Items { + is, err := parseConsulIngressService(so.Items[i]) + if err != nil { + return nil, err + } + listener.Services[i] = is + } + } + return &listener, nil +} + +func parseConsulGatewayTLS(o *ast.ObjectItem) (*api.ConsulGatewayTLSConfig, error) { + valid := []string{ + "enabled", + } + + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, multierror.Prefix(err, "tls ->") + } + + var tls api.ConsulGatewayTLSConfig + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + Result: &tls, + }) + if err != nil { + return nil, err + } + + if err := dec.Decode(m); err != nil { + return nil, err + } + + return &tls, nil +} + +func parseIngressConfigEntry(o *ast.ObjectItem) (*api.ConsulIngressConfigEntry, error) { + valid := []string{ + "tls", + "listener", + } + + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, multierror.Prefix(err, "ingress ->") + } + + var ingress api.ConsulIngressConfigEntry + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + delete(m, "tls") + delete(m, "listener") + + // Parse tls and listener(s) + + var listVal *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("ingress: should be an object") + } + + if to := listVal.Filter("tls"); len(to.Items) > 1 { + return nil, fmt.Errorf("only 1 tls object supported") + } else if len(to.Items) == 1 { + if tls, err := parseConsulGatewayTLS(to.Items[0]); err != nil { + return nil, err + } else { + ingress.TLS = tls + } + } + + lo := listVal.Filter("listener") + if len(lo.Items) > 0 { + ingress.Listeners = make([]*api.ConsulIngressListener, len(lo.Items)) + for i := range lo.Items { + listener, err := parseConsulIngressListener(lo.Items[i]) + if err != nil { + return nil, err + } + ingress.Listeners[i] = listener + } + } + + return &ingress, nil +} + func parseSidecarService(o *ast.ObjectItem) (*api.ConsulSidecarService, error) { valid := []string{ "port", @@ -340,6 +677,8 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) { return nil, fmt.Errorf("proxy: %v", err) } + // Parse upstreams, expose, and config + var listVal *ast.ObjectList if ot, ok := o.Val.(*ast.ObjectType); ok { listVal = ot.List @@ -347,8 +686,6 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) { return nil, fmt.Errorf("proxy: should be an object") } - // Parse the proxy - uo := listVal.Filter("upstreams") if len(uo.Items) > 0 { proxy.Upstreams = make([]*api.ConsulUpstream, len(uo.Items)) diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index c294b750ccf3..05af3a655b0f 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -690,6 +690,37 @@ func TestParse(t *testing.T) { }, false, }, + { + "service-check-pass-fail.hcl", + &api.Job{ + ID: helper.StringToPtr("check_pass_fail"), + Name: helper.StringToPtr("check_pass_fail"), + Type: helper.StringToPtr("service"), + TaskGroups: []*api.TaskGroup{{ + Name: helper.StringToPtr("group"), + Count: helper.IntToPtr(1), + Tasks: []*api.Task{{ + Name: "task", + Services: []*api.Service{{ + Name: "service", + PortLabel: "http", + Checks: []api.ServiceCheck{{ + Name: "check-name", + Type: "http", + Path: "/", + Interval: 10 * time.Second, + Timeout: 2 * time.Second, + InitialStatus: capi.HealthPassing, + Method: "POST", + SuccessBeforePassing: 3, + FailuresBeforeCritical: 4, + }}, + }}, + }}, + }}, + }, + false, + }, { "service-check-bad-header.hcl", nil, @@ -1366,7 +1397,63 @@ func TestParse(t *testing.T) { }, false, }, - + { + "tg-service-connect-gateway-ingress.hcl", + &api.Job{ + ID: helper.StringToPtr("connect_gateway_ingress"), + Name: helper.StringToPtr("connect_gateway_ingress"), + TaskGroups: []*api.TaskGroup{{ + Name: helper.StringToPtr("group"), + Services: []*api.Service{{ + Name: "ingress-gateway-service", + Connect: &api.ConsulConnect{ + Gateway: &api.ConsulGateway{ + Proxy: &api.ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(3 * time.Second), + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: map[string]*api.ConsulGatewayBindAddress{ + "listener1": {Address: "10.0.0.1", Port: 8888}, + "listener2": {Address: "10.0.0.2", Port: 8889}, + }, + EnvoyGatewayNoDefaultBind: true, + Config: map[string]interface{}{"foo": "bar"}, + }, + Ingress: &api.ConsulIngressConfigEntry{ + TLS: &api.ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*api.ConsulIngressListener{{ + Port: 8001, + Protocol: "tcp", + Services: []*api.ConsulIngressService{{ + Name: "service1", + Hosts: []string{ + "127.0.0.1:8001", + "[::1]:8001", + }}, { + Name: "service2", + Hosts: []string{ + "10.0.0.1:8001", + }}, + }}, { + Port: 8080, + Protocol: "http", + Services: []*api.ConsulIngressService{{ + Name: "nginx", + Hosts: []string{ + "2.2.2.2:8080", + }, + }}, + }, + }, + }, + }, + }, + }}, + }}, + }, + false, + }, { "tg-scaling-policy-minimal.hcl", &api.Job{ diff --git a/jobspec/test-fixtures/tg-service-connect-gateway-ingress.hcl b/jobspec/test-fixtures/tg-service-connect-gateway-ingress.hcl new file mode 100644 index 000000000000..453440df7039 --- /dev/null +++ b/jobspec/test-fixtures/tg-service-connect-gateway-ingress.hcl @@ -0,0 +1,55 @@ +job "connect_gateway_ingress" { + group "group" { + service { + name = "ingress-gateway-service" + + connect { + gateway { + proxy { + connect_timeout = "3s" + envoy_gateway_bind_tagged_addresses = true + envoy_gateway_bind_addresses "listener1" { + address = "10.0.0.1" + port = 8888 + } + envoy_gateway_bind_addresses "listener2" { + address = "10.0.0.2" + port = 8889 + } + envoy_gateway_no_default_bind = true + config { + foo = "bar" + } + } + ingress { + tls { + enabled = true + } + + listener { + port = 8001 + protocol = "tcp" + service { + name = "service1" + hosts = ["127.0.0.1:8001", "[::1]:8001"] + } + service { + name = "service2" + hosts = ["10.0.0.1:8001"] + } + } + + listener { + port = 8080 + protocol = "http" + service { + name = "nginx" + hosts = ["2.2.2.2:8080"] + } + } + } + } + } + } + } +} diff --git a/nomad/consul.go b/nomad/consul.go index 7036ee480a66..f6bd1960c0e8 100644 --- a/nomad/consul.go +++ b/nomad/consul.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/command/agent/consul" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -35,6 +36,13 @@ const ( siTokenRevocationInterval = 5 * time.Minute ) +const ( + // configEntriesRequestRateLimit is the maximum number of requests per second + // Nomad will make against Consul for operations on global Configuration Entry + // objects. + configEntriesRequestRateLimit rate.Limit = 10 +) + const ( // ConsulPolicyWrite is the literal text of the policy field of a Consul Policy // Rule that we check when validating an Operator Consul token against the @@ -435,3 +443,107 @@ func (s *Server) purgeSITokenAccessors(accessors []*structs.SITokenAccessor) err _, _, err := s.raftApply(structs.ServiceIdentityAccessorDeregisterRequestType, request) return err } + +// ConsulConfigsAPI is an abstraction over the consul/api.ConfigEntries API used by +// Nomad Server. +// +// Nomad will only perform write operations on Consul Ingress Gateway Configuration Entries. +// Removing the entries is not particularly safe, given that multiple Nomad clusters +// may be writing to the same config entries, which are global in the Consul scope. +type ConsulConfigsAPI interface { + // SetIngressGatewayConfigEntry adds the given ConfigEntry to Consul, overwriting + // the previous entry if set. + SetIngressGatewayConfigEntry(ctx context.Context, service string, entry *structs.ConsulIngressConfigEntry) error + + // Stop is used to stop additional creations of Configuration Entries. Intended to + // be used on Nomad Server shutdown. + Stop() +} + +type consulConfigsAPI struct { + // configsClient is the API subset of the real Consul client we need for + // managing Configuration Entries. + configsClient consul.ConfigAPI + + // limiter is used to rate limit requests to Consul + limiter *rate.Limiter + + // logger is used to log messages + logger hclog.Logger + + // lock protects the stopped flag, which prevents use of the consul configs API + // client after shutdown. + lock sync.Mutex + stopped bool +} + +func NewConsulConfigsAPI(configsClient consul.ConfigAPI, logger hclog.Logger) *consulConfigsAPI { + return &consulConfigsAPI{ + configsClient: configsClient, + limiter: rate.NewLimiter(configEntriesRequestRateLimit, int(configEntriesRequestRateLimit)), + logger: logger, + } +} + +func (c *consulConfigsAPI) Stop() { + c.lock.Lock() + defer c.lock.Unlock() + c.stopped = true +} + +func (c *consulConfigsAPI) SetIngressGatewayConfigEntry(ctx context.Context, service string, entry *structs.ConsulIngressConfigEntry) error { + configEntry := convertIngressGatewayConfig(service, entry) + return c.setConfigEntry(ctx, configEntry) +} + +// setConfigEntry will set the Configuration Entry of any type Consul supports. +func (c *consulConfigsAPI) setConfigEntry(ctx context.Context, entry api.ConfigEntry) error { + defer metrics.MeasureSince([]string{"nomad", "consul", "create_config_entry"}, time.Now()) + + // make sure the background deletion goroutine has not been stopped + c.lock.Lock() + stopped := c.stopped + c.lock.Unlock() + + if stopped { + return errors.New("client stopped and may not longer create config entries") + } + + // ensure we are under our wait limit + if err := c.limiter.Wait(ctx); err != nil { + return err + } + + _, _, err := c.configsClient.Set(entry, nil) + return err +} + +func convertIngressGatewayConfig(service string, entry *structs.ConsulIngressConfigEntry) api.ConfigEntry { + var listeners []api.IngressListener = nil + for _, listener := range entry.Listeners { + var services []api.IngressService = nil + for _, service := range listener.Services { + services = append(services, api.IngressService{ + Name: service.Name, + Hosts: helper.CopySliceString(service.Hosts), + }) + } + listeners = append(listeners, api.IngressListener{ + Port: listener.Port, + Protocol: listener.Protocol, + Services: services, + }) + } + + tlsEnabled := false + if entry.TLS != nil && entry.TLS.Enabled { + tlsEnabled = true + } + + return &api.IngressGatewayConfigEntry{ + Kind: api.IngressGateway, + Name: service, + TLS: api.GatewayTLSConfig{Enabled: tlsEnabled}, + Listeners: listeners, + } +} diff --git a/nomad/consul_test.go b/nomad/consul_test.go index 333cd2e38196..3eacbfe65bf9 100644 --- a/nomad/consul_test.go +++ b/nomad/consul_test.go @@ -18,6 +18,39 @@ import ( var _ ConsulACLsAPI = (*consulACLsAPI)(nil) var _ ConsulACLsAPI = (*mockConsulACLsAPI)(nil) +var _ ConsulConfigsAPI = (*consulConfigsAPI)(nil) + +func TestConsulConfigsAPI_SetIngressGatewayConfigEntry(t *testing.T) { + t.Parallel() + + try := func(t *testing.T, expErr error) { + logger := testlog.HCLogger(t) + configsAPI := consul.NewMockConfigsAPI(logger) // agent + configsAPI.SetError(expErr) + + c := NewConsulConfigsAPI(configsAPI, logger) + + ctx := context.Background() + err := c.SetIngressGatewayConfigEntry(ctx, "service1", &structs.ConsulIngressConfigEntry{ + TLS: nil, + Listeners: nil, + }) + + if expErr != nil { + require.Equal(t, expErr, err) + } else { + require.NoError(t, err) + } + } + + t.Run("set ingress CE success", func(t *testing.T) { + try(t, nil) + }) + + t.Run("set ingress CE failure", func(t *testing.T) { + try(t, errors.New("consul broke")) + }) +} type revokeRequest struct { accessorID string diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index e1eb04ff6d3e..f57b980364f4 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -277,6 +277,22 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis } } + // Create or Update Consul Configuration Entries defined in the job. For now + // Nomad only supports Configuration Entries of type "ingress-gateway" for managing + // Consul Connect Ingress Gateway tasks derived from TaskGroup services. + // + // This is done as a blocking operation that prevents the job from being + // submitted if the configuration entries cannot be set in Consul. + // + // Every job update will re-write the Configuration Entry into Consul. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for service, entry := range args.Job.ConfigEntries() { + if err := j.srv.consulConfigEntries.SetIngressGatewayConfigEntry(ctx, service, entry); err != nil { + return err + } + } + // Enforce Sentinel policies. Pass a copy of the job to prevent // sentinel from altering it. policyWarnings, err := j.enforceSubmitJob(args.PolicyOverride, args.Job.Copy()) diff --git a/nomad/job_endpoint_hook_connect.go b/nomad/job_endpoint_hook_connect.go index b25601d65042..6b8b9269eeb0 100644 --- a/nomad/job_endpoint_hook_connect.go +++ b/nomad/job_endpoint_hook_connect.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" "github.com/pkg/errors" @@ -19,9 +20,9 @@ var ( } } - // connectDriverConfig is the driver configuration used by the injected - // connect proxy sidecar task - connectDriverConfig = func() map[string]interface{} { + // connectSidecarDriverConfig is the driver configuration used by the injected + // connect proxy sidecar task. + connectSidecarDriverConfig = func() map[string]interface{} { return map[string]interface{}{ "image": "${meta.connect.sidecar_image}", "args": []interface{}{ @@ -32,17 +33,51 @@ var ( } } - // connectVersionConstraint is used when building the sidecar task to ensure + // connectGatewayDriverConfig is the Docker driver configuration used by the + // injected connect proxy sidecar task. + // + // A gateway may run in a group with bridge or host networking, and if host + // networking is being used the network_mode driver configuration is set here. + connectGatewayDriverConfig = func(hostNetwork bool) map[string]interface{} { + m := map[string]interface{}{ + "image": "${meta.connect.gateway_image}", + "args": []interface{}{ + "-c", structs.EnvoyBootstrapPath, + "-l", "${meta.connect.log_level}", + "--disable-hot-restart", + }, + } + + if hostNetwork { + m["network_mode"] = "host" + } + + return m + } + + // connectMinimalVersionConstraint is used when building the sidecar task to ensure // the proper Consul version is used that supports the necessary Connect // features. This includes bootstrapping envoy with a unix socket for Consul's // gRPC xDS API. - connectVersionConstraint = func() *structs.Constraint { + connectMinimalVersionConstraint = func() *structs.Constraint { return &structs.Constraint{ LTarget: "${attr.consul.version}", RTarget: ">= 1.6.0-beta1", Operand: structs.ConstraintSemver, } } + + // connectGatewayVersionConstraint is used when building a connect gateway + // task to ensure proper Consul version is used that supports Connect Gateway + // features. This includes making use of Consul Configuration Entries of type + // {ingress,terminating,mesh}-gateway. + connectGatewayVersionConstraint = func() *structs.Constraint { + return &structs.Constraint{ + LTarget: "${attr.consul.version}", + RTarget: ">= 1.8.0", + Operand: structs.ConstraintSemver, + } + } ) // jobConnectHook implements a job Mutating and Validating admission controller @@ -97,7 +132,22 @@ func getSidecarTaskForService(tg *structs.TaskGroup, svc string) *structs.Task { } func isSidecarForService(t *structs.Task, svc string) bool { - return t.Kind == structs.TaskKind(fmt.Sprintf("%s:%s", structs.ConnectProxyPrefix, svc)) + return t.Kind == structs.NewTaskKind(structs.ConnectProxyPrefix, svc) +} + +func hasGatewayTaskForService(tg *structs.TaskGroup, svc string) bool { + for _, t := range tg.Tasks { + switch { + case isIngressGatewayForService(t, svc): + // also terminating and mesh in the future + return true + } + } + return false +} + +func isIngressGatewayForService(t *structs.Task, svc string) bool { + return t.Kind == structs.NewTaskKind(structs.ConnectIngressPrefix, svc) } // getNamedTaskForNativeService retrieves the Task with the name specified in the @@ -123,7 +173,10 @@ func getNamedTaskForNativeService(tg *structs.TaskGroup, serviceName, taskName s // qualify, configure a port for envoy to use to expose their paths. func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error { for _, service := range g.Services { - if service.Connect.HasSidecar() { + switch { + // mutate depending on what the connect block is being used for + + case service.Connect.HasSidecar(): // Check to see if the sidecar task already exists task := getSidecarTaskForService(g, service.Name) @@ -167,7 +220,8 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error { // create a port for the sidecar task's proxy port makePort(fmt.Sprintf("%s-%s", structs.ConnectProxyPrefix, service.Name)) - } else if service.Connect.IsNative() { + + case service.Connect.IsNative(): // find the task backing this connect native service and set the kind nativeTaskName := service.TaskName if t, err := getNamedTaskForNativeService(g, service.Name, nativeTaskName); err != nil { @@ -176,6 +230,23 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error { t.Kind = structs.NewTaskKind(structs.ConnectNativePrefix, service.Name) service.TaskName = t.Name // in case the task was inferred } + + case service.Connect.IsGateway(): + netHost := g.Networks[0].Mode == "host" + if !netHost && service.Connect.Gateway.Ingress != nil { + // Modify the gateway proxy service configuration to automatically + // do the correct envoy bind address plumbing when inside a net + // namespace, but only if things are not explicitly configured. + service.Connect.Gateway.Proxy = gatewayProxyForBridge(service.Connect.Gateway) + } + + // inject the gateway task only if it does not yet already exist + if !hasGatewayTaskForService(g, service.Name) { + // use the default envoy image, for now there is no support for a custom task + task := newConnectGatewayTask(service.Name, netHost) + g.Tasks = append(g.Tasks, task) + task.Canonicalize(job, g) + } } } @@ -184,13 +255,99 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error { return nil } +// gatewayProxyIsDefault returns false if any of these gateway proxy configuration +// have been modified from their default values, indicating the operator wants +// custom behavior. Otherwise, we assume the operator wants Nomad to do the Right +// Thing, setting the configuration automatically. +// +// - envoy_gateway_no_default_bind +// - envoy_gateway_bind_tagged_addresses +// - envoy_gateway_bind_addresses +func gatewayProxyIsDefault(proxy *structs.ConsulGatewayProxy) bool { + if proxy == nil { + return true + } + if !proxy.EnvoyGatewayNoDefaultBind && + !proxy.EnvoyGatewayBindTaggedAddresses && + len(proxy.EnvoyGatewayBindAddresses) == 0 { + return true + } + return false +} + +// gatewayProxyForBridge scans an existing gateway proxy configuration and tweaks +// it given an associated configuration entry so that it works as intended from +// inside a network namespace. +func gatewayProxyForBridge(gateway *structs.ConsulGateway) *structs.ConsulGatewayProxy { + if gateway == nil { + return nil + } + + // operator has supplied custom proxy configuration, just use that without + // modification + if !gatewayProxyIsDefault(gateway.Proxy) { + return gateway.Proxy + } + + // copy over unrelated fields if proxy block exists + proxy := new(structs.ConsulGatewayProxy) + if gateway.Proxy != nil { + proxy.ConnectTimeout = gateway.Proxy.ConnectTimeout + proxy.Config = gateway.Proxy.Config + } + + // magically set the fields where Nomad knows what to do + proxy.EnvoyGatewayNoDefaultBind = true + proxy.EnvoyGatewayBindTaggedAddresses = false + proxy.EnvoyGatewayBindAddresses = gatewayBindAddresses(gateway.Ingress) + + return proxy +} + +func gatewayBindAddresses(ingress *structs.ConsulIngressConfigEntry) map[string]*structs.ConsulGatewayBindAddress { + if ingress == nil || len(ingress.Listeners) == 0 { + return nil + } + + addresses := make(map[string]*structs.ConsulGatewayBindAddress) + for _, listener := range ingress.Listeners { + port := listener.Port + for _, service := range listener.Services { + addresses[service.Name] = &structs.ConsulGatewayBindAddress{ + Address: "0.0.0.0", + Port: port, + } + } + } + return addresses +} + +func newConnectGatewayTask(serviceName string, netHost bool) *structs.Task { + return &structs.Task{ + // Name is used in container name so must start with '[A-Za-z0-9]' + Name: fmt.Sprintf("%s-%s", structs.ConnectIngressPrefix, serviceName), + Kind: structs.NewTaskKind(structs.ConnectIngressPrefix, serviceName), + Driver: "docker", + Config: connectGatewayDriverConfig(netHost), + ShutdownDelay: 5 * time.Second, + LogConfig: &structs.LogConfig{ + MaxFiles: 2, + MaxFileSizeMB: 2, + }, + Resources: connectSidecarResources(), + Constraints: structs.Constraints{ + connectGatewayVersionConstraint(), + }, + } +} + func newConnectTask(serviceName string) *structs.Task { - task := &structs.Task{ + return &structs.Task{ // Name is used in container name so must start with '[A-Za-z0-9]' Name: fmt.Sprintf("%s-%s", structs.ConnectProxyPrefix, serviceName), Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, serviceName), Driver: "docker", - Config: connectDriverConfig(), + Config: connectSidecarDriverConfig(), ShutdownDelay: 5 * time.Second, LogConfig: &structs.LogConfig{ MaxFiles: 2, @@ -202,23 +359,26 @@ func newConnectTask(serviceName string) *structs.Task { Sidecar: true, }, Constraints: structs.Constraints{ - connectVersionConstraint(), + connectMinimalVersionConstraint(), }, } - - return task } func groupConnectValidate(g *structs.TaskGroup) (warnings []error, err error) { for _, s := range g.Services { - if s.Connect.HasSidecar() { + switch { + case s.Connect.HasSidecar(): if err := groupConnectSidecarValidate(g); err != nil { return nil, err } - } else if s.Connect.IsNative() { + case s.Connect.IsNative(): if err := groupConnectNativeValidate(g, s); err != nil { return nil, err } + case s.Connect.IsGateway(): + if err := groupConnectGatewayValidate(g); err != nil { + return nil, err + } } } return nil, nil @@ -243,3 +403,19 @@ func groupConnectNativeValidate(g *structs.TaskGroup, s *structs.Service) error } return nil } + +func groupConnectGatewayValidate(g *structs.TaskGroup) error { + // the group needs to be either bridge or host mode so we know how to configure + // the docker driver config + + if n := len(g.Networks); n != 1 { + return fmt.Errorf("Consul Connect gateways require exactly 1 network, found %d in group %q", n, g.Name) + } + + modes := []string{"bridge", "host"} + if !helper.SliceStringContains(modes, g.Networks[0].Mode) { + return fmt.Errorf(`Consul Connect Gateway service requires Task Group with network mode of type "bridge" or "host"`) + } + + return nil +} diff --git a/nomad/job_endpoint_hook_connect_test.go b/nomad/job_endpoint_hook_connect_test.go index cb9ca321f182..b348213845b9 100644 --- a/nomad/job_endpoint_hook_connect_test.go +++ b/nomad/job_endpoint_hook_connect_test.go @@ -3,7 +3,9 @@ package nomad import ( "fmt" "testing" + "time" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -110,6 +112,33 @@ func TestJobEndpointConnect_groupConnectHook(t *testing.T) { require.Exactly(t, tgOut, job.TaskGroups[0]) } +func TestJobEndpointConnect_groupConnectHook_IngressGateway(t *testing.T) { + t.Parallel() + + // Test that the connect gateway task is inserted if a gateway service exists + // and since this is a bridge network, will rewrite the default gateway proxy + // block with correct configuration. + job := mock.ConnectIngressGatewayJob("bridge", false) + + expTG := job.TaskGroups[0].Copy() + expTG.Tasks = []*structs.Task{ + // inject the gateway task + newConnectGatewayTask(expTG.Services[0].Name, false), + } + expTG.Tasks[0].Canonicalize(job, expTG) + expTG.Networks[0].Canonicalize() + + // rewrite the service gateway proxy configuration + expTG.Services[0].Connect.Gateway.Proxy = gatewayProxyForBridge(expTG.Services[0].Connect.Gateway) + + require.NoError(t, groupConnectHook(job, job.TaskGroups[0])) + require.Exactly(t, expTG, job.TaskGroups[0]) + + // Test that the hook is idempotent + require.NoError(t, groupConnectHook(job, job.TaskGroups[0])) + require.Exactly(t, expTG, job.TaskGroups[0]) +} + // TestJobEndpoint_ConnectInterpolation asserts that when a Connect sidecar // proxy task is being created for a group service with an interpolated name, // the service name is interpolated *before the task is created. @@ -132,6 +161,8 @@ func TestJobEndpointConnect_ConnectInterpolation(t *testing.T) { } func TestJobEndpointConnect_groupConnectSidecarValidate(t *testing.T) { + t.Parallel() + t.Run("sidecar 0 networks", func(t *testing.T) { require.EqualError(t, groupConnectSidecarValidate(&structs.TaskGroup{ Name: "g1", @@ -159,6 +190,8 @@ func TestJobEndpointConnect_groupConnectSidecarValidate(t *testing.T) { } func TestJobEndpointConnect_getNamedTaskForNativeService(t *testing.T) { + t.Parallel() + t.Run("named exists", func(t *testing.T) { task, err := getNamedTaskForNativeService(&structs.TaskGroup{ Name: "g1", @@ -195,3 +228,254 @@ func TestJobEndpointConnect_getNamedTaskForNativeService(t *testing.T) { require.Nil(t, task) }) } + +func TestJobEndpointConnect_groupConnectGatewayValidate(t *testing.T) { + t.Parallel() + + t.Run("no group network", func(t *testing.T) { + err := groupConnectGatewayValidate(&structs.TaskGroup{ + Name: "g1", + Networks: nil, + }) + require.EqualError(t, err, `Consul Connect gateways require exactly 1 network, found 0 in group "g1"`) + }) + + t.Run("bad network mode", func(t *testing.T) { + err := groupConnectGatewayValidate(&structs.TaskGroup{ + Name: "g1", + Networks: structs.Networks{{ + Mode: "", + }}, + }) + require.EqualError(t, err, `Consul Connect Gateway service requires Task Group with network mode of type "bridge" or "host"`) + }) +} + +func TestJobEndpointConnect_newConnectGatewayTask_host(t *testing.T) { + task := newConnectGatewayTask("service1", true) + require.Equal(t, "connect-ingress-service1", task.Name) + require.Equal(t, "connect-ingress:service1", string(task.Kind)) + require.Equal(t, ">= 1.8.0", task.Constraints[0].RTarget) + require.Equal(t, "host", task.Config["network_mode"]) + require.Nil(t, task.Lifecycle) +} + +func TestJobEndpointConnect_newConnectGatewayTask_bridge(t *testing.T) { + task := newConnectGatewayTask("service1", false) + require.NotContains(t, task.Config, "network_mode") +} + +func TestJobEndpointConnect_hasGatewayTaskForService(t *testing.T) { + t.Run("no gateway task", func(t *testing.T) { + result := hasGatewayTaskForService(&structs.TaskGroup{ + Name: "group", + Tasks: []*structs.Task{{ + Name: "task1", + Kind: "", + }}, + }, "my-service") + require.False(t, result) + }) + + t.Run("has gateway task", func(t *testing.T) { + result := hasGatewayTaskForService(&structs.TaskGroup{ + Name: "group", + Tasks: []*structs.Task{{ + Name: "task1", + Kind: "", + }, { + Name: "ingress-gateway-my-service", + Kind: structs.NewTaskKind(structs.ConnectIngressPrefix, "my-service"), + }}, + }, "my-service") + require.True(t, result) + }) +} + +func TestJobEndpointConnect_gatewayProxyIsDefault(t *testing.T) { + t.Run("nil", func(t *testing.T) { + result := gatewayProxyIsDefault(nil) + require.True(t, result) + }) + + t.Run("unrelated fields set", func(t *testing.T) { + result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(2 * time.Second), + Config: map[string]interface{}{"foo": 1}, + }) + require.True(t, result) + }) + + t.Run("no-bind set", func(t *testing.T) { + result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{ + EnvoyGatewayNoDefaultBind: true, + }) + require.False(t, result) + }) + + t.Run("bind-tagged set", func(t *testing.T) { + result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{ + EnvoyGatewayBindTaggedAddresses: true, + }) + require.False(t, result) + }) + + t.Run("bind-addresses set", func(t *testing.T) { + result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{ + EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{ + "listener1": &structs.ConsulGatewayBindAddress{ + Address: "1.1.1.1", + Port: 9000, + }, + }, + }) + require.False(t, result) + }) +} + +func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) { + t.Run("nil", func(t *testing.T) { + result := gatewayBindAddresses(nil) + require.Nil(t, result) + }) + + t.Run("no listeners", func(t *testing.T) { + result := gatewayBindAddresses(&structs.ConsulIngressConfigEntry{Listeners: nil}) + require.Nil(t, result) + }) + + t.Run("simple", func(t *testing.T) { + result := gatewayBindAddresses(&structs.ConsulIngressConfigEntry{ + Listeners: []*structs.ConsulIngressListener{{ + Port: 3000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "service1", + }}, + }}, + }) + require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{ + "service1": &structs.ConsulGatewayBindAddress{ + Address: "0.0.0.0", + Port: 3000, + }, + }, result) + }) + + t.Run("complex", func(t *testing.T) { + result := gatewayBindAddresses(&structs.ConsulIngressConfigEntry{ + Listeners: []*structs.ConsulIngressListener{{ + Port: 3000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "service1", + }, { + Name: "service2", + }}, + }, { + Port: 3001, + Protocol: "http", + Services: []*structs.ConsulIngressService{{ + Name: "service3", + }}, + }}, + }) + require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{ + "service1": &structs.ConsulGatewayBindAddress{ + Address: "0.0.0.0", + Port: 3000, + }, + "service2": &structs.ConsulGatewayBindAddress{ + Address: "0.0.0.0", + Port: 3000, + }, + "service3": &structs.ConsulGatewayBindAddress{ + Address: "0.0.0.0", + Port: 3001, + }, + }, result) + }) +} + +func TestJobEndpointConnect_gatewayProxyForBridge(t *testing.T) { + t.Run("nil", func(t *testing.T) { + result := gatewayProxyForBridge(nil) + require.Nil(t, result) + }) + + t.Run("nil proxy", func(t *testing.T) { + result := gatewayProxyForBridge(&structs.ConsulGateway{ + Ingress: &structs.ConsulIngressConfigEntry{ + Listeners: []*structs.ConsulIngressListener{{ + Port: 3000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "service1", + }}, + }}, + }, + }) + require.Equal(t, &structs.ConsulGatewayProxy{ + EnvoyGatewayNoDefaultBind: true, + EnvoyGatewayBindTaggedAddresses: false, + EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{ + "service1": { + Address: "0.0.0.0", + Port: 3000, + }}, + }, result) + }) + + t.Run("fill in defaults", func(t *testing.T) { + result := gatewayProxyForBridge(&structs.ConsulGateway{ + Proxy: &structs.ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(2 * time.Second), + Config: map[string]interface{}{"foo": 1}, + }, + Ingress: &structs.ConsulIngressConfigEntry{ + Listeners: []*structs.ConsulIngressListener{{ + Port: 3000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "service1", + }}, + }}, + }, + }) + require.Equal(t, &structs.ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(2 * time.Second), + Config: map[string]interface{}{"foo": 1}, + EnvoyGatewayNoDefaultBind: true, + EnvoyGatewayBindTaggedAddresses: false, + EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{ + "service1": { + Address: "0.0.0.0", + Port: 3000, + }}, + }, result) + }) + + t.Run("leave as-is", func(t *testing.T) { + result := gatewayProxyForBridge(&structs.ConsulGateway{ + Proxy: &structs.ConsulGatewayProxy{ + Config: map[string]interface{}{"foo": 1}, + EnvoyGatewayBindTaggedAddresses: true, + }, + Ingress: &structs.ConsulIngressConfigEntry{ + Listeners: []*structs.ConsulIngressListener{{ + Port: 3000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "service1", + }}, + }}, + }, + }) + require.Equal(t, &structs.ConsulGatewayProxy{ + Config: map[string]interface{}{"foo": 1}, + EnvoyGatewayNoDefaultBind: false, + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: nil, + }, result) + }) +} diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 7e72030a4064..970a35b38992 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -237,6 +237,208 @@ func TestJobEndpoint_Register_Connect(t *testing.T) { require.Exactly(sidecarTask, out.TaskGroups[0].Tasks[1]) } +func TestJobEndpoint_Register_ConnectIngressGateway_minimum(t *testing.T) { + t.Parallel() + r := require.New(t) + + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // job contains the minimalist possible gateway service definition + job := mock.ConnectIngressGatewayJob("host", false) + + // Create the register request + req := &structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: job.Namespace, + }, + } + + // Fetch the response + var resp structs.JobRegisterResponse + r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) + r.NotZero(resp.Index) + + // Check for the node in the FSM + state := s1.fsm.State() + ws := memdb.NewWatchSet() + out, err := state.JobByID(ws, job.Namespace, job.ID) + r.NoError(err) + r.NotNil(out) + r.Equal(resp.JobModifyIndex, out.CreateIndex) + + // Check that the gateway task got injected + r.Len(out.TaskGroups[0].Tasks, 1) + task := out.TaskGroups[0].Tasks[0] + r.Equal("connect-ingress-my-ingress-service", task.Name) + r.Equal("connect-ingress:my-ingress-service", string(task.Kind)) + r.Equal("docker", task.Driver) + r.NotNil(task.Config) + + // Check the CE fields got set + service := out.TaskGroups[0].Services[0] + r.Equal(&structs.ConsulIngressConfigEntry{ + TLS: nil, + Listeners: []*structs.ConsulIngressListener{{ + Port: 2000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "service1", + }}, + }}, + }, service.Connect.Gateway.Ingress) + + // Check that round-tripping does not inject a duplicate task + out.Meta["test"] = "abc" + req.Job = out + r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) + r.NotZero(resp.Index) + + // Check for the new node in the fsm + state = s1.fsm.State() + ws = memdb.NewWatchSet() + out, err = state.JobByID(ws, job.Namespace, job.ID) + r.NoError(err) + r.NotNil(out) + r.Equal(resp.JobModifyIndex, out.CreateIndex) + + // Check we did not re-add the task that was added the first time + r.Len(out.TaskGroups[0].Tasks, 1) +} + +func TestJobEndpoint_Register_ConnectIngressGateway_full(t *testing.T) { + t.Parallel() + r := require.New(t) + + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // reconfigure job to fill in all the possible fields + job := mock.ConnectIngressGatewayJob("bridge", false) + job.TaskGroups[0].Services[0].Connect = &structs.ConsulConnect{ + Gateway: &structs.ConsulGateway{ + Proxy: &structs.ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(1 * time.Second), + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{ + "service1": { + Address: "10.0.0.1", + Port: 2001, + }, + "service2": { + Address: "10.0.0.2", + Port: 2002, + }, + }, + EnvoyGatewayNoDefaultBind: true, + Config: map[string]interface{}{ + "foo": 1, + "bar": "baz", + }, + }, + Ingress: &structs.ConsulIngressConfigEntry{ + TLS: &structs.ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*structs.ConsulIngressListener{{ + Port: 3000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "db", + }}, + }, { + Port: 3001, + Protocol: "http", + Services: []*structs.ConsulIngressService{{ + Name: "website", + Hosts: []string{"10.0.1.0", "10.0.1.0:3001"}, + }}, + }}, + }, + }, + } + + // Create the register request + req := &structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: job.Namespace, + }, + } + + // Fetch the response + var resp structs.JobRegisterResponse + r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) + r.NotZero(resp.Index) + + // Check for the node in the FSM + state := s1.fsm.State() + ws := memdb.NewWatchSet() + out, err := state.JobByID(ws, job.Namespace, job.ID) + r.NoError(err) + r.NotNil(out) + r.Equal(resp.JobModifyIndex, out.CreateIndex) + + // Check that the gateway task got injected + r.Len(out.TaskGroups[0].Tasks, 1) + task := out.TaskGroups[0].Tasks[0] + r.Equal("connect-ingress-my-ingress-service", task.Name) + r.Equal("connect-ingress:my-ingress-service", string(task.Kind)) + r.Equal("docker", task.Driver) + r.NotNil(task.Config) + + // Check that the ingress service is all set + service := out.TaskGroups[0].Services[0] + r.Equal("my-ingress-service", service.Name) + r.Equal(&structs.ConsulIngressConfigEntry{ + TLS: &structs.ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*structs.ConsulIngressListener{{ + Port: 3000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "db", + }}, + }, { + Port: 3001, + Protocol: "http", + Services: []*structs.ConsulIngressService{{ + Name: "website", + Hosts: []string{"10.0.1.0", "10.0.1.0:3001"}, + }}, + }}, + }, service.Connect.Gateway.Ingress) + + // Check that round-tripping does not inject a duplicate task + out.Meta["test"] = "abc" + req.Job = out + r.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) + r.NotZero(resp.Index) + + // Check for the new node in the fsm + state = s1.fsm.State() + ws = memdb.NewWatchSet() + out, err = state.JobByID(ws, job.Namespace, job.ID) + r.NoError(err) + r.NotNil(out) + r.Equal(resp.JobModifyIndex, out.CreateIndex) + + // Check we did not re-add the task that was added the first time + r.Len(out.TaskGroups[0].Tasks, 1) +} + func TestJobEndpoint_Register_ConnectExposeCheck(t *testing.T) { t.Parallel() r := require.New(t) @@ -421,7 +623,7 @@ func TestJobEndpoint_Register_ConnectWithSidecarTask(t *testing.T) { require.Equal("test", sidecarTask.Meta["source"]) require.Equal(500, sidecarTask.Resources.CPU) require.Equal(connectSidecarResources().MemoryMB, sidecarTask.Resources.MemoryMB) - cfg := connectDriverConfig() + cfg := connectSidecarDriverConfig() cfg["labels"] = map[string]interface{}{ "foo": "bar", } diff --git a/nomad/leader_test.go b/nomad/leader_test.go index ea31d576a263..ba784684d084 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -651,7 +651,7 @@ func TestLeader_revokeSITokenAccessorsOnRestore(t *testing.T) { defer cleanupS1() testutil.WaitForLeader(t, s1.RPC) - // replace consul ACLs api with a mock for tracking calls + // replace consul ACLs API with a mock for tracking calls in tests var consulACLsAPI mockConsulACLsAPI s1.consulACLs = &consulACLsAPI diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index b4f29dc2f7af..159dc40b2924 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" psstructs "github.com/hashicorp/nomad/plugins/shared/structs" @@ -668,6 +669,56 @@ func ConnectNativeJob(mode string) *structs.Job { return job } +// ConnectIngressGatewayJob creates a structs.Job that contains the definition +// of a Consul Ingress Gateway service. The mode is the name of the network +// mode assumed by the task group. If inject is true, a corresponding Task is +// set on the group's Tasks (i.e. what the job would look like after job mutation). +func ConnectIngressGatewayJob(mode string, inject bool) *structs.Job { + job := Job() + tg := job.TaskGroups[0] + tg.Networks = []*structs.NetworkResource{{ + Mode: mode, + }} + tg.Services = []*structs.Service{{ + Name: "my-ingress-service", + PortLabel: "9999", + Connect: &structs.ConsulConnect{ + Gateway: &structs.ConsulGateway{ + Proxy: &structs.ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(3 * time.Second), + }, + Ingress: &structs.ConsulIngressConfigEntry{ + Listeners: []*structs.ConsulIngressListener{{ + Port: 2000, + Protocol: "tcp", + Services: []*structs.ConsulIngressService{{ + Name: "service1", + }}, + }}, + }, + }, + }, + }} + // some tests need to assume the gateway proxy task has already been injected + if inject { + tg.Tasks = []*structs.Task{{ + Name: fmt.Sprintf("%s-%s", structs.ConnectIngressPrefix, "my-ingress-service"), + Kind: structs.NewTaskKind(structs.ConnectIngressPrefix, "my-ingress-service"), + Driver: "docker", + Config: make(map[string]interface{}), + ShutdownDelay: 5 * time.Second, + LogConfig: &structs.LogConfig{ + MaxFiles: 2, + MaxFileSizeMB: 2, + }, + }} + } else { + // otherwise there are no tasks in the group yet + tg.Tasks = nil + } + return job +} + func BatchJob() *structs.Job { job := &structs.Job{ Region: "global", @@ -932,6 +983,7 @@ func ConnectAlloc() *structs.Allocation { return alloc } +// ConnectNativeAlloc creates an alloc with a connect native task. func ConnectNativeAlloc(mode string) *structs.Allocation { alloc := Alloc() alloc.Job = ConnectNativeJob(mode) @@ -942,6 +994,16 @@ func ConnectNativeAlloc(mode string) *structs.Allocation { return alloc } +func ConnectIngressGatewayAlloc(mode string) *structs.Allocation { + alloc := Alloc() + alloc.Job = ConnectIngressGatewayJob(mode, true) + alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{{ + Mode: mode, + IP: "10.0.0.1", + }} + return alloc +} + func BatchConnectJob() *structs.Job { job := &structs.Job{ Region: "global", diff --git a/nomad/server.go b/nomad/server.go index 396e5a47c57b..d0977c1e7b4a 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -216,6 +216,9 @@ type Server struct { // consulCatalog is used for discovering other Nomad Servers via Consul consulCatalog consul.CatalogAPI + // consulConfigEntries is used for managing Consul Configuration Entries. + consulConfigEntries ConsulConfigsAPI + // consulACLs is used for managing Consul Service Identity tokens. consulACLs ConsulACLsAPI @@ -283,7 +286,7 @@ type endpoints struct { // NewServer is used to construct a new Nomad server from the // configuration, potentially returning an error -func NewServer(config *Config, consulCatalog consul.CatalogAPI, consulACLs consul.ACLsAPI) (*Server, error) { +func NewServer(config *Config, consulCatalog consul.CatalogAPI, consulConfigEntries consul.ConfigAPI, consulACLs consul.ACLsAPI) (*Server, error) { // Check the protocol version if err := config.CheckVersion(); err != nil { return nil, err @@ -362,7 +365,7 @@ func NewServer(config *Config, consulCatalog consul.CatalogAPI, consulACLs consu s.statsFetcher = NewStatsFetcher(s.logger, s.connPool, s.config.Region) // Setup Consul (more) - s.setupConsul(consulACLs) + s.setupConsul(consulConfigEntries, consulACLs) // Setup Vault if err := s.setupVaultClient(); err != nil { @@ -662,6 +665,9 @@ func (s *Server) Shutdown() error { // Stop the Consul ACLs token revocations s.consulACLs.Stop() + // Stop being able to set Configuration Entries + s.consulConfigEntries.Stop() + return nil } @@ -1042,7 +1048,8 @@ func (s *Server) setupNodeDrainer() { } // setupConsul is used to setup Server specific consul components. -func (s *Server) setupConsul(consulACLs consul.ACLsAPI) { +func (s *Server) setupConsul(consulConfigEntries consul.ConfigAPI, consulACLs consul.ACLsAPI) { + s.consulConfigEntries = NewConsulConfigsAPI(consulConfigEntries, s.logger) s.consulACLs = NewConsulACLsAPI(consulACLs, s.logger, s.purgeSITokenAccessors) } diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go index 2f54c930f7b1..95ee44b6969f 100644 --- a/nomad/structs/diff.go +++ b/nomad/structs/diff.go @@ -777,17 +777,344 @@ func connectDiffs(old, new *ConsulConnect, contextual bool) *ObjectDiff { // Diff the primitive fields. diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) - sidecarSvcDiff := connectSidecarServiceDiff( - old.SidecarService, new.SidecarService, contextual) + // Diff the object field SidecarService. + sidecarSvcDiff := connectSidecarServiceDiff(old.SidecarService, new.SidecarService, contextual) if sidecarSvcDiff != nil { diff.Objects = append(diff.Objects, sidecarSvcDiff) } + // Diff the object field SidecarTask. sidecarTaskDiff := sidecarTaskDiff(old.SidecarTask, new.SidecarTask, contextual) if sidecarTaskDiff != nil { diff.Objects = append(diff.Objects, sidecarTaskDiff) } + // Diff the object field ConsulGateway. + gatewayDiff := connectGatewayDiff(old.Gateway, new.Gateway, contextual) + if gatewayDiff != nil { + diff.Objects = append(diff.Objects, gatewayDiff) + } + + return diff +} + +func connectGatewayDiff(prev, next *ConsulGateway, contextual bool) *ObjectDiff { + diff := &ObjectDiff{Type: DiffTypeNone, Name: "Gateway"} + var oldPrimitiveFlat, newPrimitiveFlat map[string]string + + if reflect.DeepEqual(prev, next) { + return nil + } else if prev == nil { + prev = new(ConsulGateway) + diff.Type = DiffTypeAdded + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } else if next == nil { + next = new(ConsulGateway) + diff.Type = DiffTypeDeleted + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + } else { + diff.Type = DiffTypeEdited + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } + + // Diff the primitive fields. + diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) + + // Diff the ConsulGatewayProxy fields. + gatewayProxyDiff := connectGatewayProxyDiff(prev.Proxy, next.Proxy, contextual) + if gatewayProxyDiff != nil { + diff.Objects = append(diff.Objects, gatewayProxyDiff) + } + + // Diff the ConsulGatewayIngress fields. + gatewayIngressDiff := connectGatewayIngressDiff(prev.Ingress, next.Ingress, contextual) + if gatewayIngressDiff != nil { + diff.Objects = append(diff.Objects, gatewayIngressDiff) + } + + return diff +} + +func connectGatewayIngressDiff(prev, next *ConsulIngressConfigEntry, contextual bool) *ObjectDiff { + diff := &ObjectDiff{Type: DiffTypeNone, Name: "Ingress"} + var oldPrimitiveFlat, newPrimitiveFlat map[string]string + + if reflect.DeepEqual(prev, next) { + return nil + } else if prev == nil { + prev = new(ConsulIngressConfigEntry) + diff.Type = DiffTypeAdded + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } else if next == nil { + next = new(ConsulIngressConfigEntry) + diff.Type = DiffTypeDeleted + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + } else { + diff.Type = DiffTypeEdited + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } + + // Diff the primitive fields. + diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) + + // Diff the ConsulGatewayTLSConfig objects. + tlsConfigDiff := connectGatewayTLSConfigDiff(prev.TLS, next.TLS, contextual) + if tlsConfigDiff != nil { + diff.Objects = append(diff.Objects, tlsConfigDiff) + } + + // Diff the Listeners lists. + gatewayIngressListenersDiff := connectGatewayIngressListenersDiff(prev.Listeners, next.Listeners, contextual) + if gatewayIngressListenersDiff != nil { + diff.Objects = append(diff.Objects, gatewayIngressListenersDiff...) + } + + return diff +} + +func connectGatewayTLSConfigDiff(prev, next *ConsulGatewayTLSConfig, contextual bool) *ObjectDiff { + diff := &ObjectDiff{Type: DiffTypeNone, Name: "TLS"} + var oldPrimitiveFlat, newPrimitiveFlat map[string]string + + if reflect.DeepEqual(prev, next) { + return nil + } else if prev == nil { + diff.Type = DiffTypeAdded + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } else if next == nil { + diff.Type = DiffTypeDeleted + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + } else { + diff.Type = DiffTypeEdited + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } + + // Diff the primitive field. + diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) + + return diff +} + +// connectGatewayIngressListenersDiff diffs are a set of listeners keyed by "protocol/port", which is +// a nifty workaround having slices instead of maps. Presumably such a key will be unique, because if +// it is not the config entry is not going to work anyway. +func connectGatewayIngressListenersDiff(prev, next []*ConsulIngressListener, contextual bool) []*ObjectDiff { + // create maps, diff the maps, keys are fields, keys are (port+protocol) + + key := func(l *ConsulIngressListener) string { + return fmt.Sprintf("%s/%d", l.Protocol, l.Port) + } + + prevMap := make(map[string]*ConsulIngressListener, len(prev)) + nextMap := make(map[string]*ConsulIngressListener, len(next)) + + for _, l := range prev { + prevMap[key(l)] = l + } + for _, l := range next { + nextMap[key(l)] = l + } + + var diffs []*ObjectDiff + for k, prevL := range prevMap { + // Diff the same, deleted, and edited + if diff := connectGatewayIngressListenerDiff(prevL, nextMap[k], contextual); diff != nil { + diffs = append(diffs, diff) + } + } + for k, nextL := range nextMap { + // Diff the added + if old, ok := prevMap[k]; !ok { + if diff := connectGatewayIngressListenerDiff(old, nextL, contextual); diff != nil { + diffs = append(diffs, diff) + } + } + } + + sort.Sort(ObjectDiffs(diffs)) + return diffs +} + +func connectGatewayIngressListenerDiff(prev, next *ConsulIngressListener, contextual bool) *ObjectDiff { + diff := &ObjectDiff{Type: DiffTypeNone, Name: "Listener"} + var oldPrimitiveFlat, newPrimitiveFlat map[string]string + + if reflect.DeepEqual(prev, next) { + return nil + } else if prev == nil { + prev = new(ConsulIngressListener) + diff.Type = DiffTypeAdded + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } else if next == nil { + next = new(ConsulIngressListener) + diff.Type = DiffTypeDeleted + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + } else { + diff.Type = DiffTypeEdited + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } + + // Diff the primitive fields. + diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) + + // Diff the Ingress Service objects. + if diffs := connectGatewayIngressServicesDiff(prev.Services, next.Services, contextual); diffs != nil { + diff.Objects = append(diff.Objects, diffs...) + } + + return diff +} + +// connectGatewayIngressServicesDiff diffs are a set of ingress services keyed by their service name, which +// is a workaround for having slices instead of maps. Presumably the service name is a unique key, because if +// no the config entry is not going to make sense anyway. +func connectGatewayIngressServicesDiff(prev, next []*ConsulIngressService, contextual bool) []*ObjectDiff { + + prevMap := make(map[string]*ConsulIngressService, len(prev)) + nextMap := make(map[string]*ConsulIngressService, len(next)) + + for _, s := range prev { + prevMap[s.Name] = s + } + for _, s := range next { + nextMap[s.Name] = s + } + + var diffs []*ObjectDiff + for name, oldIS := range prevMap { + // Diff the same, deleted, and edited + if diff := connectGatewayIngressServiceDiff(oldIS, nextMap[name], contextual); diff != nil { + diffs = append(diffs, diff) + } + } + for name, newIS := range nextMap { + // Diff the added + if old, ok := prevMap[name]; !ok { + if diff := connectGatewayIngressServiceDiff(old, newIS, contextual); diff != nil { + diffs = append(diffs, diff) + } + } + } + + sort.Sort(ObjectDiffs(diffs)) + return diffs +} + +func connectGatewayIngressServiceDiff(prev, next *ConsulIngressService, contextual bool) *ObjectDiff { + diff := &ObjectDiff{Type: DiffTypeNone, Name: "ConsulIngressService"} + var oldPrimitiveFlat, newPrimitiveFlat map[string]string + + if reflect.DeepEqual(prev, next) { + return nil + } else if prev == nil { + prev = new(ConsulIngressService) + diff.Type = DiffTypeAdded + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } else if next == nil { + next = new(ConsulIngressService) + diff.Type = DiffTypeDeleted + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + } else { + diff.Type = DiffTypeEdited + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } + + // Diff the primitive fields. + diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) + + // Diff the hosts. + if hDiffs := stringSetDiff(prev.Hosts, next.Hosts, "Hosts", contextual); hDiffs != nil { + diff.Objects = append(diff.Objects, hDiffs) + } + + return diff +} + +func connectGatewayProxyDiff(prev, next *ConsulGatewayProxy, contextual bool) *ObjectDiff { + diff := &ObjectDiff{Type: DiffTypeNone, Name: "Proxy"} + var oldPrimitiveFlat, newPrimitiveFlat map[string]string + + if reflect.DeepEqual(prev, next) { + return nil + } else if prev == nil { + prev = new(ConsulGatewayProxy) + diff.Type = DiffTypeAdded + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } else if next == nil { + next = new(ConsulGatewayProxy) + diff.Type = DiffTypeDeleted + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + } else { + diff.Type = DiffTypeEdited + oldPrimitiveFlat = flatmap.Flatten(prev, nil, true) + newPrimitiveFlat = flatmap.Flatten(next, nil, true) + } + + // Diff the ConnectTimeout field (dur ptr). (i.e. convert to string for comparison) + if oldPrimitiveFlat != nil && newPrimitiveFlat != nil { + if prev.ConnectTimeout == nil { + oldPrimitiveFlat["ConnectTimeout"] = "" + } else { + oldPrimitiveFlat["ConnectTimeout"] = fmt.Sprintf("%s", *prev.ConnectTimeout) + } + if next.ConnectTimeout == nil { + newPrimitiveFlat["ConnectTimeout"] = "" + } else { + newPrimitiveFlat["ConnectTimeout"] = fmt.Sprintf("%s", *next.ConnectTimeout) + } + } + + // Diff the primitive fields. + diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) + + // Diff the EnvoyGatewayBindAddresses map. + bindAddrsDiff := connectGatewayProxyEnvoyBindAddrsDiff(prev.EnvoyGatewayBindAddresses, next.EnvoyGatewayBindAddresses, contextual) + if bindAddrsDiff != nil { + diff.Objects = append(diff.Objects, bindAddrsDiff) + } + + // Diff the opaque Config map. + if cDiff := configDiff(prev.Config, next.Config, contextual); cDiff != nil { + diff.Objects = append(diff.Objects, cDiff) + } + + return diff +} + +// connectGatewayProxyEnvoyBindAddrsDiff returns the diff of two maps. If contextual +// diff is enabled, all fields will be returned, even if no diff occurred. +func connectGatewayProxyEnvoyBindAddrsDiff(prev, next map[string]*ConsulGatewayBindAddress, contextual bool) *ObjectDiff { + diff := &ObjectDiff{Type: DiffTypeNone, Name: "EnvoyGatewayBindAddresses"} + if reflect.DeepEqual(prev, next) { + return nil + } else if len(prev) == 0 { + diff.Type = DiffTypeAdded + } else if len(next) == 0 { + diff.Type = DiffTypeDeleted + } else { + diff.Type = DiffTypeEdited + } + + // convert to string representation + prevMap := make(map[string]string, len(prev)) + nextMap := make(map[string]string, len(next)) + + for k, v := range prev { + prevMap[k] = fmt.Sprintf("%s:%d", v.Address, v.Port) + } + + for k, v := range next { + nextMap[k] = fmt.Sprintf("%s:%d", v.Address, v.Port) + } + + oldPrimitiveFlat := flatmap.Flatten(prevMap, nil, false) + newPrimitiveFlat := flatmap.Flatten(nextMap, nil, false) + diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) return diff } diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index d6e7b5c00547..7ab44f0e059f 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -2618,6 +2618,34 @@ func TestTaskGroupDiff(t *testing.T) { "foo": "baz", }, }, + Gateway: &ConsulGateway{ + Proxy: &ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(1 * time.Second), + EnvoyGatewayBindTaggedAddresses: false, + EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{ + "service1": { + Address: "10.0.0.1", + Port: 2001, + }, + }, + EnvoyGatewayNoDefaultBind: false, + Config: map[string]interface{}{ + "foo": 1, + }, + }, + Ingress: &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: false, + }, + Listeners: []*ConsulIngressListener{{ + Port: 3001, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "listener1", + }}, + }}, + }, + }, }, }, }, @@ -2664,6 +2692,35 @@ func TestTaskGroupDiff(t *testing.T) { }, }, }, + Gateway: &ConsulGateway{ + Proxy: &ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(2 * time.Second), + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{ + "service1": { + Address: "10.0.0.2", + Port: 2002, + }, + }, + EnvoyGatewayNoDefaultBind: true, + Config: map[string]interface{}{ + "foo": 2, + }, + }, + Ingress: &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*ConsulIngressListener{{ + Port: 3002, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "listener2", + Hosts: []string{"127.0.0.1", "127.0.0.1:3002"}, + }}, + }}, + }, + }, }, }, }, @@ -2836,7 +2893,6 @@ func TestTaskGroupDiff(t *testing.T) { }, }, }, - { Type: DiffTypeEdited, Name: "ConsulConnect", @@ -2952,6 +3008,164 @@ func TestTaskGroupDiff(t *testing.T) { }, }, }, + { + Type: DiffTypeEdited, + Name: "Gateway", + Objects: []*ObjectDiff{ + { + Type: DiffTypeEdited, + Name: "Proxy", + Fields: []*FieldDiff{ + { + Type: DiffTypeEdited, + Name: "ConnectTimeout", + Old: "1s", + New: "2s", + }, + { + Type: DiffTypeEdited, + Name: "EnvoyGatewayBindTaggedAddresses", + Old: "false", + New: "true", + }, + { + Type: DiffTypeEdited, + Name: "EnvoyGatewayNoDefaultBind", + Old: "false", + New: "true", + }, + }, + Objects: []*ObjectDiff{ + { + Type: DiffTypeEdited, + Name: "EnvoyGatewayBindAddresses", + Fields: []*FieldDiff{ + { + Type: DiffTypeEdited, + Name: "service1", + Old: "10.0.0.1:2001", + New: "10.0.0.2:2002", + }, + }, + }, + { + Type: DiffTypeEdited, + Name: "Config", + Fields: []*FieldDiff{ + { + Type: DiffTypeEdited, + Name: "foo", + Old: "1", + New: "2", + }, + }, + }, + }, + }, + { + Type: DiffTypeEdited, + Name: "Ingress", + Objects: []*ObjectDiff{ + { + Type: DiffTypeEdited, + Name: "TLS", + Fields: []*FieldDiff{ + { + Type: DiffTypeEdited, + Name: "Enabled", + Old: "false", + New: "true", + }, + }, + }, + { + Type: DiffTypeAdded, + Name: "Listener", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "Port", + Old: "", + New: "3002", + }, + { + Type: DiffTypeAdded, + Name: "Protocol", + Old: "", + New: "http", + }, + }, + Objects: []*ObjectDiff{ + { + Type: DiffTypeAdded, + Name: "ConsulIngressService", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "Name", + Old: "", + New: "listener2", + }, + }, + Objects: []*ObjectDiff{ + { + Type: DiffTypeAdded, + Name: "Hosts", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "Hosts", + Old: "", + New: "127.0.0.1", + }, + { + Type: DiffTypeAdded, + Name: "Hosts", + Old: "", + New: "127.0.0.1:3002", + }, + }, + }, + }, + }, + }, + }, + { + Type: DiffTypeDeleted, + Name: "Listener", + Fields: []*FieldDiff{ + { + Type: DiffTypeDeleted, + Name: "Port", + Old: "3001", + New: "", + }, + { + Type: DiffTypeDeleted, + Name: "Protocol", + Old: "tcp", + New: "", + }, + }, + Objects: []*ObjectDiff{ + { + Type: DiffTypeDeleted, + Name: "ConsulIngressService", + Fields: []*FieldDiff{ + { + Type: DiffTypeDeleted, + Name: "Name", + Old: "listener1", + New: "", + }, + }, + }, + }, + }, + }, + }, + }, + }, }, }, }, diff --git a/nomad/structs/services.go b/nomad/structs/services.go index 06f446c94e26..504cc8dce4c5 100644 --- a/nomad/structs/services.go +++ b/nomad/structs/services.go @@ -677,6 +677,9 @@ type ConsulConnect struct { // SidecarTask is non-nil if sidecar overrides are set SidecarTask *SidecarTask + + // Gateway is a Consul Connect Gateway Proxy. + Gateway *ConsulGateway } // Copy the stanza recursively. Returns nil if nil. @@ -689,6 +692,7 @@ func (c *ConsulConnect) Copy() *ConsulConnect { Native: c.Native, SidecarService: c.SidecarService.Copy(), SidecarTask: c.SidecarTask.Copy(), + Gateway: c.Gateway.Copy(), } } @@ -702,32 +706,70 @@ func (c *ConsulConnect) Equals(o *ConsulConnect) bool { return false } - return c.SidecarService.Equals(o.SidecarService) + if !c.SidecarService.Equals(o.SidecarService) { + return false + } + + // todo(shoenig) task has never been compared, should it be? + + if !c.Gateway.Equals(o.Gateway) { + return false + } + + return true } -// HasSidecar checks if a sidecar task is needed +// HasSidecar checks if a sidecar task is configured. func (c *ConsulConnect) HasSidecar() bool { return c != nil && c.SidecarService != nil } +// IsNative checks if the service is connect native. func (c *ConsulConnect) IsNative() bool { return c != nil && c.Native } -// Validate that the Connect stanza has exactly one of Native or sidecar. +func (c *ConsulConnect) IsGateway() bool { + return c != nil && c.Gateway != nil +} + +// Validate that the Connect block represents exactly one of: +// - Connect non-native service sidecar proxy +// - Connect native service +// - Connect gateway (any type) func (c *ConsulConnect) Validate() error { if c == nil { return nil } - if c.IsNative() && c.HasSidecar() { - return fmt.Errorf("Consul Connect must be native or use a sidecar service; not both") + // Count the number of things actually configured. If that number is not 1, + // the config is not valid. + count := 0 + + if c.HasSidecar() { + count++ + } + + if c.IsNative() { + count++ + } + + if c.IsGateway() { + count++ + } + + if count != 1 { + return fmt.Errorf("Consul Connect must be exclusively native, make use of a sidecar, or represent a Gateway") } - if !c.IsNative() && !c.HasSidecar() { - return fmt.Errorf("Consul Connect must be native or use a sidecar service") + if c.IsGateway() { + if err := c.Gateway.Validate(); err != nil { + return err + } } + // The Native and Sidecar cases are validated up at the service level. + return nil } @@ -981,6 +1023,15 @@ func (p *ConsulProxy) Copy() *ConsulProxy { return newP } +// opaqueMapsEqual compares map[string]interface{} commonly used for opaque +// config blocks. Interprets nil and {} as the same. +func opaqueMapsEqual(a, b map[string]interface{}) bool { + if len(a) == 0 && len(b) == 0 { + return true + } + return reflect.DeepEqual(a, b) +} + // Equals returns true if the structs are recursively equal. func (p *ConsulProxy) Equals(o *ConsulProxy) bool { if p == nil || o == nil { @@ -1003,11 +1054,8 @@ func (p *ConsulProxy) Equals(o *ConsulProxy) bool { return false } - // Avoid nil vs {} differences - if len(p.Config) != 0 && len(o.Config) != 0 { - if !reflect.DeepEqual(p.Config, o.Config) { - return false - } + if !opaqueMapsEqual(p.Config, o.Config) { + return false } return true @@ -1112,3 +1160,452 @@ func (e *ConsulExposeConfig) Equals(o *ConsulExposeConfig) bool { } return exposePathsEqual(e.Paths, o.Paths) } + +// ConsulGateway is used to configure one of the Consul Connect Gateway types. +type ConsulGateway struct { + // Proxy is used to configure the Envoy instance acting as the gateway. + Proxy *ConsulGatewayProxy + + // Ingress represents the Consul Configuration Entry for an Ingress Gateway. + Ingress *ConsulIngressConfigEntry + + // Terminating is not yet supported. + // Terminating *ConsulTerminatingConfigEntry + + // Mesh is not yet supported. + // Mesh *ConsulMeshConfigEntry +} + +func (g *ConsulGateway) Copy() *ConsulGateway { + if g == nil { + return nil + } + + return &ConsulGateway{ + Proxy: g.Proxy.Copy(), + Ingress: g.Ingress.Copy(), + } +} + +func (g *ConsulGateway) Equals(o *ConsulGateway) bool { + if g == nil || o == nil { + return g == o + } + + if !g.Proxy.Equals(o.Proxy) { + return false + } + + if !g.Ingress.Equals(o.Ingress) { + return false + } + + return true +} + +func (g *ConsulGateway) Validate() error { + if g == nil { + return nil + } + + if g.Proxy != nil { + if err := g.Proxy.Validate(); err != nil { + return err + } + } + + // eventually one of: ingress, terminating, mesh + if g.Ingress != nil { + return g.Ingress.Validate() + } + + return fmt.Errorf("Consul Gateway ingress Configuration Entry must be set") +} + +// ConsulGatewayBindAddress is equivalent to Consul's api/catalog.go ServiceAddress +// struct, as this is used to encode values to pass along to Envoy (i.e. via +// JSON encoding). +type ConsulGatewayBindAddress struct { + Address string + Port int +} + +func (a *ConsulGatewayBindAddress) Equals(o *ConsulGatewayBindAddress) bool { + if a == nil || o == nil { + return a == o + } + + if a.Address != o.Address { + return false + } + + if a.Port != o.Port { + return false + } + + return true +} + +func (a *ConsulGatewayBindAddress) Copy() *ConsulGatewayBindAddress { + if a == nil { + return nil + } + + return &ConsulGatewayBindAddress{ + Address: a.Address, + Port: a.Port, + } +} + +func (a *ConsulGatewayBindAddress) Validate() error { + if a == nil { + return nil + } + + if a.Address == "" { + return fmt.Errorf("Consul Gateway Bind Address must be set") + } + + if a.Port <= 0 { + return fmt.Errorf("Consul Gateway Bind Address must set valid Port") + } + + return nil +} + +// ConsulGatewayProxy is used to tune parameters of the proxy instance acting as +// one of the forms of Connect gateways that Consul supports. +// +// https://www.consul.io/docs/connect/proxies/envoy#gateway-options +type ConsulGatewayProxy struct { + ConnectTimeout *time.Duration + EnvoyGatewayBindTaggedAddresses bool + EnvoyGatewayBindAddresses map[string]*ConsulGatewayBindAddress + EnvoyGatewayNoDefaultBind bool + Config map[string]interface{} +} + +func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy { + if p == nil { + return nil + } + + bindAddresses := make(map[string]*ConsulGatewayBindAddress, len(p.EnvoyGatewayBindAddresses)) + for k, v := range p.EnvoyGatewayBindAddresses { + bindAddresses[k] = v.Copy() + } + + return &ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(*p.ConnectTimeout), + EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses, + EnvoyGatewayBindAddresses: bindAddresses, + EnvoyGatewayNoDefaultBind: p.EnvoyGatewayNoDefaultBind, + Config: helper.CopyMapStringInterface(p.Config), + } +} + +func (p *ConsulGatewayProxy) equalBindAddresses(o map[string]*ConsulGatewayBindAddress) bool { + if len(p.EnvoyGatewayBindAddresses) != len(o) { + return false + } + + for listener, addr := range p.EnvoyGatewayBindAddresses { + if !o[listener].Equals(addr) { + return false + } + } + + return true +} + +func (p *ConsulGatewayProxy) Equals(o *ConsulGatewayProxy) bool { + if p == nil || o == nil { + return p == o + } + + if !helper.CompareTimePtrs(p.ConnectTimeout, o.ConnectTimeout) { + return false + } + + if p.EnvoyGatewayBindTaggedAddresses != o.EnvoyGatewayBindTaggedAddresses { + return false + } + + if !p.equalBindAddresses(o.EnvoyGatewayBindAddresses) { + return false + } + + if p.EnvoyGatewayNoDefaultBind != o.EnvoyGatewayNoDefaultBind { + return false + } + + if !opaqueMapsEqual(p.Config, o.Config) { + return false + } + + return true +} + +func (p *ConsulGatewayProxy) Validate() error { + if p == nil { + return nil + } + + if p.ConnectTimeout == nil { + return fmt.Errorf("Consul Gateway Proxy connection_timeout must be set") + } + + for _, bindAddr := range p.EnvoyGatewayBindAddresses { + if err := bindAddr.Validate(); err != nil { + return err + } + } + + return nil +} + +// ConsulGatewayTLSConfig is used to configure TLS for a gateway. +type ConsulGatewayTLSConfig struct { + Enabled bool +} + +func (c *ConsulGatewayTLSConfig) Copy() *ConsulGatewayTLSConfig { + if c == nil { + return nil + } + + return &ConsulGatewayTLSConfig{ + Enabled: c.Enabled, + } +} + +func (c *ConsulGatewayTLSConfig) Equals(o *ConsulGatewayTLSConfig) bool { + if c == nil || o == nil { + return c == o + } + + return c.Enabled == o.Enabled +} + +// ConsulIngressService is used to configure a service fronted by the ingress gateway. +type ConsulIngressService struct { + // Namespace is not yet supported. + // Namespace string + + Name string + + Hosts []string +} + +func (s *ConsulIngressService) Copy() *ConsulIngressService { + if s == nil { + return nil + } + + var hosts []string = nil + if n := len(s.Hosts); n > 0 { + hosts = make([]string, n) + copy(hosts, s.Hosts) + } + + return &ConsulIngressService{ + Name: s.Name, + Hosts: hosts, + } +} + +func (s *ConsulIngressService) Equals(o *ConsulIngressService) bool { + if s == nil || o == nil { + return s == o + } + + if s.Name != o.Name { + return false + } + + return helper.CompareSliceSetString(s.Hosts, o.Hosts) +} + +func (s *ConsulIngressService) Validate(isHTTP bool) error { + if s == nil { + return nil + } + + if s.Name == "" { + return fmt.Errorf("Consul Ingress Service requires a name") + } + + if isHTTP && len(s.Hosts) == 0 { + return fmt.Errorf("Consul Ingress Service requires one or more hosts when using HTTP protocol") + } else if !isHTTP && len(s.Hosts) > 0 { + return fmt.Errorf("Consul Ingress Service supports hosts only when using HTTP protocol") + } + + return nil +} + +// ConsulIngressListener is used to configure a listener on a Consul Ingress +// Gateway. +type ConsulIngressListener struct { + Port int + Protocol string + Services []*ConsulIngressService +} + +func (l *ConsulIngressListener) Copy() *ConsulIngressListener { + if l == nil { + return nil + } + + var services []*ConsulIngressService = nil + if n := len(l.Services); n > 0 { + services = make([]*ConsulIngressService, n) + for i := 0; i < n; i++ { + services[i] = l.Services[i].Copy() + } + } + + return &ConsulIngressListener{ + Port: l.Port, + Protocol: l.Protocol, + Services: services, + } +} + +func (l *ConsulIngressListener) Equals(o *ConsulIngressListener) bool { + if l == nil || o == nil { + return l == o + } + + if l.Port != o.Port { + return false + } + + if l.Protocol != o.Protocol { + return false + } + + return ingressServicesEqual(l.Services, o.Services) +} + +func (l *ConsulIngressListener) Validate() error { + if l == nil { + return nil + } + + if l.Port <= 0 { + return fmt.Errorf("Consul Ingress Listener requires valid Port") + } + + protocols := []string{"http", "tcp"} + if !helper.SliceStringContains(protocols, l.Protocol) { + return fmt.Errorf(`Consul Ingress Listener requires protocol of "http" or "tcp", got %q`, l.Protocol) + } + + if len(l.Services) == 0 { + return fmt.Errorf("Consul Ingress Listener requires one or more services") + } + + for _, service := range l.Services { + if err := service.Validate(l.Protocol == "http"); err != nil { + return err + } + } + + return nil +} + +func ingressServicesEqual(servicesA, servicesB []*ConsulIngressService) bool { + if len(servicesA) != len(servicesB) { + return false + } + +COMPARE: // order does not matter + for _, serviceA := range servicesA { + for _, serviceB := range servicesB { + if serviceA.Equals(serviceB) { + continue COMPARE + } + } + return false + } + return true +} + +// ConsulIngressConfigEntry represents the Consul Configuration Entry type for +// an Ingress Gateway. +// +// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields +type ConsulIngressConfigEntry struct { + // Namespace is not yet supported. + // Namespace string + + TLS *ConsulGatewayTLSConfig + Listeners []*ConsulIngressListener +} + +func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry { + if e == nil { + return nil + } + + var listeners []*ConsulIngressListener = nil + if n := len(e.Listeners); n > 0 { + listeners = make([]*ConsulIngressListener, n) + for i := 0; i < n; i++ { + listeners[i] = e.Listeners[i].Copy() + } + } + + return &ConsulIngressConfigEntry{ + TLS: e.TLS.Copy(), + Listeners: listeners, + } +} + +func (e *ConsulIngressConfigEntry) Equals(o *ConsulIngressConfigEntry) bool { + if e == nil || o == nil { + return e == o + } + + if !e.TLS.Equals(o.TLS) { + return false + } + + return ingressListenersEqual(e.Listeners, o.Listeners) +} + +func (e *ConsulIngressConfigEntry) Validate() error { + if e == nil { + return nil + } + + if len(e.Listeners) == 0 { + return fmt.Errorf("Consul Ingress Gateway requires at least one listener") + } + + for _, listener := range e.Listeners { + if err := listener.Validate(); err != nil { + return err + } + } + + return nil +} + +func ingressListenersEqual(listenersA, listenersB []*ConsulIngressListener) bool { + if len(listenersA) != len(listenersB) { + return false + } + +COMPARE: // order does not matter + for _, listenerA := range listenersA { + for _, listenerB := range listenersB { + if listenerA.Equals(listenerB) { + continue COMPARE + } + } + return false + } + return true +} diff --git a/nomad/structs/services_test.go b/nomad/structs/services_test.go index 077e260c27ae..45b29b54fa9a 100644 --- a/nomad/structs/services_test.go +++ b/nomad/structs/services_test.go @@ -474,3 +474,422 @@ func TestConsulSidecarService_Copy(t *testing.T) { }, result) }) } + +var ( + consulIngressGateway1 = &ConsulGateway{ + Proxy: &ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(1 * time.Second), + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{ + "listener1": &ConsulGatewayBindAddress{Address: "10.0.0.1", Port: 2001}, + "listener2": &ConsulGatewayBindAddress{Address: "10.0.0.1", Port: 2002}, + }, + EnvoyGatewayNoDefaultBind: true, + Config: map[string]interface{}{ + "foo": 1, + }, + }, + Ingress: &ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*ConsulIngressListener{{ + Port: 3000, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"10.0.0.1", "10.0.0.1:3000"}, + }, { + Name: "service2", + Hosts: []string{"10.0.0.2", "10.0.0.2:3000"}, + }}, + }, { + Port: 3001, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "service3", + }}, + }}, + }, + } +) + +func TestConsulGateway_Copy(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + g := (*ConsulGateway)(nil) + result := g.Copy() + require.Nil(t, result) + }) + + t.Run("as ingress", func(t *testing.T) { + result := consulIngressGateway1.Copy() + require.Equal(t, consulIngressGateway1, result) + require.True(t, result.Equals(consulIngressGateway1)) + require.True(t, consulIngressGateway1.Equals(result)) + }) +} + +func TestConsulGateway_Equals_ingress(t *testing.T) { + t.Parallel() + + t.Run("nil", func(t *testing.T) { + a := (*ConsulGateway)(nil) + b := (*ConsulGateway)(nil) + require.True(t, a.Equals(b)) + require.False(t, a.Equals(consulIngressGateway1)) + require.False(t, consulIngressGateway1.Equals(a)) + }) + + original := consulIngressGateway1.Copy() + + type gway = ConsulGateway + type tweaker = func(g *gway) + + t.Run("reflexive", func(t *testing.T) { + require.True(t, original.Equals(original)) + }) + + try := func(t *testing.T, tweak tweaker) { + modifiable := original.Copy() + tweak(modifiable) + require.False(t, original.Equals(modifiable)) + require.False(t, modifiable.Equals(original)) + require.True(t, modifiable.Equals(modifiable)) + } + + // proxy stanza equality checks + + t.Run("mod gateway timeout", func(t *testing.T) { + try(t, func(g *gway) { g.Proxy.ConnectTimeout = helper.TimeToPtr(9 * time.Second) }) + }) + + t.Run("mod gateway envoy_gateway_bind_tagged_addresses", func(t *testing.T) { + try(t, func(g *gway) { g.Proxy.EnvoyGatewayBindTaggedAddresses = false }) + }) + + t.Run("mod gateway envoy_gateway_bind_addresses", func(t *testing.T) { + try(t, func(g *gway) { + g.Proxy.EnvoyGatewayBindAddresses = map[string]*ConsulGatewayBindAddress{ + "listener3": &ConsulGatewayBindAddress{Address: "9.9.9.9", Port: 9999}, + } + }) + }) + + t.Run("mod gateway envoy_gateway_no_default_bind", func(t *testing.T) { + try(t, func(g *gway) { g.Proxy.EnvoyGatewayNoDefaultBind = false }) + }) + + t.Run("mod gateway config", func(t *testing.T) { + try(t, func(g *gway) { + g.Proxy.Config = map[string]interface{}{ + "foo": 2, + } + }) + }) + + // ingress config entry equality checks + + t.Run("mod ingress tls", func(t *testing.T) { + try(t, func(g *gway) { g.Ingress.TLS = nil }) + try(t, func(g *gway) { g.Ingress.TLS.Enabled = false }) + }) + + t.Run("mod ingress listeners count", func(t *testing.T) { + try(t, func(g *gway) { g.Ingress.Listeners = g.Ingress.Listeners[:1] }) + }) + + t.Run("mod ingress listeners port", func(t *testing.T) { + try(t, func(g *gway) { g.Ingress.Listeners[0].Port = 7777 }) + }) + + t.Run("mod ingress listeners protocol", func(t *testing.T) { + try(t, func(g *gway) { g.Ingress.Listeners[0].Protocol = "tcp" }) + }) + + t.Run("mod ingress listeners services count", func(t *testing.T) { + try(t, func(g *gway) { g.Ingress.Listeners[0].Services = g.Ingress.Listeners[0].Services[:1] }) + }) + + t.Run("mod ingress listeners services name", func(t *testing.T) { + try(t, func(g *gway) { g.Ingress.Listeners[0].Services[0].Name = "serviceX" }) + }) + + t.Run("mod ingress listeners services hosts count", func(t *testing.T) { + try(t, func(g *gway) { g.Ingress.Listeners[0].Services[0].Hosts = g.Ingress.Listeners[0].Services[0].Hosts[:1] }) + }) + + t.Run("mod ingress listeners services hosts content", func(t *testing.T) { + try(t, func(g *gway) { g.Ingress.Listeners[0].Services[0].Hosts[0] = "255.255.255.255" }) + }) +} + +func TestConsulGateway_ingressServicesEqual(t *testing.T) { + igs1 := []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"host1", "host2"}, + }, { + Name: "service2", + Hosts: []string{"host3"}, + }} + + require.False(t, ingressServicesEqual(igs1, nil)) + + reversed := []*ConsulIngressService{ + igs1[1], igs1[0], // services reversed + } + + require.True(t, ingressServicesEqual(igs1, reversed)) + + hostOrder := []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"host2", "host1"}, // hosts reversed + }, { + Name: "service2", + Hosts: []string{"host3"}, + }} + + require.True(t, ingressServicesEqual(igs1, hostOrder)) +} + +func TestConsulGateway_ingressListenersEqual(t *testing.T) { + ils1 := []*ConsulIngressListener{{ + Port: 2000, + Protocol: "http", + Services: []*ConsulIngressService{{ + Name: "service1", + Hosts: []string{"host1", "host2"}, + }}, + }, { + Port: 2001, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "service2", + }}, + }} + + require.False(t, ingressListenersEqual(ils1, nil)) + + reversed := []*ConsulIngressListener{ + ils1[1], ils1[0], + } + + require.True(t, ingressListenersEqual(ils1, reversed)) +} + +func TestConsulGateway_Validate(t *testing.T) { + t.Run("bad proxy", func(t *testing.T) { + err := (&ConsulGateway{ + Proxy: &ConsulGatewayProxy{ + ConnectTimeout: nil, + }, + Ingress: nil, + }).Validate() + require.EqualError(t, err, "Consul Gateway Proxy connection_timeout must be set") + }) + + t.Run("bad ingress config entry", func(t *testing.T) { + err := (&ConsulGateway{ + Ingress: &ConsulIngressConfigEntry{ + Listeners: nil, + }, + }).Validate() + require.EqualError(t, err, "Consul Ingress Gateway requires at least one listener") + }) +} + +func TestConsulGatewayBindAddress_Validate(t *testing.T) { + t.Run("no address", func(t *testing.T) { + err := (&ConsulGatewayBindAddress{ + Address: "", + Port: 2000, + }).Validate() + require.EqualError(t, err, "Consul Gateway Bind Address must be set") + }) + + t.Run("invalid port", func(t *testing.T) { + err := (&ConsulGatewayBindAddress{ + Address: "10.0.0.1", + Port: 0, + }).Validate() + require.EqualError(t, err, "Consul Gateway Bind Address must set valid Port") + }) + + t.Run("ok", func(t *testing.T) { + err := (&ConsulGatewayBindAddress{ + Address: "10.0.0.1", + Port: 2000, + }).Validate() + require.NoError(t, err) + }) +} + +func TestConsulGatewayProxy_Validate(t *testing.T) { + t.Run("no timeout", func(t *testing.T) { + err := (&ConsulGatewayProxy{ + ConnectTimeout: nil, + }).Validate() + require.EqualError(t, err, "Consul Gateway Proxy connection_timeout must be set") + }) + + t.Run("invalid bind address", func(t *testing.T) { + err := (&ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(1 * time.Second), + EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{ + "service1": { + Address: "10.0.0.1", + Port: 0, + }}, + }).Validate() + require.EqualError(t, err, "Consul Gateway Bind Address must set valid Port") + }) + + t.Run("ok with nothing set", func(t *testing.T) { + err := (&ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(1 * time.Second), + }).Validate() + require.NoError(t, err) + }) + + t.Run("ok with everything set", func(t *testing.T) { + err := (&ConsulGatewayProxy{ + ConnectTimeout: helper.TimeToPtr(1 * time.Second), + EnvoyGatewayBindAddresses: map[string]*ConsulGatewayBindAddress{ + "service1": { + Address: "10.0.0.1", + Port: 2000, + }}, + EnvoyGatewayBindTaggedAddresses: true, + EnvoyGatewayNoDefaultBind: true, + }).Validate() + require.NoError(t, err) + }) +} + +func TestConsulIngressService_Validate(t *testing.T) { + t.Run("invalid name", func(t *testing.T) { + err := (&ConsulIngressService{ + Name: "", + }).Validate(true) + require.EqualError(t, err, "Consul Ingress Service requires a name") + }) + + t.Run("http missing hosts", func(t *testing.T) { + err := (&ConsulIngressService{ + Name: "service1", + }).Validate(true) + require.EqualError(t, err, "Consul Ingress Service requires one or more hosts when using HTTP protocol") + }) + + t.Run("tcp extraneous hosts", func(t *testing.T) { + err := (&ConsulIngressService{ + Name: "service1", + Hosts: []string{"host1"}, + }).Validate(false) + require.EqualError(t, err, "Consul Ingress Service supports hosts only when using HTTP protocol") + }) + + t.Run("ok tcp", func(t *testing.T) { + err := (&ConsulIngressService{ + Name: "service1", + }).Validate(false) + require.NoError(t, err) + }) + + t.Run("ok http", func(t *testing.T) { + err := (&ConsulIngressService{ + Name: "service1", + Hosts: []string{"host1"}, + }).Validate(true) + require.NoError(t, err) + }) +} + +func TestConsulIngressListener_Validate(t *testing.T) { + t.Run("invalid port", func(t *testing.T) { + err := (&ConsulIngressListener{ + Port: 0, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "service1", + }}, + }).Validate() + require.EqualError(t, err, "Consul Ingress Listener requires valid Port") + }) + + t.Run("invalid protocol", func(t *testing.T) { + err := (&ConsulIngressListener{ + Port: 2000, + Protocol: "gopher", + Services: []*ConsulIngressService{{ + Name: "service1", + }}, + }).Validate() + require.EqualError(t, err, `Consul Ingress Listener requires protocol of "http" or "tcp", got "gopher"`) + }) + + t.Run("no services", func(t *testing.T) { + err := (&ConsulIngressListener{ + Port: 2000, + Protocol: "tcp", + Services: nil, + }).Validate() + require.EqualError(t, err, "Consul Ingress Listener requires one or more services") + }) + + t.Run("invalid service", func(t *testing.T) { + err := (&ConsulIngressListener{ + Port: 2000, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "", + }}, + }).Validate() + require.EqualError(t, err, "Consul Ingress Service requires a name") + }) + + t.Run("ok", func(t *testing.T) { + err := (&ConsulIngressListener{ + Port: 2000, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "service1", + }}, + }).Validate() + require.NoError(t, err) + }) +} + +func TestConsulIngressConfigEntry_Validate(t *testing.T) { + t.Run("no listeners", func(t *testing.T) { + err := (&ConsulIngressConfigEntry{}).Validate() + require.EqualError(t, err, "Consul Ingress Gateway requires at least one listener") + }) + + t.Run("invalid listener", func(t *testing.T) { + err := (&ConsulIngressConfigEntry{ + Listeners: []*ConsulIngressListener{{ + Port: 9000, + Protocol: "tcp", + }}, + }).Validate() + require.EqualError(t, err, "Consul Ingress Listener requires one or more services") + }) + + t.Run("full", func(t *testing.T) { + err := (&ConsulIngressConfigEntry{ + TLS: &ConsulGatewayTLSConfig{ + Enabled: true, + }, + Listeners: []*ConsulIngressListener{{ + Port: 9000, + Protocol: "tcp", + Services: []*ConsulIngressService{{ + Name: "service1", + }}, + }}, + }).Validate() + require.NoError(t, err) + }) +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 85d16d81d690..b2bffd9087fe 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -4241,9 +4241,9 @@ func (j *Job) ConnectTasks() map[string][]string { m := make(map[string][]string) for _, tg := range j.TaskGroups { for _, task := range tg.Tasks { - if task.Kind.IsConnectProxy() { - // todo(shoenig): when we support native, probably need to check - // an additional TBD TaskKind as well. + if task.Kind.IsConnectProxy() || + task.Kind.IsConnectNative() || + task.Kind.IsAnyConnectGateway() { m[tg.Name] = append(m[tg.Name], task.Name) } } @@ -4251,6 +4251,25 @@ func (j *Job) ConnectTasks() map[string][]string { return m } +// ConfigEntries accumulates the Consul Configuration Entries defined in task groups +// of j. +// +// Currently Nomad only supports entries for connect ingress gateways. +func (j *Job) ConfigEntries() map[string]*ConsulIngressConfigEntry { + igEntries := make(map[string]*ConsulIngressConfigEntry) + for _, tg := range j.TaskGroups { + for _, service := range tg.Services { + if service.Connect.IsGateway() { + if ig := service.Connect.Gateway.Ingress; ig != nil { + igEntries[service.Name] = ig + } + // imagine also accumulating other entry types in the future + } + } + } + return igEntries +} + // RequiredSignals returns a mapping of task groups to tasks to their required // set of signals func (j *Job) RequiredSignals() map[string]map[string][]string { @@ -5627,8 +5646,10 @@ func (tg *TaskGroup) Validate(j *Job) error { mErr.Errors = append(mErr.Errors, errors.New("Task group count can't be negative")) } if len(tg.Tasks) == 0 { + // could be a lone consul gateway inserted by the connect mutator mErr.Errors = append(mErr.Errors, errors.New("Missing tasks for task group")) } + for idx, constr := range tg.Constraints { if err := constr.Validate(); err != nil { outer := fmt.Errorf("Constraint %d validation failed: %s", idx+1, err) @@ -5979,10 +6000,28 @@ func (tg *TaskGroup) LookupTask(name string) *Task { return nil } +// UsesConnect for convenience returns true if the TaskGroup contains at least +// one service that makes use of Consul Connect features. +// +// Currently used for validating that the task group contains one or more connect +// aware services before generating a service identity token. func (tg *TaskGroup) UsesConnect() bool { for _, service := range tg.Services { if service.Connect != nil { - if service.Connect.IsNative() || service.Connect.HasSidecar() { + if service.Connect.IsNative() || service.Connect.HasSidecar() || service.Connect.IsGateway() { + return true + } + } + } + return false +} + +// UsesConnectGateway for convenience returns true if the TaskGroup contains at +// least one service that makes use of Consul Connect Gateway features. +func (tg *TaskGroup) UsesConnectGateway() bool { + for _, service := range tg.Services { + if service.Connect != nil { + if service.Connect.IsGateway() { return true } } @@ -6183,9 +6222,9 @@ type Task struct { // UsesConnect is for conveniently detecting if the Task is able to make use // of Consul Connect features. This will be indicated in the TaskKind of the // Task, which exports known types of Tasks. UsesConnect will be true if the -// task is a connect proxy, or if the task is connect native. +// task is a connect proxy, connect native, or is a connect gateway. func (t *Task) UsesConnect() bool { - return t.Kind.IsConnectProxy() || t.Kind.IsConnectNative() + return t.Kind.IsConnectProxy() || t.Kind.IsConnectNative() || t.Kind.IsAnyConnectGateway() } func (t *Task) Copy() *Task { @@ -6621,13 +6660,31 @@ func (k TaskKind) Value() string { return "" } -// IsConnectProxy returns true if the TaskKind is connect-proxy +func (k TaskKind) hasPrefix(prefix string) bool { + return strings.HasPrefix(string(k), prefix+":") && len(k) > len(prefix)+1 +} + +// IsConnectProxy returns true if the TaskKind is connect-proxy. func (k TaskKind) IsConnectProxy() bool { - return strings.HasPrefix(string(k), ConnectProxyPrefix+":") && len(k) > len(ConnectProxyPrefix)+1 + return k.hasPrefix(ConnectProxyPrefix) } +// IsConnectNative returns true if the TaskKind is connect-native. func (k TaskKind) IsConnectNative() bool { - return strings.HasPrefix(string(k), ConnectNativePrefix+":") && len(k) > len(ConnectNativePrefix)+1 + return k.hasPrefix(ConnectNativePrefix) +} + +func (k TaskKind) IsConnectIngress() bool { + return k.hasPrefix(ConnectIngressPrefix) +} + +func (k TaskKind) IsAnyConnectGateway() bool { + switch { + case k.IsConnectIngress(): + return true + default: + return false + } } const ( @@ -6638,6 +6695,22 @@ const ( // ConnectNativePrefix is the prefix used for fields referencing a Connect // Native Task ConnectNativePrefix = "connect-native" + + // ConnectIngressPrefix is the prefix used for fields referencing a Consul + // Connect Ingress Gateway Proxy. + ConnectIngressPrefix = "connect-ingress" + + // ConnectTerminatingPrefix is the prefix used for fields referencing a Consul + // Connect Terminating Gateway Proxy. + // + // Not yet supported. + // ConnectTerminatingPrefix = "connect-terminating" + + // ConnectMeshPrefix is the prefix used for fields referencing a Consul Connect + // Mesh Gateway Proxy. + // + // Not yet supported. + // ConnectMeshPrefix = "connect-mesh" ) // ValidateConnectProxyService checks that the service that is being diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index b48a5aeb3f8d..be3bee582197 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -828,6 +828,15 @@ func TestTask_UsesConnect(t *testing.T) { usesConnect := task.UsesConnect() require.True(t, usesConnect) }) + + t.Run("ingress gateway", func(t *testing.T) { + task := &Task{ + Name: "task1", + Kind: NewTaskKind(ConnectIngressPrefix, "task1"), + } + usesConnect := task.UsesConnect() + require.True(t, usesConnect) + }) } func TestTaskGroup_UsesConnect(t *testing.T) { @@ -859,6 +868,16 @@ func TestTaskGroup_UsesConnect(t *testing.T) { }, true) }) + t.Run("tg uses gateway", func(t *testing.T) { + try(t, &TaskGroup{ + Services: []*Service{{ + Connect: &ConsulConnect{ + Gateway: consulIngressGateway1, + }, + }}, + }, true) + }) + t.Run("tg does not use connect", func(t *testing.T) { try(t, &TaskGroup{ Services: []*Service{ diff --git a/nomad/testing.go b/nomad/testing.go index a93035e6019b..f575139ccc2a 100644 --- a/nomad/testing.go +++ b/nomad/testing.go @@ -99,9 +99,9 @@ func TestServer(t testing.T, cb func(*Config)) (*Server, func()) { cb(config) } - catalog := consul.NewMockCatalog(config.Logger) - - acls := consul.NewMockACLsAPI(config.Logger) + cCatalog := consul.NewMockCatalog(config.Logger) + cConfigs := consul.NewMockConfigsAPI(config.Logger) + cACLs := consul.NewMockACLsAPI(config.Logger) for i := 10; i >= 0; i-- { // Get random ports, need to cleanup later @@ -114,7 +114,7 @@ func TestServer(t testing.T, cb func(*Config)) (*Server, func()) { config.SerfConfig.MemberlistConfig.BindPort = ports[1] // Create server - server, err := NewServer(config, catalog, acls) + server, err := NewServer(config, cCatalog, cConfigs, cACLs) if err == nil { return server, func() { ch := make(chan error) diff --git a/vendor/github.com/hashicorp/nomad/api/services.go b/vendor/github.com/hashicorp/nomad/api/services.go index ce6bcd9928fd..9640119d64bb 100644 --- a/vendor/github.com/hashicorp/nomad/api/services.go +++ b/vendor/github.com/hashicorp/nomad/api/services.go @@ -152,6 +152,7 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) { // ConsulConnect represents a Consul Connect jobspec stanza. type ConsulConnect struct { Native bool + Gateway *ConsulGateway SidecarService *ConsulSidecarService `mapstructure:"sidecar_service"` SidecarTask *SidecarTask `mapstructure:"sidecar_task"` } @@ -163,6 +164,7 @@ func (cc *ConsulConnect) Canonicalize() { cc.SidecarService.Canonicalize() cc.SidecarTask.Canonicalize() + cc.Gateway.Canonicalize() } // ConsulSidecarService represents a Consul Connect SidecarService jobspec @@ -290,3 +292,263 @@ type ConsulExposePath struct { LocalPathPort int `mapstructure:"local_path_port"` ListenerPort string `mapstructure:"listener_port"` } + +// ConsulGateway is used to configure one of the Consul Connect Gateway types. +type ConsulGateway struct { + // Proxy is used to configure the Envoy instance acting as the gateway. + Proxy *ConsulGatewayProxy + + // Ingress represents the Consul Configuration Entry for an Ingress Gateway. + Ingress *ConsulIngressConfigEntry + + // Terminating is not yet supported. + // Terminating *ConsulTerminatingConfigEntry + + // Mesh is not yet supported. + // Mesh *ConsulMeshConfigEntry +} + +func (g *ConsulGateway) Canonicalize() { + if g == nil { + return + } + g.Proxy.Canonicalize() + g.Ingress.Canonicalize() +} + +func (g *ConsulGateway) Copy() *ConsulGateway { + if g == nil { + return nil + } + + return &ConsulGateway{ + Proxy: g.Proxy.Copy(), + Ingress: g.Ingress.Copy(), + } +} + +type ConsulGatewayBindAddress struct { + Address string `mapstructure:"address"` + Port int `mapstructure:"port"` +} + +var ( + defaultGatewayConnectTimeout = 5 * time.Second +) + +// ConsulGatewayProxy is used to tune parameters of the proxy instance acting as +// one of the forms of Connect gateways that Consul supports. +// +// https://www.consul.io/docs/connect/proxies/envoy#gateway-options +type ConsulGatewayProxy struct { + ConnectTimeout *time.Duration `mapstructure:"connect_timeout"` + EnvoyGatewayBindTaggedAddresses bool `mapstructure:"envoy_gateway_bind_tagged_addresses"` + EnvoyGatewayBindAddresses map[string]*ConsulGatewayBindAddress `mapstructure:"envoy_gateway_bind_addresses"` + EnvoyGatewayNoDefaultBind bool `mapstructure:"envoy_gateway_no_default_bind"` + Config map[string]interface{} // escape hatch envoy config +} + +func (p *ConsulGatewayProxy) Canonicalize() { + if p == nil { + return + } + + if p.ConnectTimeout == nil { + // same as the default from consul + p.ConnectTimeout = timeToPtr(defaultGatewayConnectTimeout) + } + + if len(p.EnvoyGatewayBindAddresses) == 0 { + p.EnvoyGatewayBindAddresses = nil + } + + if len(p.Config) == 0 { + p.Config = nil + } +} + +func (p *ConsulGatewayProxy) Copy() *ConsulGatewayProxy { + if p == nil { + return nil + } + + var binds map[string]*ConsulGatewayBindAddress = nil + if p.EnvoyGatewayBindAddresses != nil { + binds = make(map[string]*ConsulGatewayBindAddress, len(p.EnvoyGatewayBindAddresses)) + for k, v := range p.EnvoyGatewayBindAddresses { + binds[k] = v + } + } + + var config map[string]interface{} = nil + if p.Config != nil { + config = make(map[string]interface{}, len(p.Config)) + for k, v := range p.Config { + config[k] = v + } + } + + return &ConsulGatewayProxy{ + ConnectTimeout: timeToPtr(*p.ConnectTimeout), + EnvoyGatewayBindTaggedAddresses: p.EnvoyGatewayBindTaggedAddresses, + EnvoyGatewayBindAddresses: binds, + EnvoyGatewayNoDefaultBind: p.EnvoyGatewayNoDefaultBind, + Config: config, + } +} + +// ConsulGatewayTLSConfig is used to configure TLS for a gateway. +type ConsulGatewayTLSConfig struct { + Enabled bool +} + +func (tc *ConsulGatewayTLSConfig) Canonicalize() { +} + +func (tc *ConsulGatewayTLSConfig) Copy() *ConsulGatewayTLSConfig { + if tc == nil { + return nil + } + + return &ConsulGatewayTLSConfig{ + Enabled: tc.Enabled, + } +} + +// ConsulIngressService is used to configure a service fronted by the ingress gateway. +type ConsulIngressService struct { + // Namespace is not yet supported. + // Namespace string + Name string + + Hosts []string +} + +func (s *ConsulIngressService) Canonicalize() { + if s == nil { + return + } + + if len(s.Hosts) == 0 { + s.Hosts = nil + } +} + +func (s *ConsulIngressService) Copy() *ConsulIngressService { + if s == nil { + return nil + } + + var hosts []string = nil + if n := len(s.Hosts); n > 0 { + hosts = make([]string, n) + copy(hosts, s.Hosts) + } + + return &ConsulIngressService{ + Name: s.Name, + Hosts: hosts, + } +} + +const ( + defaultIngressListenerProtocol = "tcp" +) + +// ConsulIngressListener is used to configure a listener on a Consul Ingress +// Gateway. +type ConsulIngressListener struct { + Port int + Protocol string + Services []*ConsulIngressService +} + +func (l *ConsulIngressListener) Canonicalize() { + if l == nil { + return + } + + if l.Protocol == "" { + // same as default from consul + l.Protocol = defaultIngressListenerProtocol + } + + if len(l.Services) == 0 { + l.Services = nil + } +} + +func (l *ConsulIngressListener) Copy() *ConsulIngressListener { + if l == nil { + return nil + } + + var services []*ConsulIngressService = nil + if n := len(l.Services); n > 0 { + services = make([]*ConsulIngressService, n) + for i := 0; i < n; i++ { + services[i] = l.Services[i].Copy() + } + } + + return &ConsulIngressListener{ + Port: l.Port, + Protocol: l.Protocol, + Services: services, + } +} + +// ConsulIngressConfigEntry represents the Consul Configuration Entry type for +// an Ingress Gateway. +// +// https://www.consul.io/docs/agent/config-entries/ingress-gateway#available-fields +type ConsulIngressConfigEntry struct { + // Namespace is not yet supported. + // Namespace string + + TLS *ConsulGatewayTLSConfig + Listeners []*ConsulIngressListener +} + +func (e *ConsulIngressConfigEntry) Canonicalize() { + if e == nil { + return + } + + e.TLS.Canonicalize() + + if len(e.Listeners) == 0 { + e.Listeners = nil + } + + for _, listener := range e.Listeners { + listener.Canonicalize() + } +} + +func (e *ConsulIngressConfigEntry) Copy() *ConsulIngressConfigEntry { + if e == nil { + return nil + } + + var listeners []*ConsulIngressListener = nil + if n := len(e.Listeners); n > 0 { + listeners = make([]*ConsulIngressListener, n) + for i := 0; i < n; i++ { + listeners[i] = e.Listeners[i].Copy() + } + } + + return &ConsulIngressConfigEntry{ + TLS: e.TLS.Copy(), + Listeners: listeners, + } +} + +// ConsulTerminatingConfigEntry is not yet supported. +// type ConsulTerminatingConfigEntry struct { +// } + +// ConsulMeshConfigEntry is not yet supported. +// type ConsulMeshConfigEntry struct { +// } diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index 637773f4961c..91c7585950aa 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -166,6 +166,7 @@ export default [ 'env', 'ephemeral_disk', 'expose', + 'gateway', 'group', 'job', 'lifecycle', diff --git a/website/pages/docs/job-specification/gateway.mdx b/website/pages/docs/job-specification/gateway.mdx new file mode 100644 index 000000000000..0964083199a0 --- /dev/null +++ b/website/pages/docs/job-specification/gateway.mdx @@ -0,0 +1,207 @@ +--- +layout: docs +page_title: gateway Stanza - Job Specification +sidebar_title: gateway +description: |- + The "gateway" stanza allows specifying options for configuring Consul Gateways + used in the Consul Connect integration +--- + +# `gateway` Stanza + + + +The `gateway` stanza allows configuration of [Consul Connect Gateways](https://www.consul.io/docs/connect/gateways). Nomad will +automatically create the necessary Gateway [Configuration Entry](https://www.consul.io/docs/agent/config-entries) +as well as inject an Envoy proxy task into the Nomad job to serve as the Gateway. + +The `gateway` configuration is valid within the context of a `connect` stanza. +Additional information about Gateway configurations can be found in Consul's +[Connect Gateways](https://www.consul.io/docs/connect/gateways) documentation. + +~> **Note:** [Ingress Gateways](https://www.consul.io/docs/connect/gateways/ingress-gateway) +are generally intended for enabling access into a Consul service mesh from within the +same network. For public ingress products like [NGINX](https://learn.hashicorp.com/tutorials/nomad/load-balancing-nginx?in=nomad/load-balancing) +provide more suitable features. + +```hcl +job "ingress-example" { + + datacenters = ["dc1"] + + group "ingress-group" { + + network { + mode = "bridge" + + port "inbound" { + static = 8080 + } + } + + service { + name = "ingress-service" + port = "8080" + + connect { + gateway { + proxy { + // Consul Gateway Proxy configuration options + connect_timeout = "500ms" + } + + ingress { + // Consul Ingress Gateway Configuration Entry + + tls { + enabled = false + } + + listener { + port = 8080 + protocol = "http" + service { + name = "web" + hosts = ["example.com", "example.com:8080"] + } + } + + listener { + port = 3306 + protocol = "tcp" + service { + name = "database" + } + } + } + } + } + } + } +} +``` + +## `gateway` Parameters + +- `proxy` ([proxy]: nil) - Configuration of the Envoy proxy that will + be injected into the task group. +- `ingress` ([ingress]: nil) - Configuration Entry of type `ingress-gateway` + that will be associated with the service. + +### `proxy` Parameters + +- `connect_timeout` `(string: "5s")` - The amount of time to allow when making upstream + connections before timing out. Defaults to 5 seconds. If the upstream service has + the configuration option [connect_timeout_ms] set for the `service-resolver`, that + timeout value will take precedence over this gateway proxy option. +- `envoy_gateway_bind_tagged_addresses` `(bool: false)` - Indicates that the gateway + services tagged addresses should be bound to listeners in addition to the default + listener address. +- `envoy_gateway_bind_addresses` (map: nil) - A map of additional addresses to be bound. + The keys to this map are the same of the listeners to be created and the values are + a map with two keys - address and port, that combined make the address to bind the + listener to. These are bound in addition to the default address. + If `bridge` networking is in use, this map is automatically populated with additional + listeners enabling the Envoy proxy to work from inside the network namespace. + +``` +envoy_gateway_bind_addresses "" { + address = "0.0.0.0" + port = +} +``` + +- `envoy_gateway_no_default_bind` `(bool: false)` - Prevents binding to the default + address of the gateway service. This should be used with one of the other options + to configure the gateway's bind addresses. If `bridge` networking is in use, this + value will default to `true` since the Envoy proxy does not need to bind to the + service address from inside the network namespace. +- `config` `(map: nil)` - Escape hatch for [Advanced Configuration] of Envoy. + +#### `address` Parameters + +- `address` `(string: required)` - The address to bind to when combined with `port`. +- `port` `(int: required)` - The port to listen to. + +### `ingress` Parameters + +- `tls` ([tls]: nil) - TLS configuration for this gateway. +- `listener` (array<[listener]> : required) - One or more listeners that the + ingress gateway should setup, uniquely identified by their port number. + + +#### `tls` Parameters + +- `enabled` `(bool: false)` - Set this configuration to enable TLS for every listener + on the gateway. If TLS is enabled, then each host defined in the `host` field will + be added as a DNSSAN to the gateway's x509 certificate. + +#### `listener` Parameters + +- `port` `(int: required)` - The port that the listener should receive traffic on. +- `protocol` `(string: "tcp")` - The protocol associated with the listener. Either + `tcp` or `http`. + + ~> **Note:** If using `http`, preconfiguring a [service-default] in Consul to + set the [Protocol](https://www.consul.io/docs/agent/config-entries/service-defaults#protocol) + of the service to `http` is recommended. + +- `service` (array<[service]>: required) - One or more services to be + exposed via this listener. For `tcp` listeners, only a single service is allowed. + +#### `service` Parameters + +- `name` `(string: required)` - The name of the service that should be exposed through + this listener. This can be either a service registered in the catalog, or a + service defined by other config entries, or a service that is going to be configured + by Nomad. If the wildcard specifier `*` is provided, then ALL services will be + exposed through this listener. This is not supported for a listener with protocol `tcp`. +- `hosts` `(array: nil)` - A list of hosts that specify what requests will + match this service. This cannot be used with a `tcp` listener, and cannot be + specified alongside a wildcard (`*`) service name. If not specified, the default + domain `.ingress.*` will be used to match services. Requests *must* + send the correct host to be routed to the defined service. + + The wildcard specifier `*` can be used by itself to match all traffic coming to + the ingress gateway, if TLS is not enabled. This allows a user to route all traffic + to a single service without specifying a host, allowing simpler tests and demos. + Otherwise, the wildcard specifier can be used as part of the host to match + multiple hosts, but only in the leftmost DNS label. This ensures that all defined + hosts are valid DNS records. For example, `*.example.com` is valid while `example.*` + and `*-suffix.example.com` are not. + + ~> **Note:** If a well-known port is not used, i.e. a port other than 80 (http) or 443 (https), + then the port must be appended to the host to correctly match traffic. This is + defined in the [HTTP/1.1 RFC](https://tools.ietf.org/html/rfc2616#section-14.23). + If TLS is enabled, then the host **without** the port must be added to the `hosts` + field as well. TLS verification only matches against the hostname of the incoming + connection, and does not take into account the port. + +### Gateway with host networking + +Nomad supports running gateways using host networking. A static port must be allocated +for use by the [Envoy admin interface](https://www.envoyproxy.io/docs/envoy/latest/operations/admin) +and assigned to the proxy service definition. + +!> **Warning:** There is no way to disable the Envoy admin interface, which will be +accessible to any workload running on the same Nomad client. The admin interface exposes +information about the proxy, including a Consul Service Identity token if Consul ACLs +are enabled. + +[proxy]: /docs/job-specification/gateway#proxy-parameters +[ingress]: /docs/job-specification/gateway#ingress-parameters +[tls]: /docs/job-specification/gateway#tlsconfig-parameters +[listener]: /docs/job-specification/gateway#listener-parameters +[service]: /docs/job-specification/gateway#service-parameters +[service-default]: https://www.consul.io/docs/agent/config-entries/service-defaults +[connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout +[address]: /docs/job-specification/gateway#address-parameters +[Advanced Configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration