connect: enable automatic expose paths for individual group service c…

…hecks

Part of #6120

Building on the support for enabling connect proxy paths in #7323, this change
adds the ability to configure the 'service.check.expose' flag on group-level
service check definitions for services that are connect-enabled. This is a slight
deviation from the "magic" that Consul provides. With Consul, the 'expose' flag
exists on the connect.proxy stanza, which will then auto-generate expose paths
for every HTTP and gRPC service check associated with that connect-enabled
service.

A first attempt at providing similar magic for Nomad's Consul Connect integration
followed that pattern exactly, as seen in #7396. However, on reviewing the PR
we realized having the `expose` flag on the proxy stanza inseperably ties together
the automatic path generation with every HTTP/gRPC defined on the service. This
makes sense in Consul's context, because a service definition is reasonably
associated with a single "task". With Nomad's group level service definitions
however, there is a reasonable expectation that a service definition is more
abstractly representative of multiple services within the task group. In this
case, one would want to define checks of that service which concretely make HTTP
or gRPC requests to different underlying tasks. Such a model is not possible
with the course `proxy.expose` flag.

Instead, we now have the flag made available within the check definitions themselves.
By making the expose feature resolute to each check, it is possible to have
some HTTP/gRPC checks which make use of the envoy exposed paths, as well as
some HTTP/gRPC checks which make use of some orthongonal port-mapping to do
checks on some other task (or even some other bound port of the same task)
within the task group.

Given this example,

group "server-group" {
  network {
    mode = "bridge"
    port "forchecks" {
      to = -1
    }
  }

  service {
    name = "myserver"
    port = 2000

    connect {
      sidecar_service {
      }
    }

    check {
      name     = "mycheck-myserver"
      type     = "http"
      port     = "forchecks"
      interval = "3s"
      timeout  = "2s"
      method   = "GET"
      path     = "/classic/responder/health"
      expose   = true
    }
  }
}

Nomad will automatically inject (via job endpoint mutator) the
extrapolated expose path configuration, i.e.

expose {
  path {
    path            = "/classic/responder/health"
    protocol        = "http"
    local_path_port = 2000
    listener_port   = "forchecks"
  }
}

Documentation is coming in #7440 (needs updating, doing next)

Modifications to the `countdash` examples in hashicorp/demo-consul-101#6
which will make the examples in the documentation actually runnable.

Will add some e2e tests based on the above when it becomes available.

api/services.go

@@ -81,6 +81,7 @@ type ServiceCheck struct {
 	Path          string
 	Protocol      string
 	PortLabel     string `mapstructure:"port"`
+	Expose        bool
 	AddressMode   string `mapstructure:"address_mode"`
 	Interval      time.Duration
 	Timeout       time.Duration
@@ -183,7 +184,6 @@ type ConsulUpstream struct {
 
 type ConsulExposeConfig struct {
 	Path []*ConsulExposePath `mapstructure:"path"`
-	// todo(shoenig): add magic for 'checks' option
 }
 
 type ConsulExposePath struct {

command/agent/job_endpoint.go

@@ -1151,6 +1151,7 @@ func ApiServicesToStructs(in []*api.Service) []*structs.Service {
 					Path:          check.Path,
 					Protocol:      check.Protocol,
 					PortLabel:     check.PortLabel,
+					Expose:        check.Expose,
 					AddressMode:   check.AddressMode,
 					Interval:      check.Interval,
 					Timeout:       check.Timeout,

jobspec/parse_service.go

@@ -405,7 +405,6 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
 func parseExpose(eo *ast.ObjectItem) (*api.ConsulExposeConfig, error) {
 	valid := []string{
 		"path", // an array of path blocks
-		// todo(shoenig) checks boolean
 	}
 
 	if err := helper.CheckHCLKeys(eo.Val, valid); err != nil {
@@ -514,6 +513,7 @@ func parseChecks(service *api.Service, checkObjs *ast.ObjectList) error {
 			"path",
 			"protocol",
 			"port",
+			"expose",
 			"command",
 			"args",
 			"initial_status",

jobspec/parse_test.go

@@ -1150,6 +1150,32 @@ func TestParse(t *testing.T) {
 			},
 			false,
 		},
+		{
+			"tg-service-check-expose.hcl",
+			&api.Job{
+				ID:   helper.StringToPtr("group_service_proxy_expose"),
+				Name: helper.StringToPtr("group_service_proxy_expose"),
+				TaskGroups: []*api.TaskGroup{{
+					Name: helper.StringToPtr("group"),
+					Services: []*api.Service{{
+						Name: "example",
+						Connect: &api.ConsulConnect{
+							SidecarService: &api.ConsulSidecarService{
+								Proxy: &api.ConsulProxy{},
+							},
+						},
+						Checks: []api.ServiceCheck{{
+							Name:   "example-check1",
+							Expose: true,
+						}, {
+							Name:   "example-check2",
+							Expose: false,
+						}},
+					}},
+				}},
+			},
+			false,
+		},
 		{
 			"tg-service-enable-tag-override.hcl",
 			&api.Job{

jobspec/test-fixtures/tg-service-check-expose.hcl

@@ -0,0 +1,23 @@
+job "group_service_proxy_expose" {
+  group "group" {
+    service {
+      name = "example"
+      connect {
+        sidecar_service {
+          proxy {
+          }
+        }
+      }
+
+      check {
+        name = "example-check1"
+        expose = true
+      }
+
+      check {
+        name = "example-check2"
+        expose = false
+      }
+    }
+  }
+}

nomad/job_endpoint.go

@@ -61,10 +61,12 @@ func NewJobEndpoints(s *Server) *Job {
 		mutators: []jobMutator{
 			jobCanonicalizer{},
 			jobConnectHook{},
+			jobExposeCheckHook{},
 			jobImpliedConstraints{},
 		},
 		validators: []jobValidator{
 			jobConnectHook{},
+			jobExposeCheckHook{},
 			jobValidate{},
 		},
 	}

nomad/job_endpoint_hook_expose_check.go

@@ -0,0 +1,228 @@
+package nomad
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/pkg/errors"
+)
+
+type jobExposeCheckHook struct{}
+
+func (jobExposeCheckHook) Name() string {
+	return "expose-check"
+}
+
+// Mutate will scan every task group for group-services which have checks defined
+// that have the Expose field configured, and generate expose path configurations
+// extrapolated from those check definitions.
+func (jobExposeCheckHook) Mutate(job *structs.Job) (_ *structs.Job, warnings []error, err error) {
+	for _, tg := range job.TaskGroups {
+		for _, s := range tg.Services {
+			for _, c := range s.Checks {
+				if c.Expose {
+					if exposePath, err := exposePathForCheck(tg, s, c); err != nil {
+						return nil, nil, err
+					} else if exposePath != nil {
+						serviceExposeConfig := serviceExposeConfig(s)
+						// insert only if not already present - required for job
+						// updates which would otherwise create duplicates
+						if !containsExposePath(serviceExposeConfig.Paths, *exposePath) {
+							serviceExposeConfig.Paths = append(
+								serviceExposeConfig.Paths, *exposePath,
+							)
+						}
+					}
+				}
+			}
+		}
+	}
+	return job, nil, nil
+}
+
+// Validate will ensure:
+// - The job contains valid network configuration for each task group in which
+//	an expose path is configured. The network must be of type bridge mode.
+// - The check Expose field is configured only for connect-enabled group-services.
+func (jobExposeCheckHook) Validate(job *structs.Job) (warnings []error, err error) {
+	for _, tg := range job.TaskGroups {
+		// Make sure any group that contains a group-service that enables expose
+		// is configured with one network that is in "bridge" mode. This check
+		// is being done independently of the preceding Connect task injection
+		// hook, because at some point in the future Connect will not require the
+		// use of network namespaces, whereas the use of "expose" does not make
+		// sense without the use of network namespace.
+		if err := tgValidateUseOfBridgeMode(tg); err != nil {
+			return nil, err
+		}
+		// Make sure any group-service that contains a check that enables expose
+		// is connect-enabled and does not specify a custom sidecar task. We only
+		// support the expose feature when using the built-in Envoy integration.
+		if err := tgValidateUseOfCheckExpose(tg); err != nil {
+			return nil, err
+		}
+	}
+	return nil, nil
+}
+
+// serviceExposeConfig digs through s to extract the connect sidecar service proxy
+// expose configuration. It is not required of the user to provide this, so it
+// is created on demand here as needed in the case where any service check exposes
+// itself.
+//
+// The service, connect, and sidecar_service are assumed not to be nil, as they
+// are enforced in previous hooks / validation.
+func serviceExposeConfig(s *structs.Service) *structs.ConsulExposeConfig {
+	if s.Connect.SidecarService.Proxy == nil {
+		s.Connect.SidecarService.Proxy = new(structs.ConsulProxy)
+	}
+	if s.Connect.SidecarService.Proxy.Expose == nil {
+		s.Connect.SidecarService.Proxy.Expose = new(structs.ConsulExposeConfig)
+	}
+	return s.Connect.SidecarService.Proxy.Expose
+}
+
+// containsExposePath returns true if path is contained in paths.
+func containsExposePath(paths []structs.ConsulExposePath, path structs.ConsulExposePath) bool {
+	for _, p := range paths {
+		if p == path {
+			return true
+		}
+	}
+	return false
+}
+
+// tgValidateUseOfCheckExpose ensures that any service check in tg making use
+// of the expose field is within an appropriate context to do so. The check must
+// be a group level check, and must use the builtin envoy proxy.
+func tgValidateUseOfCheckExpose(tg *structs.TaskGroup) error {
+	// validation for group services (which must use built-in connect proxy)
+	for _, s := range tg.Services {
+		for _, check := range s.Checks {
+			if check.Expose && !serviceUsesConnectEnvoy(s) {
+				return errors.Errorf(
+					"exposed service check %s->%s->%s requires use of Nomad's builtin Connect proxy",
+					tg.Name, s.Name, check.Name,
+				)
+			}
+		}
+	}
+
+	// validation for task services (which must not be configured to use Expose)
+	for _, t := range tg.Tasks {
+		for _, s := range t.Services {
+			for _, check := range s.Checks {
+				if check.Expose {
+					return errors.Errorf(
+						"exposed service check %s[%s]->%s->%s is not a task-group service",
+						tg.Name, t.Name, s.Name, check.Name,
+					)
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// tgValidateUseOfBridgeMode ensures there is exactly 1 network configured for
+// the task group, and that it makes use of "bridge" mode (i.e. enables network
+// namespaces).
+func tgValidateUseOfBridgeMode(tg *structs.TaskGroup) error {
+	if tgUsesExposeCheck(tg) {
+		if len(tg.Networks) != 1 {
+			return errors.Errorf("group %q must specify one bridge network for exposing service check(s)", tg.Name)
+		}
+		if tg.Networks[0].Mode != "bridge" {
+			return errors.Errorf("group %q must use bridge network for exposing service check(s)", tg.Name)
+		}
+	}
+	return nil
+}
+
+// tgUsesExposeCheck returns true if any group service in the task group makes
+// use of the expose field.
+func tgUsesExposeCheck(tg *structs.TaskGroup) bool {
+	for _, s := range tg.Services {
+		for _, check := range s.Checks {
+			if check.Expose {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// serviceUsesConnectEnvoy returns true if the service is going to end up using
+// the built-in envoy proxy.
+//
+// This implementation is kind of reading tea leaves - firstly Connect
+// must be enabled, and second the sidecar_task must not be overridden. If these
+// conditions are met, the preceding connect hook will have injected a Connect
+// sidecar task, the configuration of which is interpolated at runtime.
+func serviceUsesConnectEnvoy(s *structs.Service) bool {
+	// A non-nil connect stanza implies this service isn't connect enabled in
+	// the first place.
+	if s.Connect == nil {
+		return false
+	}
+
+	// A non-nil connect.sidecar_task stanza implies the sidecar task is being
+	// overridden (i.e. the default Envoy is not being uesd).
+	if s.Connect.SidecarTask != nil {
+		return false
+	}
+
+	return true
+}
+
+// checkIsExposable returns true if check is qualified for automatic generation
+// of connect proxy expose path configuration based on configured consul checks.
+// To qualify, the check must be of type "http" or "grpc", and must have a Path
+// configured.
+func checkIsExposable(check *structs.ServiceCheck) bool {
+	switch strings.ToLower(check.Type) {
+	case "grpc", "http":
+		return strings.HasPrefix(check.Path, "/")
+	default:
+		return false
+	}
+}
+
+// exposePathForCheck extrapolates the necessary expose path configuration for
+// the given consul service check. If the check is not compatible, nil is
+// returned.
+func exposePathForCheck(tg *structs.TaskGroup, s *structs.Service, check *structs.ServiceCheck) (*structs.ConsulExposePath, error) {
+	if !checkIsExposable(check) {
+		return nil, nil
+	}
+
+	// Determine the local service port (i.e. what port the service is actually
+	// listening to inside the network namespace).
+	//
+	// Similar logic exists in getAddress of client.go which is used for
+	// creating check & service registration objects.
+	//
+	// The difference here is the address is predestined to be localhost since
+	// it is binding inside the namespace.
+	var port int
+	if _, port = tg.Networks.Port(s.PortLabel); port <= 0 { // try looking up by port label
+		if port, _ = strconv.Atoi(s.PortLabel); port <= 0 { // then try direct port value
+			return nil, errors.Errorf(
+				"unable to determine local service port for service check %s->%s->%s",
+				tg.Name, s.Name, check.Name,
+			)
+		}
+	}
+
+	// The Path, Protocol, and PortLabel are just copied over from the service
+	// check definition. It is required that the user configure their own port
+	// mapping for each check, including setting the 'to = -1' sentinel value
+	// enabling the network namespace pass-through.
+	return &structs.ConsulExposePath{
+		Path:          check.Path,
+		Protocol:      check.Protocol,
+		LocalPathPort: port,
+		ListenerPort:  check.PortLabel,
+	}, nil
+}