Allow Nomads Consul health checks to be configurable.

This change allows the client HTTP and the server HTTP, Serf and
RPC health check names within Consul to be configurable with the
defaults as previous. The configuration can be done via either a
config file or using CLI flags.

Closes hashicorp#3988

command/agent/agent.go

@@ -15,6 +15,7 @@ import (
 
 	metrics "github.com/armon/go-metrics"
 	"github.com/hashicorp/consul/api"
+	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/nomad/client"
 	clientconfig "github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/command/agent/consul"
@@ -56,6 +57,10 @@ type Agent struct {
 	// consulCatalog is the subset of Consul's Catalog API Nomad uses.
 	consulCatalog consul.CatalogAPI
 
+	// consulSupportsTLSSkipVerify flags whether or not Nomad can register
+	// checks with TLSSkipVerify
+	consulSupportsTLSSkipVerify bool
+
 	client *client.Client
 
 	server *nomad.Server
@@ -224,18 +229,9 @@ func convertServerConfig(agentConfig *Config, logOutput io.Writer) (*nomad.Confi
 	if err != nil {
 		return nil, fmt.Errorf("Failed to parse Serf advertise address %q: %v", agentConfig.AdvertiseAddrs.Serf, err)
 	}
-
-	// Server address is the serf advertise address and rpc port. This is the
-	// address that all servers should be able to communicate over RPC with.
-	serverAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(serfAddr.IP.String(), fmt.Sprintf("%d", rpcAddr.Port)))
-	if err != nil {
-		return nil, fmt.Errorf("Failed to resolve Serf advertise address %q: %v", agentConfig.AdvertiseAddrs.Serf, err)
-	}
-
+	conf.RPCAdvertise = rpcAddr
 	conf.SerfConfig.MemberlistConfig.AdvertiseAddr = serfAddr.IP.String()
 	conf.SerfConfig.MemberlistConfig.AdvertisePort = serfAddr.Port
-	conf.ClientRPCAdvertise = rpcAddr
-	conf.ServerRPCAdvertise = serverAddr
 
 	// Set up gc threshold and heartbeat grace period
 	if gcThreshold := agentConfig.Server.NodeGCThreshold; gcThreshold != "" {
@@ -478,7 +474,7 @@ func (a *Agent) setupServer() error {
 			Tags:      []string{consul.ServiceTagRPC},
 			Checks: []*structs.ServiceCheck{
 				{
-					Name:      "Nomad Server RPC Check",
+					Name:      a.config.Consul.ServerRPCHealthCheckName,
 					Type:      "tcp",
 					Interval:  serverRpcCheckInterval,
 					Timeout:   serverRpcCheckTimeout,
@@ -492,7 +488,7 @@ func (a *Agent) setupServer() error {
 			Tags:      []string{consul.ServiceTagSerf},
 			Checks: []*structs.ServiceCheck{
 				{
-					Name:      "Nomad Server Serf Check",
+					Name:      a.config.Consul.ServerSerfHealthCheckName,
 					Type:      "tcp",
 					Interval:  serverSerfCheckInterval,
 					Timeout:   serverSerfCheckTimeout,
@@ -592,7 +588,7 @@ func (a *Agent) agentHTTPCheck(server bool) *structs.ServiceCheck {
 		httpCheckAddr = a.config.AdvertiseAddrs.HTTP
 	}
 	check := structs.ServiceCheck{
-		Name:      "Nomad Client HTTP Check",
+		Name:      a.config.Consul.ClientHTTPHealthCheckName,
 		Type:      "http",
 		Path:      "/v1/agent/health?type=client",
 		Protocol:  "http",
@@ -602,13 +598,17 @@ func (a *Agent) agentHTTPCheck(server bool) *structs.ServiceCheck {
 	}
 	// Switch to endpoint that doesn't require a leader for servers
 	if server {
-		check.Name = "Nomad Server HTTP Check"
+		check.Name = a.config.Consul.ServerHTTPHealthCheckName
 		check.Path = "/v1/agent/health?type=server"
 	}
 	if !a.config.TLSConfig.EnableHTTP {
 		// No HTTPS, return a plain http check
 		return &check
 	}
+	if !a.consulSupportsTLSSkipVerify {
+		a.logger.Printf("[WARN] agent: not registering Nomad HTTPS Health Check because it requires Consul>=0.7.2")
+		return nil
+	}
 	if a.config.TLSConfig.VerifyHTTPSClient {
 		a.logger.Printf("[WARN] agent: not registering Nomad HTTPS Health Check because verify_https_client enabled")
 		return nil
@@ -850,14 +850,55 @@ func (a *Agent) setupConsul(consulConfig *config.ConsulConfig) error {
 	}
 
 	// Determine version for TLSSkipVerify
+	if self, err := client.Agent().Self(); err == nil {
+		a.consulSupportsTLSSkipVerify = consulSupportsTLSSkipVerify(self)
+	}
 
 	// Create Consul Catalog client for service discovery.
 	a.consulCatalog = client.Catalog()
 
 	// Create Consul Service client for service advertisement and checks.
-	a.consulService = consul.NewServiceClient(client.Agent(), a.logger)
+	a.consulService = consul.NewServiceClient(client.Agent(), a.consulSupportsTLSSkipVerify, a.logger)
 
 	// Run the Consul service client's sync'ing main loop
 	go a.consulService.Run()
 	return nil
 }
+
+var consulTLSSkipVerifyMinVersion = version.Must(version.NewVersion("0.7.2"))
+
+// consulSupportsTLSSkipVerify returns true if Consul supports TLSSkipVerify.
+func consulSupportsTLSSkipVerify(self map[string]map[string]interface{}) bool {
+	member, ok := self["Member"]
+	if !ok {
+		return false
+	}
+	tagsI, ok := member["Tags"]
+	if !ok {
+		return false
+	}
+	tags, ok := tagsI.(map[string]interface{})
+	if !ok {
+		return false
+	}
+	buildI, ok := tags["build"]
+	if !ok {
+		return false
+	}
+	build, ok := buildI.(string)
+	if !ok {
+		return false
+	}
+	parts := strings.SplitN(build, ":", 2)
+	if len(parts) != 2 {
+		return false
+	}
+	v, err := version.NewVersion(parts[0])
+	if err != nil {
+		return false
+	}
+	if v.LessThan(consulTLSSkipVerifyMinVersion) {
+		return false
+	}
+	return true
+}

command/agent/command.go

@@ -120,8 +120,12 @@ func (c *Command) readConfig() *Config {
 		return nil
 	}), "consul-client-auto-join", "")
 	flags.StringVar(&cmdConfig.Consul.ClientServiceName, "consul-client-service-name", "", "")
+	flags.StringVar(&cmdConfig.Consul.ClientHTTPHealthCheckName, "consul-client-http-health-check-name", "", "")
 	flags.StringVar(&cmdConfig.Consul.KeyFile, "consul-key-file", "", "")
 	flags.StringVar(&cmdConfig.Consul.ServerServiceName, "consul-server-service-name", "", "")
+	flags.StringVar(&cmdConfig.Consul.ServerHTTPHealthCheckName, "consul-server-http-health-check-name", "", "")
+	flags.StringVar(&cmdConfig.Consul.ServerSerfHealthCheckName, "consul-server-serf-health-check-name", "", "")
+	flags.StringVar(&cmdConfig.Consul.ServerRPCHealthCheckName, "consul-server-rpc-health-check-name", "", "")
 	flags.Var((flaghelper.FuncBoolVar)(func(b bool) error {
 		cmdConfig.Consul.ServerAutoJoin = &b
 		return nil
@@ -1027,13 +1031,25 @@ Consul Options:
   -consul-client-service-name=<name>
     Specifies the name of the service in Consul for the Nomad clients.
 
+  -consul-client-http-health-check-name=<name>
+    Specifies the HTTP health check name in Consul for the Nomad clients.
+
   -consul-key-file=<path>
     Specifies the path to the private key used for Consul communication. If this
     is set then you need to also set cert_file.
 
   -consul-server-service-name=<name>
     Specifies the name of the service in Consul for the Nomad servers.
 
+  -consul-server-http-health-check-name=<name>
+    Specifies the HTTP health check name in Consul for the Nomad servers.
+
+  -consul-server-serf-health-check-name=<name>
+    Specifies the Serf health check name in Consul for the Nomad servers.
+
+  -consul-server-rpc-health-check-name=<name>
+    Specifies the RPC health check name in Consul for the Nomad servers.
+
   -consul-server-auto-join
     Specifies if the Nomad servers should automatically discover and join other
     Nomad servers by searching for the Consul service name defined in the

command/agent/config-test-fixtures/basic.hcl

@@ -115,7 +115,11 @@ http_api_response_headers {
 }
 consul {
     server_service_name = "nomad"
+    server_http_health_check_name = "nomad-server-http-health-check"
+    server_serf_health_check_name = "nomad-server-serf-health-check"
+    server_rpc_health_check_name = "nomad-server-rpc-health-check"
     client_service_name = "nomad-client"
+    client_http_health_check_name = "nomad-client-http-health-check"
     address = "127.0.0.1:9500"
     token = "token1"
     auth = "username:pass"

command/agent/config_parse.go

@@ -702,9 +702,13 @@ func parseConsulConfig(result **config.ConsulConfig, list *ast.ObjectList) error
 		"checks_use_advertise",
 		"client_auto_join",
 		"client_service_name",
+		"client_http_health_check_name",
 		"key_file",
 		"server_auto_join",
 		"server_service_name",
+		"server_http_health_check_name",
+		"server_serf_health_check_name",
+		"server_rpc_health_check_name",
 		"ssl",
 		"timeout",
 		"token",

command/agent/config_parse_test.go

@@ -134,8 +134,12 @@ func TestConfig_Parse(t *testing.T) {
 				DisableUpdateCheck:        true,
 				DisableAnonymousSignature: true,
 				Consul: &config.ConsulConfig{
-					ServerServiceName:  "nomad",
-					ClientServiceName:  "nomad-client",
+					ServerServiceName:         "nomad",
+					ServerHTTPHealthCheckName: "nomad-server-http-health-check",
+					ServerSerfHealthCheckName: "nomad-server-serf-health-check",
+					ServerRPCHealthCheckName:  "nomad-server-rpc-health-check",
+					ClientServiceName:         "nomad-client",
+					ClientHTTPHealthCheckName: "nomad-client-http-health-check",
 					Addr:               "127.0.0.1:9500",
 					Token:              "token1",
 					Auth:               "username:pass",

nomad/structs/config/consul.go

@@ -23,10 +23,26 @@ type ConsulConfig struct {
 	// servers with Consul
 	ServerServiceName string `mapstructure:"server_service_name"`
 
+	// ServerHTTPHealthCheckName is the name of the health check that Nomad uses
+	// to register the server HTTP health check with Consul
+	ServerHTTPHealthCheckName string `mapstructure:"server_http_health_check_name"`
+
+	// ServerSerfHealthCheckName is the name of the health check that Nomad uses
+	// to register the server Serf health check with Consul
+	ServerSerfHealthCheckName string `mapstructure:"server_serf_health_check_name"`
+
+	// ServerSerfHealthCheckName is the name of the health check that Nomad uses
+	// to register the server RPC health check with Consul
+	ServerRPCHealthCheckName string `mapstructure:"server_rpc_health_check_name"`
+
 	// ClientServiceName is the name of the service that Nomad uses to register
 	// clients with Consul
 	ClientServiceName string `mapstructure:"client_service_name"`
 
+	// ClientHTTPHealthCheckName is the name of the health check that Nomad uses
+	// to register the client HTTP health check with Consul
+	ClientHTTPHealthCheckName string `mapstructure:"client_http_health_check_name"`
+
 	// AutoAdvertise determines if this Nomad Agent will advertise its
 	// services via Consul.  When true, Nomad Agent will register
 	// services with Consul.
@@ -78,15 +94,19 @@ type ConsulConfig struct {
 // `consul` configuration.
 func DefaultConsulConfig() *ConsulConfig {
 	return &ConsulConfig{
-		ServerServiceName:  "nomad",
-		ClientServiceName:  "nomad-client",
-		AutoAdvertise:      helper.BoolToPtr(true),
-		ChecksUseAdvertise: helper.BoolToPtr(false),
-		EnableSSL:          helper.BoolToPtr(false),
-		VerifySSL:          helper.BoolToPtr(true),
-		ServerAutoJoin:     helper.BoolToPtr(true),
-		ClientAutoJoin:     helper.BoolToPtr(true),
-		Timeout:            5 * time.Second,
+		ServerServiceName:         "nomad",
+		ServerHTTPHealthCheckName: "Nomad Server HTTP Check",
+		ServerSerfHealthCheckName: "Nomad Server Serf Check",
+		ServerRPCHealthCheckName:  "Nomad Server RPC Check",
+		ClientServiceName:         "nomad-client",
+		ClientHTTPHealthCheckName: "Nomad Client HTTP Check",
+		AutoAdvertise:             helper.BoolToPtr(true),
+		ChecksUseAdvertise:        helper.BoolToPtr(false),
+		EnableSSL:                 helper.BoolToPtr(false),
+		VerifySSL:                 helper.BoolToPtr(true),
+		ServerAutoJoin:            helper.BoolToPtr(true),
+		ClientAutoJoin:            helper.BoolToPtr(true),
+		Timeout:                   5 * time.Second,
 	}
 }
 
@@ -97,9 +117,21 @@ func (a *ConsulConfig) Merge(b *ConsulConfig) *ConsulConfig {
 	if b.ServerServiceName != "" {
 		result.ServerServiceName = b.ServerServiceName
 	}
+	if b.ServerHTTPHealthCheckName != "" {
+		result.ServerHTTPHealthCheckName = b.ServerHTTPHealthCheckName
+	}
+	if b.ServerSerfHealthCheckName != "" {
+		result.ServerSerfHealthCheckName = b.ServerSerfHealthCheckName
+	}
+	if b.ServerRPCHealthCheckName != "" {
+		result.ServerRPCHealthCheckName = b.ServerRPCHealthCheckName
+	}
 	if b.ClientServiceName != "" {
 		result.ClientServiceName = b.ClientServiceName
 	}
+	if b.ClientHTTPHealthCheckName != "" {
+		result.ClientHTTPHealthCheckName = b.ClientHTTPHealthCheckName
+	}
 	if b.AutoAdvertise != nil {
 		result.AutoAdvertise = helper.BoolToPtr(*b.AutoAdvertise)
 	}