consul/connect: fix bug causing high cpu with multiple connect sidecars in group

.changelog/10883.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+consul/connect: Fixed a bug causing high CPU with multiple connect sidecars in one group
+```

client/allocrunner/taskrunner/envoy_bootstrap_hook.go

@@ -5,9 +5,11 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"time"
 
 	"github.com/hashicorp/go-hclog"
@@ -92,8 +94,10 @@ func newEnvoyBootstrapHookConfig(alloc *structs.Allocation, consul *config.Consu
 }
 
 const (
-	envoyBaseAdminPort      = 19000
+	envoyBaseAdminPort      = 19000 // Consul default (bridge only)
+	envoyBaseReadyPort      = 19100 // Consul default (bridge only)
 	envoyAdminBindEnvPrefix = "NOMAD_ENVOY_ADMIN_ADDR_"
+	envoyReadyBindEnvPrefix = "NOMAD_ENVOY_READY_ADDR_"
 )
 
 const (
@@ -240,12 +244,17 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart
 
 	h.logger.Debug("bootstrapping Consul "+serviceKind, "task", req.Task.Name, "service", serviceName)
 
-	// Envoy runs an administrative API on the loopback interface. There is no
-	// way to turn this feature off.
+	// Envoy runs an administrative listener. There is no way to turn this feature off.
 	// https://github.com/envoyproxy/envoy/issues/1297
 	envoyAdminBind := buildEnvoyAdminBind(h.alloc, serviceName, req.Task.Name, req.TaskEnv)
+
+	// Consul configures a ready listener. There is no way to turn this feature off.
+	envoyReadyBind := buildEnvoyReadyBind(h.alloc, serviceName, req.Task.Name, req.TaskEnv)
+
+	// Set runtime environment variables for the envoy admin and ready listeners.
 	resp.Env = map[string]string{
 		helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_'): envoyAdminBind,
+		helper.CleanEnvVar(envoyReadyBindEnvPrefix+serviceName, '_'): envoyReadyBind,
 	}
 
 	// Envoy bootstrap configuration may contain a Consul token, so write
@@ -259,7 +268,7 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart
 	}
 	h.logger.Debug("check for SI token for task", "task", req.Task.Name, "exists", siToken != "")
 
-	bootstrap := h.newEnvoyBootstrapArgs(h.alloc.TaskGroup, service, grpcAddr, envoyAdminBind, siToken, bootstrapFilePath)
+	bootstrap := h.newEnvoyBootstrapArgs(h.alloc.TaskGroup, service, grpcAddr, envoyAdminBind, envoyReadyBind, siToken, bootstrapFilePath)
 	bootstrapArgs := bootstrap.args()
 	bootstrapEnv := bootstrap.env(os.Environ())
 
@@ -331,37 +340,50 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *ifs.TaskPrestart
 	return nil
 }
 
-// buildEnvoyAdminBind determines a unique port for use by the envoy admin
-// listener.
+// buildEnvoyAdminBind determines a unique port for use by the envoy admin listener.
+//
+// This listener will be bound to 127.0.0.2.
+func buildEnvoyAdminBind(alloc *structs.Allocation, service, task string, env *taskenv.TaskEnv) string {
+	return buildEnvoyBind(alloc, "127.0.0.2", service, task, env, envoyBaseAdminPort)
+}
+
+// buildEnvoyAdminBind determines a unique port for use by the envoy ready listener.
+//
+// This listener will be bound to 127.0.0.1.
+func buildEnvoyReadyBind(alloc *structs.Allocation, service, task string, env *taskenv.TaskEnv) string {
+	return buildEnvoyBind(alloc, "127.0.0.1", service, task, env, envoyBaseReadyPort)
+}
+
+// buildEnvoyBind is used to determine a unique port for an envoy listener.
 //
 // In bridge mode, if multiple sidecars are running, the bind addresses need
-// to be unique within the namespace, so we simply start at 19000 and increment
+// to be unique within the namespace, so we simply start at basePort and increment
 // by the index of the task.
 //
 // In host mode, use the port provided through the service definition, which can
 // be a port chosen by Nomad.
-func buildEnvoyAdminBind(alloc *structs.Allocation, serviceName, taskName string, taskEnv *taskenv.TaskEnv) string {
+func buildEnvoyBind(alloc *structs.Allocation, ifce, service, task string, taskEnv *taskenv.TaskEnv, basePort int) string {
 	tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
-	port := envoyBaseAdminPort
+	port := basePort
 	switch tg.Networks[0].Mode {
 	case "host":
 		interpolatedServices := taskenv.InterpolateServices(taskEnv, tg.Services)
-		for _, service := range interpolatedServices {
-			if service.Name == serviceName {
-				mapping := tg.Networks.Port(service.PortLabel)
+		for _, svc := range interpolatedServices {
+			if svc.Name == service {
+				mapping := tg.Networks.Port(svc.PortLabel)
 				port = mapping.Value
 				break
 			}
 		}
 	default:
-		for idx, task := range tg.Tasks {
-			if task.Name == taskName {
+		for idx, tgTask := range tg.Tasks {
+			if tgTask.Name == task {
 				port += idx
 				break
 			}
 		}
 	}
-	return fmt.Sprintf("localhost:%d", port)
+	return net.JoinHostPort(ifce, strconv.Itoa(port))
 }
 
 func (h *envoyBootstrapHook) writeConfig(filename, config string) error {
@@ -421,7 +443,7 @@ func (h *envoyBootstrapHook) proxyServiceID(group string, service *structs.Servi
 // https://www.consul.io/commands/connect/envoy#consul-connect-envoy
 func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
 	group string, service *structs.Service,
-	grpcAddr, envoyAdminBind, siToken, filepath string,
+	grpcAddr, envoyAdminBind, envoyReadyBind, siToken, filepath string,
 ) envoyBootstrapArgs {
 	var (
 		sidecarForID string // sidecar only
@@ -450,15 +472,16 @@ func (h *envoyBootstrapHook) newEnvoyBootstrapArgs(
 	h.logger.Info("bootstrapping envoy",
 		"sidecar_for", service.Name, "bootstrap_file", filepath,
 		"sidecar_for_id", sidecarForID, "grpc_addr", grpcAddr,
-		"admin_bind", envoyAdminBind, "gateway", gateway,
-		"proxy_id", proxyID, "namespace", namespace,
+		"admin_bind", envoyAdminBind, "ready_bind", envoyReadyBind,
+		"gateway", gateway, "proxy_id", proxyID, "namespace", namespace,
 	)
 
 	return envoyBootstrapArgs{
 		consulConfig:   h.consulConfig,
 		sidecarFor:     sidecarForID,
 		grpcAddr:       grpcAddr,
 		envoyAdminBind: envoyAdminBind,
+		envoyReadyBind: envoyReadyBind,
 		siToken:        siToken,
 		gateway:        gateway,
 		proxyID:        proxyID,
@@ -474,6 +497,7 @@ type envoyBootstrapArgs struct {
 	sidecarFor     string // sidecars only
 	grpcAddr       string
 	envoyAdminBind string
+	envoyReadyBind string
 	siToken        string
 	gateway        string // gateways only
 	proxyID        string // gateways only
@@ -489,6 +513,7 @@ func (e envoyBootstrapArgs) args() []string {
 		"-grpc-addr", e.grpcAddr,
 		"-http-addr", e.consulConfig.HTTPAddr,
 		"-admin-bind", e.envoyAdminBind,
+		"-address", e.envoyReadyBind,
 		"-bootstrap",
 	}
 

client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go

@@ -123,13 +123,15 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
 			sidecarFor:     "s1",
 			grpcAddr:       "1.1.1.1",
 			consulConfig:   consulPlainConfig,
-			envoyAdminBind: "localhost:3333",
+			envoyAdminBind: "127.0.0.2:19000",
+			envoyReadyBind: "127.0.0.1:19100",
 		}
 		result := ebArgs.args()
 		require.Equal(t, []string{"connect", "envoy",
 			"-grpc-addr", "1.1.1.1",
 			"-http-addr", "2.2.2.2",
-			"-admin-bind", "localhost:3333",
+			"-admin-bind", "127.0.0.2:19000",
+			"-address", "127.0.0.1:19100",
 			"-bootstrap",
 			"-sidecar-for", "s1",
 		}, result)
@@ -141,14 +143,16 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
 			sidecarFor:     "s1",
 			grpcAddr:       "1.1.1.1",
 			consulConfig:   consulPlainConfig,
-			envoyAdminBind: "localhost:3333",
+			envoyAdminBind: "127.0.0.2:19000",
+			envoyReadyBind: "127.0.0.1:19100",
 			siToken:        token,
 		}
 		result := ebArgs.args()
 		require.Equal(t, []string{"connect", "envoy",
 			"-grpc-addr", "1.1.1.1",
 			"-http-addr", "2.2.2.2",
-			"-admin-bind", "localhost:3333",
+			"-admin-bind", "127.0.0.2:19000",
+			"-address", "127.0.0.1:19100",
 			"-bootstrap",
 			"-sidecar-for", "s1",
 			"-token", token,
@@ -160,13 +164,15 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
 			sidecarFor:     "s1",
 			grpcAddr:       "1.1.1.1",
 			consulConfig:   consulTLSConfig,
-			envoyAdminBind: "localhost:3333",
+			envoyAdminBind: "127.0.0.2:19000",
+			envoyReadyBind: "127.0.0.1:19100",
 		}
 		result := ebArgs.args()
 		require.Equal(t, []string{"connect", "envoy",
 			"-grpc-addr", "1.1.1.1",
 			"-http-addr", "2.2.2.2",
-			"-admin-bind", "localhost:3333",
+			"-admin-bind", "127.0.0.2:19000",
+			"-address", "127.0.0.1:19100",
 			"-bootstrap",
 			"-sidecar-for", "s1",
 			"-ca-file", "/etc/tls/ca-file",
@@ -179,15 +185,17 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
 		ebArgs := envoyBootstrapArgs{
 			consulConfig:   consulPlainConfig,
 			grpcAddr:       "1.1.1.1",
-			envoyAdminBind: "localhost:3333",
+			envoyAdminBind: "127.0.0.2:19000",
+			envoyReadyBind: "127.0.0.1:19100",
 			gateway:        "my-ingress-gateway",
 			proxyID:        "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
 		}
 		result := ebArgs.args()
 		require.Equal(t, []string{"connect", "envoy",
 			"-grpc-addr", "1.1.1.1",
 			"-http-addr", "2.2.2.2",
-			"-admin-bind", "localhost:3333",
+			"-admin-bind", "127.0.0.2:19000",
+			"-address", "127.0.0.1:19100",
 			"-bootstrap",
 			"-gateway", "my-ingress-gateway",
 			"-proxy-id", "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
@@ -198,15 +206,17 @@ func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
 		ebArgs := envoyBootstrapArgs{
 			consulConfig:   consulPlainConfig,
 			grpcAddr:       "1.1.1.1",
-			envoyAdminBind: "localhost:3333",
+			envoyAdminBind: "127.0.0.2:19000",
+			envoyReadyBind: "127.0.0.1:19100",
 			gateway:        "my-mesh-gateway",
 			proxyID:        "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-mesh-mesh-8080",
 		}
 		result := ebArgs.args()
 		require.Equal(t, []string{"connect", "envoy",
 			"-grpc-addr", "1.1.1.1",
 			"-http-addr", "2.2.2.2",
-			"-admin-bind", "localhost:3333",
+			"-admin-bind", "127.0.0.2:19000",
+			"-address", "127.0.0.1:19100",
 			"-bootstrap",
 			"-gateway", "my-mesh-gateway",
 			"-proxy-id", "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-mesh-mesh-8080",
@@ -453,7 +463,7 @@ func TestTaskRunner_EnvoyBootstrapHook_sidecar_ok(t *testing.T) {
 	require.True(t, resp.Done)
 
 	require.NotNil(t, resp.Env)
-	require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])
+	require.Equal(t, "127.0.0.2:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])
 
 	// Ensure the default path matches
 	env := map[string]string{

website/content/partials/envvars.mdx

@@ -264,11 +264,23 @@
         <code>NOMAD_ENVOY_ADMIN_ADDR_&lt;service&gt;</code>
       </td>
       <td>
-        Local address <code>localhost:Port</code> for the admin port of the
+        Local address <code>127.0.0.2:Port</code> for the admin port of the
         envoy sidecar for the given <code>service</code> when defined as a
-        Consul Connect enabled service.
-      </td>
-    </tr>
+        Consul Connect enabled service. Envoy runs inside the group network
+        namespace unless configured for host networking.
+      </td>
+    </tr>
+      <tr>
+        <td>
+          <code>NOMAD_ENVOY_READY_ADDR_&lt;service&gt;</code>
+        </td>
+        <td>
+          Local address <code>127.0.0.1:Port</code> for the ready port of the
+          envoy sidecar for the given <code>service</code> when defined as a
+          Consul Connect enabled service. Envoy runs inside the group network
+          namespace unless configured for host networking.
+        </td>
+      </tr>
     <tr>
       <th colspan="2">Consul-related Variables (only set for connect native tasks)</th>
     </tr>