agent: support multiple http address in addresses.http (#11582)

command/agent/agent.go

@@ -935,7 +935,7 @@ func (a *Agent) setupClient() error {
 // If no HTTP health check can be supported nil is returned.
 func (a *Agent) agentHTTPCheck(server bool) *structs.ServiceCheck {
 	// Resolve the http check address
-	httpCheckAddr := a.config.normalizedAddrs.HTTP
+	httpCheckAddr := a.config.normalizedAddrs.HTTP[0]
 	if *a.config.Consul.ChecksUseAdvertise {
 		httpCheckAddr = a.config.AdvertiseAddrs.HTTP
 	}

command/agent/agent_test.go

@@ -105,7 +105,7 @@ func TestAgent_ServerConfig(t *testing.T) {
 	require.Equal(t, "127.0.0.2", conf.Addresses.HTTP)
 	require.Equal(t, "127.0.0.2", conf.Addresses.RPC)
 	require.Equal(t, "127.0.0.2", conf.Addresses.Serf)
-	require.Equal(t, "127.0.0.2:4646", conf.normalizedAddrs.HTTP)
+	require.Equal(t, []string{"127.0.0.2:4646"}, conf.normalizedAddrs.HTTP)
 	require.Equal(t, "127.0.0.2:4003", conf.normalizedAddrs.RPC)
 	require.Equal(t, "127.0.0.2:4004", conf.normalizedAddrs.Serf)
 	require.Equal(t, "10.0.0.10:4646", conf.AdvertiseAddrs.HTTP)
@@ -166,7 +166,7 @@ func TestAgent_ServerConfig(t *testing.T) {
 	require.Equal(t, "127.0.0.3", conf.Addresses.HTTP)
 	require.Equal(t, "127.0.0.3", conf.Addresses.RPC)
 	require.Equal(t, "127.0.0.3", conf.Addresses.Serf)
-	require.Equal(t, "127.0.0.3:4646", conf.normalizedAddrs.HTTP)
+	require.Equal(t, []string{"127.0.0.3:4646"}, conf.normalizedAddrs.HTTP)
 	require.Equal(t, "127.0.0.3:4647", conf.normalizedAddrs.RPC)
 	require.Equal(t, "127.0.0.3:4648", conf.normalizedAddrs.Serf)
 
@@ -563,7 +563,7 @@ func TestAgent_HTTPCheck(t *testing.T) {
 			logger: logger,
 			config: &Config{
 				AdvertiseAddrs:  &AdvertiseAddrs{HTTP: "advertise:4646"},
-				normalizedAddrs: &Addresses{HTTP: "normalized:4646"},
+				normalizedAddrs: &NormalizedAddrs{HTTP: []string{"normalized:4646"}},
 				Consul: &config.ConsulConfig{
 					ChecksUseAdvertise: helper.BoolToPtr(false),
 				},
@@ -587,7 +587,7 @@ func TestAgent_HTTPCheck(t *testing.T) {
 		if check.Protocol != "http" {
 			t.Errorf("expected http proto not: %q", check.Protocol)
 		}
-		if expected := a.config.normalizedAddrs.HTTP; check.PortLabel != expected {
+		if expected := a.config.normalizedAddrs.HTTP[0]; check.PortLabel != expected {
 			t.Errorf("expected normalized addr not %q", check.PortLabel)
 		}
 	})

command/agent/command.go

@@ -51,7 +51,7 @@ type Command struct {
 
 	args           []string
 	agent          *Agent
-	httpServer     *HTTPServer
+	httpServers    []*HTTPServer
 	logFilter      *logutils.LevelFilter
 	logOutput      io.Writer
 	retryJoinErrCh chan struct{}
@@ -500,13 +500,13 @@ func (c *Command) setupAgent(config *Config, logger hclog.InterceptLogger, logOu
 	c.agent = agent
 
 	// Setup the HTTP server
-	http, err := NewHTTPServer(agent, config)
+	httpServers, err := NewHTTPServers(agent, config)
 	if err != nil {
 		agent.Shutdown()
 		c.Ui.Error(fmt.Sprintf("Error starting http server: %s", err))
 		return err
 	}
-	c.httpServer = http
+	c.httpServers = httpServers
 
 	// If DisableUpdateCheck is not enabled, set up update checking
 	// (DisableUpdateCheck is false by default)
@@ -700,8 +700,10 @@ func (c *Command) Run(args []string) int {
 
 		// Shutdown the http server at the end, to ease debugging if
 		// the agent takes long to shutdown
-		if c.httpServer != nil {
-			c.httpServer.Shutdown()
+		if len(c.httpServers) > 0 {
+			for _, srv := range c.httpServers {
+				srv.Shutdown()
+			}
 		}
 	}()
 
@@ -902,13 +904,15 @@ WAIT:
 func (c *Command) reloadHTTPServer() error {
 	c.agent.logger.Info("reloading HTTP server with new TLS configuration")
 
-	c.httpServer.Shutdown()
+	for _, srv := range c.httpServers {
+		srv.Shutdown()
+	}
 
-	http, err := NewHTTPServer(c.agent, c.agent.config)
+	httpServers, err := NewHTTPServers(c.agent, c.agent.config)
 	if err != nil {
 		return err
 	}
-	c.httpServer = http
+	c.httpServers = httpServers
 
 	return nil
 }

command/agent/config.go

@@ -85,7 +85,7 @@ type Config struct {
 	Addresses *Addresses `hcl:"addresses"`
 
 	// normalizedAddr is set to the Address+Port by normalizeAddrs()
-	normalizedAddrs *Addresses
+	normalizedAddrs *NormalizedAddrs
 
 	// AdvertiseAddrs is used to control the addresses we advertise.
 	AdvertiseAddrs *AdvertiseAddrs `hcl:"advertise"`
@@ -774,6 +774,15 @@ type Addresses struct {
 	ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"`
 }
 
+// AdvertiseAddrs is used to control the addresses we advertise out for
+// different network services. All are optional and default to BindAddr and
+// their default Port.
+type NormalizedAddrs struct {
+	HTTP []string
+	RPC  string
+	Serf string
+}
+
 // AdvertiseAddrs is used to control the addresses we advertise out for
 // different network services. All are optional and default to BindAddr and
 // their default Port.
@@ -1228,13 +1237,13 @@ func (c *Config) normalizeAddrs() error {
 		c.BindAddr = ipStr
 	}
 
-	addr, err := normalizeBind(c.Addresses.HTTP, c.BindAddr)
+	httpAddrs, err := normalizeMultipleBind(c.Addresses.HTTP, c.BindAddr)
 	if err != nil {
 		return fmt.Errorf("Failed to parse HTTP address: %v", err)
 	}
-	c.Addresses.HTTP = addr
+	c.Addresses.HTTP = strings.Join(httpAddrs, " ")
 
-	addr, err = normalizeBind(c.Addresses.RPC, c.BindAddr)
+	addr, err := normalizeBind(c.Addresses.RPC, c.BindAddr)
 	if err != nil {
 		return fmt.Errorf("Failed to parse RPC address: %v", err)
 	}
@@ -1246,13 +1255,13 @@ func (c *Config) normalizeAddrs() error {
 	}
 	c.Addresses.Serf = addr
 
-	c.normalizedAddrs = &Addresses{
-		HTTP: net.JoinHostPort(c.Addresses.HTTP, strconv.Itoa(c.Ports.HTTP)),
+	c.normalizedAddrs = &NormalizedAddrs{
+		HTTP: joinHostPorts(httpAddrs, strconv.Itoa(c.Ports.HTTP)),
 		RPC:  net.JoinHostPort(c.Addresses.RPC, strconv.Itoa(c.Ports.RPC)),
 		Serf: net.JoinHostPort(c.Addresses.Serf, strconv.Itoa(c.Ports.Serf)),
 	}
 
-	addr, err = normalizeAdvertise(c.AdvertiseAddrs.HTTP, c.Addresses.HTTP, c.Ports.HTTP, c.DevMode)
+	addr, err = normalizeAdvertise(c.AdvertiseAddrs.HTTP, httpAddrs[0], c.Ports.HTTP, c.DevMode)
 	if err != nil {
 		return fmt.Errorf("Failed to parse HTTP advertise address (%v, %v, %v, %v): %v", c.AdvertiseAddrs.HTTP, c.Addresses.HTTP, c.Ports.HTTP, c.DevMode, err)
 	}
@@ -1335,6 +1344,22 @@ func parseSingleIPTemplate(ipTmpl string) (string, error) {
 	}
 }
 
+// parseMultipleIPTemplate is used as a helper function to parse out a multiple IP
+// addresses from a config parameter.
+func parseMultipleIPTemplate(ipTmpl string) ([]string, error) {
+	out, err := template.Parse(ipTmpl)
+	if err != nil {
+		return []string{}, fmt.Errorf("Unable to parse address template %q: %v", ipTmpl, err)
+	}
+
+	ips := strings.Split(out, " ")
+	if len(ips) == 0 {
+		return []string{}, errors.New("No addresses found, please configure one.")
+	}
+
+	return deduplicateAddrs(ips), nil
+}
+
 // normalizeBind returns a normalized bind address.
 //
 // If addr is set it is used, if not the default bind address is used.
@@ -1345,6 +1370,16 @@ func normalizeBind(addr, bind string) (string, error) {
 	return parseSingleIPTemplate(addr)
 }
 
+// normalizeMultipleBind returns normalized bind addresses.
+//
+// If addr is set it is used, if not the default bind address is used.
+func normalizeMultipleBind(addr, bind string) ([]string, error) {
+	if addr == "" {
+		return []string{bind}, nil
+	}
+	return parseMultipleIPTemplate(addr)
+}
+
 // normalizeAdvertise returns a normalized advertise address.
 //
 // If addr is set, it is used and the default port is appended if no port is
@@ -1996,6 +2031,17 @@ func LoadConfigDir(dir string) (*Config, error) {
 	return result, nil
 }
 
+// joinHostPorts joins every addr in addrs with the specified port
+func joinHostPorts(addrs []string, port string) []string {
+	localAddrs := make([]string, len(addrs))
+	for i, k := range addrs {
+		localAddrs[i] = net.JoinHostPort(k, port)
+
+	}
+
+	return localAddrs
+}
+
 // isTemporaryFile returns true or false depending on whether the
 // provided file name is a temporary file for the following editors:
 // emacs or vim.
@@ -2004,3 +2050,16 @@ func isTemporaryFile(name string) bool {
 		strings.HasPrefix(name, ".#") || // emacs
 		(strings.HasPrefix(name, "#") && strings.HasSuffix(name, "#")) // emacs
 }
+
+func deduplicateAddrs(addrs []string) []string {
+	keys := make(map[string]bool)
+	list := []string{}
+
+	for _, entry := range addrs {
+		if _, value := keys[entry]; !value {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+	return list
+}

command/agent/config_test.go

@@ -747,7 +747,7 @@ func TestConfig_normalizeAddrs_DevMode(t *testing.T) {
 		t.Fatalf("expected BindAddr 127.0.0.1, got %s", c.BindAddr)
 	}
 
-	if c.normalizedAddrs.HTTP != "127.0.0.1:4646" {
+	if c.normalizedAddrs.HTTP[0] != "127.0.0.1:4646" {
 		t.Fatalf("expected HTTP address 127.0.0.1:4646, got %s", c.normalizedAddrs.HTTP)
 	}
 
@@ -880,6 +880,55 @@ func TestConfig_normalizeAddrs_IPv6Loopback(t *testing.T) {
 	}
 }
 
+// TestConfig_normalizeAddrs_MultipleInterface asserts that normalizeAddrs will
+// handle normalizing multiple interfaces in a single protocol.
+func TestConfig_normalizeAddrs_MultipleInterfaces(t *testing.T) {
+	testCases := []struct {
+		name                    string
+		addressConfig           *Addresses
+		expectedNormalizedAddrs *NormalizedAddrs
+		expectErr               bool
+	}{
+		{
+			name: "multiple http addresses",
+			addressConfig: &Addresses{
+				HTTP: "127.0.0.1 127.0.0.2",
+			},
+			expectedNormalizedAddrs: &NormalizedAddrs{
+				HTTP: []string{"127.0.0.1:4646", "127.0.0.2:4646"},
+				RPC:  "127.0.0.1:4647",
+				Serf: "127.0.0.1:4648",
+			},
+			expectErr: false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			c := &Config{
+				BindAddr: "127.0.0.1",
+				Ports: &Ports{
+					HTTP: 4646,
+					RPC:  4647,
+					Serf: 4648,
+				},
+				Addresses: tc.addressConfig,
+				AdvertiseAddrs: &AdvertiseAddrs{
+					HTTP: "127.0.0.1",
+					RPC:  "127.0.0.1",
+					Serf: "127.0.0.1",
+				},
+			}
+			err := c.normalizeAddrs()
+			if tc.expectErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, tc.expectedNormalizedAddrs, c.normalizedAddrs)
+		})
+	}
+}
+
 func TestConfig_normalizeAddrs(t *testing.T) {
 	c := &Config{
 		BindAddr: "169.254.1.5",
@@ -1315,3 +1364,26 @@ func TestEventBroker_Parse(t *testing.T) {
 		require.Equal(20000, *result.EventBufferSize)
 	}
 }
+
+func TestParseMultipleIPTemplates(t *testing.T) {
+	testCases := []struct {
+		name        string
+		tmpl        string
+		expectedOut []string
+		expectErr   bool
+	}{
+		{
+			name:        "deduplicates same ip",
+			tmpl:        "127.0.0.1 127.0.0.1",
+			expectedOut: []string{"127.0.0.1"},
+			expectErr:   false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			out, err := parseMultipleIPTemplate(tc.tmpl)
+			require.NoError(t, err)
+			require.Equal(t, tc.expectedOut, out)
+		})
+	}
+}