-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add support for late binding to IP addresses using go-sockaddr/template #2399
Changes from 1 commit
a33af1c
02af50f
a61b70b
5fb0b30
544d9e9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,7 @@ package agent | |
|
||
import ( | ||
"encoding/base64" | ||
"errors" | ||
"fmt" | ||
"io" | ||
"net" | ||
|
@@ -13,6 +14,8 @@ import ( | |
"strings" | ||
"time" | ||
|
||
"github.com/hashicorp/go-sockaddr/template" | ||
|
||
client "github.com/hashicorp/nomad/client/config" | ||
"github.com/hashicorp/nomad/helper" | ||
"github.com/hashicorp/nomad/nomad" | ||
|
@@ -692,30 +695,53 @@ func (c *Config) Merge(b *Config) *Config { | |
// normalizeAddrs normalizes Addresses and AdvertiseAddrs to always be | ||
// initialized and have sane defaults. | ||
func (c *Config) normalizeAddrs() error { | ||
c.Addresses.HTTP = normalizeBind(c.Addresses.HTTP, c.BindAddr) | ||
c.Addresses.RPC = normalizeBind(c.Addresses.RPC, c.BindAddr) | ||
c.Addresses.Serf = normalizeBind(c.Addresses.Serf, c.BindAddr) | ||
if c.BindAddr != "" { | ||
ipStr, err := parseSingleIPTemplate(c.BindAddr) | ||
if err != nil { | ||
return fmt.Errorf("Bind address resolution failed: %v", err) | ||
} | ||
c.BindAddr = ipStr | ||
} | ||
|
||
addr, err := normalizeBind(c.Addresses.HTTP, c.BindAddr) | ||
if err != nil { | ||
return fmt.Errorf("Failed to parse HTTP address: %v", err) | ||
} | ||
c.Addresses.HTTP = addr | ||
|
||
addr, err = normalizeBind(c.Addresses.RPC, c.BindAddr) | ||
if err != nil { | ||
return fmt.Errorf("Failed to parse RPC address: %v", err) | ||
} | ||
c.Addresses.RPC = addr | ||
|
||
addr, err = normalizeBind(c.Addresses.Serf, c.BindAddr) | ||
if err != nil { | ||
return fmt.Errorf("Failed to parse Serf address: %v", err) | ||
} | ||
c.Addresses.Serf = addr | ||
|
||
c.normalizedAddrs = &Addresses{ | ||
HTTP: net.JoinHostPort(c.Addresses.HTTP, strconv.Itoa(c.Ports.HTTP)), | ||
RPC: net.JoinHostPort(c.Addresses.RPC, strconv.Itoa(c.Ports.RPC)), | ||
Serf: net.JoinHostPort(c.Addresses.Serf, strconv.Itoa(c.Ports.Serf)), | ||
} | ||
|
||
addr, err := normalizeAdvertise(c.AdvertiseAddrs.HTTP, c.Addresses.HTTP, c.Ports.HTTP, c.DevMode) | ||
addr, err = normalizeAdvertise(c.AdvertiseAddrs.HTTP, c.Addresses.HTTP, c.Ports.HTTP) | ||
if err != nil { | ||
return fmt.Errorf("Failed to parse HTTP advertise address: %v", err) | ||
} | ||
c.AdvertiseAddrs.HTTP = addr | ||
|
||
addr, err = normalizeAdvertise(c.AdvertiseAddrs.RPC, c.Addresses.RPC, c.Ports.RPC, c.DevMode) | ||
addr, err = normalizeAdvertise(c.AdvertiseAddrs.RPC, c.Addresses.RPC, c.Ports.RPC) | ||
if err != nil { | ||
return fmt.Errorf("Failed to parse RPC advertise address: %v", err) | ||
} | ||
c.AdvertiseAddrs.RPC = addr | ||
|
||
// Skip serf if server is disabled | ||
if c.Server != nil && c.Server.Enabled { | ||
addr, err = normalizeAdvertise(c.AdvertiseAddrs.Serf, c.Addresses.Serf, c.Ports.Serf, c.DevMode) | ||
addr, err = normalizeAdvertise(c.AdvertiseAddrs.Serf, c.Addresses.Serf, c.Ports.Serf) | ||
if err != nil { | ||
return fmt.Errorf("Failed to parse Serf advertise address: %v", err) | ||
} | ||
|
@@ -725,14 +751,34 @@ func (c *Config) normalizeAddrs() error { | |
return nil | ||
} | ||
|
||
// parseSingleIPTemplate is used as a helper function to parse out a single IP | ||
// address from a config parameter. | ||
func parseSingleIPTemplate(ipTmpl string) (string, error) { | ||
out, err := template.Parse(ipTmpl) | ||
if err != nil { | ||
return "", fmt.Errorf("Unable to parse address template %q: %v", ipTmpl, err) | ||
} | ||
|
||
ips := strings.Split(out, " ") | ||
switch len(ips) { | ||
case 0: | ||
return "", errors.New("No addresses found, please configure one.") | ||
case 1: | ||
return ips[0], nil | ||
default: | ||
return "", fmt.Errorf("Multiple addresses found (%q), please configure one.", out) | ||
} | ||
} | ||
|
||
// normalizeBind returns a normalized bind address. | ||
// | ||
// If addr is set it is used, if not the default bind address is used. | ||
func normalizeBind(addr, bind string) string { | ||
func normalizeBind(addr, bind string) (string, error) { | ||
if addr == "" { | ||
return bind | ||
return bind, nil | ||
} else { | ||
return parseSingleIPTemplate(addr) | ||
} | ||
return addr | ||
} | ||
|
||
// normalizeAdvertise returns a normalized advertise address. | ||
|
@@ -747,61 +793,28 @@ func normalizeBind(addr, bind string) string { | |
// is resolved and returned with the port. | ||
// | ||
// Loopback is only considered a valid advertise address in dev mode. | ||
func normalizeAdvertise(addr string, bind string, defport int, dev bool) (string, error) { | ||
func normalizeAdvertise(addr string, bind string, defport int) (string, error) { | ||
if addr != "" { | ||
// Default to using manually configured address | ||
_, _, err := net.SplitHostPort(addr) | ||
host, port, err := net.SplitHostPort(addr) | ||
if err != nil { | ||
if !isMissingPort(err) { | ||
return "", fmt.Errorf("Error parsing advertise address %q: %v", addr, err) | ||
} | ||
|
||
// missing port, append the default | ||
return net.JoinHostPort(addr, strconv.Itoa(defport)), nil | ||
host = addr | ||
port = strconv.Itoa(defport) | ||
} | ||
return addr, nil | ||
} | ||
|
||
// Fallback to bind address first, and then try resolving the local hostname | ||
ips, err := net.LookupIP(bind) | ||
if err != nil { | ||
return "", fmt.Errorf("Error resolving bind address %q: %v", bind, err) | ||
} | ||
|
||
// Return the first unicast address | ||
for _, ip := range ips { | ||
if ip.IsLinkLocalUnicast() || ip.IsGlobalUnicast() { | ||
return net.JoinHostPort(ip.String(), strconv.Itoa(defport)), nil | ||
} | ||
if ip.IsLoopback() && dev { | ||
// loopback is fine for dev mode | ||
return net.JoinHostPort(ip.String(), strconv.Itoa(defport)), nil | ||
ipStr, err := parseSingleIPTemplate(host) | ||
if err != nil { | ||
return "", fmt.Errorf("Error parsing advertise address template: %v", err) | ||
} | ||
} | ||
|
||
// As a last resort resolve the hostname and use it if it's not | ||
// localhost (as localhost is never a sensible default) | ||
host, err := os.Hostname() | ||
if err != nil { | ||
return "", fmt.Errorf("Unable to get hostname to set advertise address: %v", err) | ||
} | ||
|
||
ips, err = net.LookupIP(host) | ||
if err != nil { | ||
return "", fmt.Errorf("Error resolving hostname %q for advertise address: %v", host, err) | ||
return net.JoinHostPort(ipStr, port), nil | ||
} | ||
|
||
// Return the first unicast address | ||
for _, ip := range ips { | ||
if ip.IsLinkLocalUnicast() || ip.IsGlobalUnicast() { | ||
return net.JoinHostPort(ip.String(), strconv.Itoa(defport)), nil | ||
} | ||
if ip.IsLoopback() && dev { | ||
// loopback is fine for dev mode | ||
return net.JoinHostPort(ip.String(), strconv.Itoa(defport)), nil | ||
} | ||
} | ||
return "", fmt.Errorf("No valid advertise addresses, please set `advertise` manually") | ||
// Fallback to bind address, as it has been resolved before. | ||
return net.JoinHostPort(bind, strconv.Itoa(defport)), nil | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There are 2 subtle behaviors I think your code is missing:
Since 2. doesn't work well in practice I'd be open to changing it to This is out of scope for what you're trying to do, but I also think we'll move bind to default to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FWIW, Consul will be likely making a change to its defaults in time for its 0.8 release to use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @schmichael OK, so I changed the code slightly and re-added bits from the previous version, except:
|
||
} | ||
|
||
// isMissingPort returns true if an error is a "missing port" error from | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,7 @@ | ||
package agent | ||
|
||
import ( | ||
"fmt" | ||
"io/ioutil" | ||
"net" | ||
"os" | ||
|
@@ -520,6 +521,121 @@ func TestConfig_Listener(t *testing.T) { | |
} | ||
} | ||
|
||
func TestConfig_normalizeAddrs(t *testing.T) { | ||
c := &Config{ | ||
BindAddr: "127.0.0.1", | ||
Ports: &Ports{ | ||
HTTP: 4646, | ||
RPC: 4647, | ||
Serf: 4648, | ||
}, | ||
Addresses: &Addresses{}, | ||
AdvertiseAddrs: &AdvertiseAddrs{}, | ||
DevMode: true, | ||
} | ||
|
||
if err := c.normalizeAddrs(); err != nil { | ||
t.Fatalf("unable to normalize addresses: %s", err) | ||
} | ||
|
||
if c.BindAddr != "127.0.0.1" { | ||
t.Fatalf("expected BindAddr 127.0.0.1, got %s", c.BindAddr) | ||
} | ||
|
||
if c.normalizedAddrs.HTTP != "127.0.0.1:4646" { | ||
t.Fatalf("expected HTTP address 127.0.0.1:4646, got %s", c.normalizedAddrs.HTTP) | ||
} | ||
|
||
if c.normalizedAddrs.RPC != "127.0.0.1:4647" { | ||
t.Fatalf("expected RPC address 127.0.0.1:4647, got %s", c.normalizedAddrs.RPC) | ||
} | ||
|
||
if c.normalizedAddrs.Serf != "127.0.0.1:4648" { | ||
t.Fatalf("expected Serf address 127.0.0.1:4648, got %s", c.normalizedAddrs.Serf) | ||
} | ||
|
||
if c.AdvertiseAddrs.HTTP != "127.0.0.1:4646" { | ||
t.Fatalf("expected HTTP advertise address 127.0.0.1:4646, got %s", c.AdvertiseAddrs.HTTP) | ||
} | ||
|
||
if c.AdvertiseAddrs.RPC != "127.0.0.1:4647" { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah, we don't want this to happen unless AdvertiseAddrs.RPC is explicitly set to localhost. The reason is advertising localhost any time other than for a local test node can do Very Bad Things. The worst case scenario is having a server agent advertise localhost to the cluster - even briefly! Nodes will spam their own We could add better heuristics for advertising localhost... like only disallow it if There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok, but the configuration explicitly enabled dev mode in this case, so ... I guess it's OK to advertise 127.0.0.1 here? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. re: dev mode, yes, that sounds good and is a sane approach. |
||
t.Fatalf("expected RPC advertise address 127.0.0.1:4647, got %s", c.AdvertiseAddrs.RPC) | ||
} | ||
|
||
// Client mode, no Serf address defined | ||
if c.AdvertiseAddrs.Serf != "" { | ||
t.Fatalf("expected unset Serf advertise address, got %s", c.AdvertiseAddrs.Serf) | ||
} | ||
|
||
c = &Config{ | ||
BindAddr: "169.254.1.5", | ||
Ports: &Ports{ | ||
HTTP: 4646, | ||
RPC: 4647, | ||
Serf: 4648, | ||
}, | ||
Addresses: &Addresses{ | ||
HTTP: "169.254.1.10", | ||
}, | ||
AdvertiseAddrs: &AdvertiseAddrs{ | ||
RPC: "169.254.1.40", | ||
}, | ||
Server: &ServerConfig{ | ||
Enabled: true, | ||
}, | ||
} | ||
|
||
if err := c.normalizeAddrs(); err != nil { | ||
t.Fatalf("unable to normalize addresses: %s", err) | ||
} | ||
|
||
if c.BindAddr != "169.254.1.5" { | ||
t.Fatalf("expected BindAddr 169.254.1.5, got %s", c.BindAddr) | ||
} | ||
|
||
if c.AdvertiseAddrs.HTTP != "169.254.1.10:4646" { | ||
t.Fatalf("expected HTTP advertise address 169.254.1.10:4646, got %s", c.AdvertiseAddrs.HTTP) | ||
} | ||
|
||
if c.AdvertiseAddrs.RPC != "169.254.1.40:4647" { | ||
t.Fatalf("expected RPC advertise address 169.254.1.40:4647, got %s", c.AdvertiseAddrs.RPC) | ||
} | ||
|
||
if c.AdvertiseAddrs.Serf != "169.254.1.5:4648" { | ||
t.Fatalf("expected Serf advertise address 169.254.1.5:4648, got %s", c.AdvertiseAddrs.Serf) | ||
} | ||
|
||
c = &Config{ | ||
BindAddr: "{{ GetPrivateIP }}", | ||
Ports: &Ports{ | ||
HTTP: 4646, | ||
RPC: 4647, | ||
Serf: 4648, | ||
}, | ||
Addresses: &Addresses{}, | ||
AdvertiseAddrs: &AdvertiseAddrs{}, | ||
Server: &ServerConfig{ | ||
Enabled: true, | ||
}, | ||
} | ||
|
||
if err := c.normalizeAddrs(); err != nil { | ||
t.Fatalf("unable to normalize addresses: %s", err) | ||
} | ||
|
||
if c.AdvertiseAddrs.HTTP != fmt.Sprintf("%s:4646", c.BindAddr) { | ||
t.Fatalf("expected HTTP advertise address %s:4646, got %s", c.BindAddr, c.AdvertiseAddrs.HTTP) | ||
} | ||
|
||
if c.AdvertiseAddrs.RPC != fmt.Sprintf("%s:4647", c.BindAddr) { | ||
t.Fatalf("expected RPC advertise address %s:4647, got %s", c.BindAddr, c.AdvertiseAddrs.RPC) | ||
} | ||
|
||
if c.AdvertiseAddrs.Serf != fmt.Sprintf("%s:4648", c.BindAddr) { | ||
t.Fatalf("expected Serf advertise address %s:4648, got %s", c.BindAddr, c.AdvertiseAddrs.Serf) | ||
} | ||
} | ||
|
||
func TestResources_ParseReserved(t *testing.T) { | ||
cases := []struct { | ||
Input string | ||
|
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think you need to parse the template before attempting to SplitHostPort as SplitHostPort may fail on valid templates.