diff --git a/command/agent/agent.go b/command/agent/agent.go
index 6deeb234278b..5ac72eec901c 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -537,29 +537,40 @@ func (a *Agent) getSerfAddr(bind bool) (*net.TCPAddr, error) {
 	bindAddr := a.config.Addresses.Serf
 	globalBindAddr := a.config.BindAddr
 	port := a.config.Ports.Serf
-	return pickAddress(bind, globalBindAddr, advertAddr, bindAddr, port, "RPC")
+	return pickAddress(bind, globalBindAddr, advertAddr, bindAddr, port, "Serf")
 }
 
 // pickAddress is a shared helper to pick the address to either bind to or
 // advertise.
 func pickAddress(bind bool, globalBindAddr, advertiseAddr, bindAddr string, port int, service string) (*net.TCPAddr, error) {
-	portConverted := strconv.Itoa(port)
 	var serverAddr string
 	if advertiseAddr != "" && !bind {
 		serverAddr = advertiseAddr
+
+		// Check if the advertise has a port
+		if host, pport, err := net.SplitHostPort(advertiseAddr); err == nil {
+			if parsed, err := strconv.Atoi(pport); err == nil {
+				serverAddr = host
+				port = parsed
+			}
+		}
 	} else if bindAddr != "" && !(bindAddr == "0.0.0.0" && !bind) {
-		serverAddr = net.JoinHostPort(bindAddr, portConverted)
+		serverAddr = bindAddr
 	} else if globalBindAddr != "" && !(globalBindAddr == "0.0.0.0" && !bind) {
-		serverAddr = net.JoinHostPort(globalBindAddr, portConverted)
+		serverAddr = globalBindAddr
 	} else {
-		serverAddr = net.JoinHostPort("127.0.0.1", portConverted)
+		serverAddr = "127.0.0.1"
 	}
 
-	addr, err := net.ResolveTCPAddr("tcp", serverAddr)
-	if err != nil {
-		return nil, fmt.Errorf("error resolving %s addr %+q: %v", service, serverAddr, err)
+	ip := net.ParseIP(serverAddr)
+	if ip == nil {
+		return nil, fmt.Errorf("Failed to parse %s address: %q", service, serverAddr)
 	}
-	return addr, nil
+
+	return &net.TCPAddr{
+		IP:   ip,
+		Port: port,
+	}, nil
 }
 
 // reservePortsForClient reserves a range of ports for the client to use when
diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go
index 3022df7c0848..53420aa93469 100644
--- a/command/agent/agent_test.go
+++ b/command/agent/agent_test.go
@@ -98,7 +98,7 @@ func TestAgent_ServerConfig(t *testing.T) {
 	// Returns error on bad serf addr
 	conf.AdvertiseAddrs.Serf = "nope"
 	_, err := a.serverConfig()
-	if err == nil || !strings.Contains(err.Error(), "error resolving Serf addr") {
+	if err == nil || !strings.Contains(err.Error(), "Failed to parse Serf address") {
 		t.Fatalf("expected serf address error, got: %#v", err)
 	}
 	conf.AdvertiseAddrs.Serf = "127.0.0.1:4000"
@@ -106,7 +106,7 @@ func TestAgent_ServerConfig(t *testing.T) {
 	// Returns error on bad rpc addr
 	conf.AdvertiseAddrs.RPC = "nope"
 	_, err = a.serverConfig()
-	if err == nil || !strings.Contains(err.Error(), "error resolving RPC addr") {
+	if err == nil || !strings.Contains(err.Error(), "Failed to parse RPC address") {
 		t.Fatalf("expected rpc address error, got: %#v", err)
 	}
 	conf.AdvertiseAddrs.RPC = "127.0.0.1:4001"
@@ -155,7 +155,7 @@ func TestAgent_ServerConfig(t *testing.T) {
 	conf.Addresses.RPC = "127.0.0.2"
 	conf.Addresses.Serf = "127.0.0.2"
 	conf.Addresses.HTTP = "127.0.0.2"
-	conf.AdvertiseAddrs.HTTP = "10.0.0.10:4646"
+	conf.AdvertiseAddrs.HTTP = "10.0.0.10"
 	conf.AdvertiseAddrs.RPC = ""
 	conf.AdvertiseAddrs.Serf = "10.0.0.12:4004"