config: support domain:port as well as ip:port (#926)

dm/master/config.go

@@ -224,8 +224,8 @@ func (c *Config) configFromFile(path string) error {
 
 // adjust adjusts configs
 func (c *Config) adjust() error {
-	// MasterAddr's format may be "host:port" or ":port"
-	host, port, err := net.SplitHostPort(c.MasterAddr)
+	// MasterAddr's format may be "scheme://host:port", "host:port" or ":port"
+	host, port, err := net.SplitHostPort(utils.UnwrapScheme(c.MasterAddr))
 	if err != nil {
 		return terror.ErrMasterHostPortNotValid.Delegate(err, c.MasterAddr)
 	}
@@ -236,8 +236,8 @@ func (c *Config) adjust() error {
 		}
 		c.AdvertiseAddr = c.MasterAddr
 	} else {
-		// AdvertiseAddr's format must be "host:port"
-		host, port, err = net.SplitHostPort(c.AdvertiseAddr)
+		// AdvertiseAddr's format may be "scheme://host:port" or "host:port"
+		host, port, err = net.SplitHostPort(utils.UnwrapScheme(c.AdvertiseAddr))
 		if err != nil {
 			return terror.ErrMasterAdvertiseAddrNotValid.Delegate(err, c.AdvertiseAddr)
 		}
@@ -407,18 +407,15 @@ func parseURLs(s string) ([]url.URL, error) {
 	items := strings.Split(s, ",")
 	urls := make([]url.URL, 0, len(items))
 	for _, item := range items {
-		u, err := url.Parse(item)
-		// tolerate valid `master-addr`, but invalid URL format, like:
-		// `:8261`: missing protocol scheme
-		// `127.0.0.1:8261`: first path segment in URL cannot contain colon
-		if err != nil && (strings.Contains(err.Error(), "missing protocol scheme") ||
-			strings.Contains(err.Error(), "first path segment in URL cannot contain colon")) {
+		// tolerate valid `master-addr`, but invalid URL format. mainly caused by no protocol scheme
+		if !(strings.HasPrefix(item, "http://") || strings.HasPrefix(item, "https://")) {
 			prefix := "http://"
 			if atomic.LoadInt32(&useTLS) == 1 {
 				prefix = "https://"
 			}
-			u, err = url.Parse(prefix + item)
+			item = prefix + item
 		}
+		u, err := url.Parse(item)
 		if err != nil {
 			return nil, terror.ErrMasterParseURLFail.Delegate(err, item)
 		}

dm/master/config_test.go

@@ -345,6 +345,12 @@ func (t *testConfigSuite) TestParseURLs(c *check.C) {
 				{Scheme: "http", Host: "127.0.0.1:18291"},
 			},
 		},
+		{
+			str: "LancedeMacBook-Pro.local:8661",
+			urls: []url.URL{
+				{Scheme: "http", Host: "LancedeMacBook-Pro.local:8661"},
+			},
+		},
 	}
 
 	for _, cs := range cases {

dm/worker/config.go

@@ -169,7 +169,7 @@ func (c *Config) Parse(arguments []string) error {
 
 // adjust adjusts the config.
 func (c *Config) adjust() error {
-	host, port, err := net.SplitHostPort(c.WorkerAddr)
+	host, port, err := net.SplitHostPort(utils.UnwrapScheme(c.WorkerAddr))
 	if err != nil {
 		return terror.ErrWorkerHostPortNotValid.Delegate(err, c.WorkerAddr)
 	}
@@ -180,7 +180,7 @@ func (c *Config) adjust() error {
 		}
 		c.AdvertiseAddr = c.WorkerAddr
 	} else {
-		host, port, err = net.SplitHostPort(c.AdvertiseAddr)
+		host, port, err = net.SplitHostPort(utils.UnwrapScheme(c.AdvertiseAddr))
 		if err != nil {
 			return terror.ErrWorkerHostPortNotValid.Delegate(err, c.AdvertiseAddr)
 		}

pkg/utils/util.go

@@ -170,3 +170,13 @@ func HidePassword(input string) string {
 	output := passwordRegexp.ReplaceAllString(input, "$1******$4")
 	return output
 }
+
+// UnwrapScheme removes http or https scheme from input
+func UnwrapScheme(s string) string {
+	if strings.HasPrefix(s, "http://") {
+		return s[len("http://"):]
+	} else if strings.HasPrefix(s, "https://") {
+		return s[len("https://"):]
+	}
+	return s
+}

pkg/utils/util_test.go

@@ -194,3 +194,30 @@ func (t *testUtilsSuite) TestHidePassword(c *C) {
 		c.Assert(HidePassword(str.old), Equals, str.new)
 	}
 }
+
+func (t *testUtilsSuite) TestUnwrapScheme(c *C) {
+	cases := []struct {
+		old string
+		new string
+	}{
+		{
+			"http://0.0.0.0:123",
+			"0.0.0.0:123",
+		},
+		{
+			"https://0.0.0.0:123",
+			"0.0.0.0:123",
+		},
+		{
+			"http://abc.com:123",
+			"abc.com:123",
+		},
+		{
+			"httpsdfpoje.com",
+			"httpsdfpoje.com",
+		},
+	}
+	for _, ca := range cases {
+		c.Assert(UnwrapScheme(ca.old), Equals, ca.new)
+	}
+}

tests/ha_master/conf/diff_config.toml

@@ -38,21 +38,21 @@ schema = "ha_master_test"
 table  = "t2"
 
 [[source-db]]
-host = "127.0.0.1"
+host = "localhost"
 port = 3306
 user = "root"
 password = "123456"
 instance-id = "source-1"
 
 [[source-db]]
-host = "127.0.0.1"
+host = "localhost"
 port = 3307
 user = "root"
 password = "123456"
 instance-id = "source-2"
 
 [target-db]
-host = "127.0.0.1"
+host = "localhost"
 port = 4000
 user = "test"
 password = "123456"

tests/ha_master/conf/dm-master-join1.toml

@@ -1,6 +1,6 @@
 # Master Configuration.
 name = "master1"
 master-addr = ":8261"
-advertise-addr = "127.0.0.1:8261"
-peer-urls = "http://127.0.0.1:8291"
-join = "127.0.0.1:8461,127.0.0.1:8561,127.0.0.1:8661"
+advertise-addr = "localhost:8261"
+peer-urls = "http://localhost:8291"
+join = "localhost:8461,localhost:8561,localhost:8661"

tests/ha_master/conf/dm-master1.toml

@@ -1,6 +1,6 @@
 # Master Configuration.
 name = "master1"
 master-addr = ":8261"
-advertise-addr = "127.0.0.1:8261"
-peer-urls = "127.0.0.1:8291"
-initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293,master4=http://127.0.0.1:8294,master5=http://127.0.0.1:8295"
+advertise-addr = "localhost:8261"
+peer-urls = "localhost:8291"
+initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"

tests/ha_master/conf/dm-master2.toml

@@ -1,6 +1,6 @@
 # Master Configuration.
 name = "master2"
 master-addr = ":8361"
-advertise-addr = "127.0.0.1:8361"
-peer-urls = "127.0.0.1:8292"
-initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293,master4=http://127.0.0.1:8294,master5=http://127.0.0.1:8295"
+advertise-addr = "localhost:8361"
+peer-urls = "localhost:8292"
+initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"

tests/ha_master/conf/dm-master3.toml

@@ -1,6 +1,6 @@
 # Master Configuration.
 name = "master3"
 master-addr = ":8461"
-advertise-addr = "127.0.0.1:8461"
-peer-urls = "127.0.0.1:8293"
-initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293,master4=http://127.0.0.1:8294,master5=http://127.0.0.1:8295"
+advertise-addr = "localhost:8461"
+peer-urls = "localhost:8293"
+initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"

tests/ha_master/conf/dm-master4.toml

@@ -1,6 +1,6 @@
 # Master Configuration.
 name = "master4"
 master-addr = ":8561"
-advertise-addr = "127.0.0.1:8561"
-peer-urls = "127.0.0.1:8294"
-initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293,master4=http://127.0.0.1:8294,master5=http://127.0.0.1:8295"
+advertise-addr = "localhost:8561"
+peer-urls = "localhost:8294"
+initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"

tests/ha_master/conf/dm-master5.toml

@@ -1,6 +1,6 @@
 # Master Configuration.
 name = "master5"
 master-addr = ":8661"
-advertise-addr = "127.0.0.1:8661"
-peer-urls = "127.0.0.1:8295"
-initial-cluster = "master1=http://127.0.0.1:8291,master2=http://127.0.0.1:8292,master3=http://127.0.0.1:8293,master4=http://127.0.0.1:8294,master5=http://127.0.0.1:8295"
+advertise-addr = "localhost:8661"
+peer-urls = "localhost:8295"
+initial-cluster = "master1=http://localhost:8291,master2=http://localhost:8292,master3=http://localhost:8293,master4=http://localhost:8294,master5=http://localhost:8295"

tests/ha_master/conf/dm-task.yaml

@@ -9,7 +9,7 @@ heartbeat-report-interval: 1
 timezone: "Asia/Shanghai"
 
 target-database:
-  host: "127.0.0.1"
+  host: "localhost"
   port: 4000
   user: "root"
   password: ""

tests/ha_master/conf/dm-worker1.toml

@@ -1,2 +1,2 @@
 name = "worker1"
-join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461,127.0.0.1:8561,127.0.0.1:8661"
+join = "localhost:8261,localhost:8361,localhost:8461,localhost:8561,localhost:8661"

tests/ha_master/conf/dm-worker2.toml

@@ -1,2 +1,2 @@
 name = "worker2"
-join = "127.0.0.1:8261,127.0.0.1:8361,127.0.0.1:8461,127.0.0.1:8561,127.0.0.1:8661"
+join = "localhost:8261,localhost:8361,localhost:8461,localhost:8561,localhost:8661"