From 4c77726914401640f03b1371cf011bdf554c20f5 Mon Sep 17 00:00:00 2001 From: Edwin Xie Date: Fri, 24 May 2024 18:16:50 +0000 Subject: [PATCH] Implement flag --experimental-set-member-localaddr Which sets the LocalAddr to an IP address from --initial-advertise-peer-urls. Also adds e2e test that requires this flag to succeed. Co-authored-by: HighPon Signed-off-by: Edwin Xie --- CHANGELOG/CHANGELOG-3.6.md | 1 + client/pkg/transport/listener.go | 3 + client/pkg/transport/transport.go | 11 ++- server/config/config.go | 3 + server/embed/config.go | 43 ++++++++++ server/embed/config_test.go | 132 ++++++++++++++++++++++++++++++ server/embed/etcd.go | 5 ++ server/etcdmain/help.go | 2 + tests/e2e/etcd_config_test.go | 110 +++++++++++++++++++++++++ tests/e2e/utils.go | 126 ++++++++++++++++++++++++++++ 10 files changed, 435 insertions(+), 1 deletion(-) diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md index 078d2b75455..8df0b1cbaeb 100644 --- a/CHANGELOG/CHANGELOG-3.6.md +++ b/CHANGELOG/CHANGELOG-3.6.md @@ -75,6 +75,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). - Decreased [`--snapshot-count` default value from 100,000 to 10,000](https://github.com/etcd-io/etcd/pull/15408) - Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15156) to enable support for TLS 1.3. - Add [quota to endpoint status response](https://github.com/etcd-io/etcd/pull/17877) +- Add ['etcd --experimental-set-member-localaddr'](https://github.com/etcd-io/etcd/pull/17661) to enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer. ### etcd grpc-proxy diff --git a/client/pkg/transport/listener.go b/client/pkg/transport/listener.go index 69c23e50d35..bbb84af4b58 100644 --- a/client/pkg/transport/listener.go +++ b/client/pkg/transport/listener.go @@ -193,6 +193,9 @@ type TLSInfo struct { // EmptyCN indicates that the cert must have empty CN. // If true, ClientConfig() will return an error for a cert with non empty CN. EmptyCN bool + + // LocalAddr is the local IP address to use when communicating with a peer. + LocalAddr string } func (info TLSInfo) String() string { diff --git a/client/pkg/transport/transport.go b/client/pkg/transport/transport.go index 91462dcdb08..67170d7436d 100644 --- a/client/pkg/transport/transport.go +++ b/client/pkg/transport/transport.go @@ -30,10 +30,19 @@ func NewTransport(info TLSInfo, dialtimeoutd time.Duration) (*http.Transport, er return nil, err } + var ipAddr net.Addr + if info.LocalAddr != "" { + ipAddr, err = net.ResolveTCPAddr("tcp", info.LocalAddr+":0") + if err != nil { + return nil, err + } + } + t := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ - Timeout: dialtimeoutd, + Timeout: dialtimeoutd, + LocalAddr: ipAddr, // value taken from http.DefaultTransport KeepAlive: 30 * time.Second, }).DialContext, diff --git a/server/config/config.go b/server/config/config.go index 14e8fa3df98..d8edc514f45 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -204,6 +204,9 @@ type ServerConfig struct { // V2Deprecation defines a phase of v2store deprecation process. V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` + + // ExperimentalLocalAddress is the local IP address to use when communicating with a peer. + ExperimentalLocalAddress string `json:"experimental-local-address"` } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/server/embed/config.go b/server/embed/config.go index 8f304ea62a3..6a4dac475b9 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -22,6 +22,7 @@ import ( "math" "net" "net/http" + "net/netip" "net/url" "os" "path/filepath" @@ -225,6 +226,12 @@ type Config struct { ClientAutoTLS bool PeerTLSInfo transport.TLSInfo PeerAutoTLS bool + + // ExperimentalSetMemberLocalAddr enables using the first specified and + // non-loopback local address from initial-advertise-peer-urls as the local + // address when communicating with a peer. + ExperimentalSetMemberLocalAddr bool `json:"experimental-set-member-localaddr"` + // SelfSignedCertValidity specifies the validity period of the client and peer certificates // that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS, // the unit is year, and the default is 1 @@ -621,6 +628,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { "initial-advertise-peer-urls", "List of this member's peer URLs to advertise to the rest of the cluster.", ) + fs.BoolVar(&cfg.ExperimentalSetMemberLocalAddr, "experimental-set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.") + fs.Var( flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""), "advertise-client-urls", @@ -1148,6 +1157,40 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) { return ret[1:] } +// InferLocalAddr tries to determine the LocalAddr used when communicating with +// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host +// from AdvertisePeerUrls by searching for the first URL with a specified +// non-loopback address. Otherwise, it defaults to empty string and the +// LocalAddr used will be the default for the Golang HTTP client. +func (cfg *Config) InferLocalAddr() string { + if !cfg.ExperimentalSetMemberLocalAddr { + return "" + } + + lg := cfg.GetLogger() + lg.Info( + "searching for a suitable member local address in AdvertisePeerURLs", + zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()), + ) + for _, peerURL := range cfg.AdvertisePeerUrls { + if addr, err := netip.ParseAddr(peerURL.Hostname()); err == nil { + if addr.IsLoopback() || addr.IsUnspecified() { + continue + } + lg.Info( + "setting member local address", + zap.String("LocalAddr", addr.String()), + ) + return addr.String() + } + } + lg.Warn( + "unable to set a member local address due to lack of suitable local addresses", + zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()), + ) + return "" +} + func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew } func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) } diff --git a/server/embed/config_test.go b/server/embed/config_test.go index cc72a122c79..3e79f7cd43e 100644 --- a/server/embed/config_test.go +++ b/server/embed/config_test.go @@ -153,6 +153,138 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) { } } +func TestInferLocalAddr(t *testing.T) { + tests := []struct { + name string + advertisePeerURLs []string + setMemberLocalAddr bool + expectedLocalAddr string + }{ + { + "defaults, ExperimentalSetMemberLocalAddr=false ", + []string{DefaultInitialAdvertisePeerURLs}, + false, + "", + }, + { + "IPv4 address, ExperimentalSetMemberLocalAddr=false ", + []string{"https://192.168.100.110:2380"}, + false, + "", + }, + { + "defaults, ExperimentalSetMemberLocalAddr=true", + []string{DefaultInitialAdvertisePeerURLs}, + true, + "", + }, + { + "IPv4 unspecified address, ExperimentalSetMemberLocalAddr=true", + []string{"https://0.0.0.0:2380"}, + true, + "", + }, + { + "IPv6 unspecified address, ExperimentalSetMemberLocalAddr=true", + []string{"https://[::]:2380"}, + true, + "", + }, + { + "IPv4 loopback address, ExperimentalSetMemberLocalAddr=true", + []string{"https://127.0.0.1:2380"}, + true, + "", + }, + { + "IPv6 loopback address, ExperimentalSetMemberLocalAddr=true", + []string{"https://[::1]:2380"}, + true, + "", + }, + { + "IPv4 address, ExperimentalSetMemberLocalAddr=true", + []string{"https://192.168.100.110:2380"}, + true, + "192.168.100.110", + }, + { + "Hostname only, ExperimentalSetMemberLocalAddr=true", + []string{"https://123-host-3.corp.internal:2380"}, + true, + "", + }, + { + "Hostname and IPv4 address, ExperimentalSetMemberLocalAddr=true", + []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"}, + true, + "192.168.100.110", + }, + { + "IPv4 address and Hostname, ExperimentalSetMemberLocalAddr=true", + []string{"https://192.168.100.110:2380", "https://123-host-3.corp.internal:2380"}, + true, + "192.168.100.110", + }, + { + "IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true", + []string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, + true, + "192.168.100.110", + }, + { + "IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true", + // IPv4 addresses will always sort before IPv6 ones anyway + []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"}, + true, + "192.168.100.110", + }, + { + "Hostname, IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true", + []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, + true, + "192.168.100.110", + }, + { + "Hostname, IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true", + // IPv4 addresses will always sort before IPv6 ones anyway + []string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"}, + true, + "192.168.100.110", + }, + { + "IPv6 address, ExperimentalSetMemberLocalAddr=true", + []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"}, + true, + "2001:db8:85a3::8a2e:370:7334", + }, + { + "Hostname and IPv6 address, ExperimentalSetMemberLocalAddr=true", + []string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, + true, + "2001:db8:85a3::8a2e:370:7334", + }, + { + "IPv6 address and Hostname, ExperimentalSetMemberLocalAddr=true", + []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://123-host-3.corp.internal:2380"}, + true, + "2001:db8:85a3::8a2e:370:7334", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := NewConfig() + cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerURLs) + cfg.ExperimentalSetMemberLocalAddr = tt.setMemberLocalAddr + + require.NoError(t, cfg.Validate()) + require.Equal(t, tt.expectedLocalAddr, cfg.InferLocalAddr()) + }) + } + +} + func (s *securityConfig) equals(t *transport.TLSInfo) bool { return s.CertFile == t.CertFile && s.CertAuth == t.ClientCertAuth && diff --git a/server/embed/etcd.go b/server/embed/etcd.go index 1b7b3df0cb9..ce3b3398643 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes, ExperimentalMaxLearners: cfg.ExperimentalMaxLearners, V2Deprecation: cfg.V2DeprecationEffective(), + ExperimentalLocalAddress: cfg.InferLocalAddr(), } if srvcfg.ExperimentalEnableDistributedTracing { @@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ) } + srvcfg.PeerTLSInfo.LocalAddr = srvcfg.ExperimentalLocalAddress + print(e.cfg.logger, *cfg, srvcfg, memberInitialized) if e.Server, err = etcdserver.NewServer(srvcfg); err != nil { @@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized zap.Strings("advertise-client-urls", ec.getAdvertiseClientURLs()), zap.Strings("listen-client-urls", ec.getListenClientURLs()), zap.Strings("listen-metrics-urls", ec.getMetricsURLs()), + zap.Bool("experimental-set-member-localaddr", ec.ExperimentalSetMemberLocalAddr), + zap.String("experimental-local-address", sc.ExperimentalLocalAddress), zap.Strings("cors", cors), zap.Strings("host-whitelist", hss), zap.String("initial-cluster", sc.InitialPeerURLsMap.String()), diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index 9f35d68fc1e..cc8e8ef7185 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -107,6 +107,8 @@ Member: Clustering: --initial-advertise-peer-urls 'http://localhost:2380' List of this member's peer URLs to advertise to the rest of the cluster. + --experimental-set-member-localaddr 'false' + Enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer. --initial-cluster 'default=http://localhost:2380' Initial cluster configuration for bootstrapping. --initial-cluster-state 'new' diff --git a/tests/e2e/etcd_config_test.go b/tests/e2e/etcd_config_test.go index aa8370bd344..ca500eeff02 100644 --- a/tests/e2e/etcd_config_test.go +++ b/tests/e2e/etcd_config_test.go @@ -17,6 +17,7 @@ package e2e import ( "context" "fmt" + "net" "os" "strings" "testing" @@ -274,6 +275,115 @@ func TestEtcdPeerNameAuth(t *testing.T) { } } +// TestEtcdPeerLocalAddr checks that the inter peer auth works with when +// the member LocalAddr is set. +func TestEtcdPeerLocalAddr(t *testing.T) { + e2e.SkipInShortMode(t) + + nodeIP, err := getLocalIP() + t.Log("Using node IP", nodeIP) + require.NoError(t, err) + + peers, tmpdirs := make([]string, 3), make([]string, 3) + + for i := range peers { + peerIP := nodeIP + if i == 0 { + peerIP = "127.0.0.1" + } + peers[i] = fmt.Sprintf("e%d=https://%s:%d", i, peerIP, e2e.EtcdProcessBasePort+i) + tmpdirs[i] = t.TempDir() + } + procs := make([]*expect.ExpectProcess, len(peers)) + defer func() { + for i := range procs { + if procs[i] != nil { + procs[i].Stop() + procs[i].Close() + } + os.RemoveAll(tmpdirs[i]) + } + }() + + tempDir := t.TempDir() + caFile, certFiles, keyFiles, err := generateCertsForIPs(tempDir, []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(nodeIP)}) + require.NoError(t, err) + + defer func() { + os.RemoveAll(tempDir) + }() + + // node 0 (127.0.0.1) does not set `--experimental-set-member-localaddr`, + // while nodes 1 and nodes 2 do. + // + // node 0's peer certificate is signed for 127.0.0.1, but it uses the host + // IP (by default) to communicate with peers, so they don't match. + // Accordingly, other peers will reject connections from node 0. + // + // Both node 1 and node 2's peer certificates are signed for the host IP, + // and they also communicate with peers using the host IP (explicitly set + // with --initial-advertise-peer-urls and + // --experimental-set-member-localaddr), so node 0 has no issue connecting + // to them. + // + // Refer to https://github.com/etcd-io/etcd/issues/17068. + for i := range procs { + peerIP := nodeIP + if i == 0 { + peerIP = "127.0.0.1" + } + ic := strings.Join(peers, ",") + commonArgs := []string{ + e2e.BinPath.Etcd, + "--name", fmt.Sprintf("e%d", i), + "--listen-client-urls", "http://0.0.0.0:0", + "--data-dir", tmpdirs[i], + "--advertise-client-urls", "http://0.0.0.0:0", + "--initial-advertise-peer-urls", fmt.Sprintf("https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i), + "--listen-peer-urls", fmt.Sprintf("https://%s:%d,https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i, peerIP, e2e.EtcdProcessBasePort+len(peers)+i), + "--initial-cluster", ic, + } + + var args []string + if i == 0 { + args = []string{ + "--peer-cert-file", certFiles[0], + "--peer-key-file", keyFiles[0], + "--peer-trusted-ca-file", caFile, + "--peer-client-cert-auth", + } + } else { + args = []string{ + "--peer-cert-file", certFiles[1], + "--peer-key-file", keyFiles[1], + "--peer-trusted-ca-file", caFile, + "--peer-client-cert-auth", + "--experimental-set-member-localaddr", + } + } + + commonArgs = append(commonArgs, args...) + + p, err := e2e.SpawnCmd(commonArgs, nil) + if err != nil { + t.Fatal(err) + } + procs[i] = p + } + + for i, p := range procs { + var expect []string + if i == 0 { + expect = e2e.EtcdServerReadyLines + } else { + expect = []string{"x509: certificate is valid for 127.0.0.1, not "} + } + if err := e2e.WaitReadyExpectProc(context.TODO(), p, expect); err != nil { + t.Fatal(err) + } + } +} + func TestGrpcproxyAndCommonName(t *testing.T) { e2e.SkipInShortMode(t) diff --git a/tests/e2e/utils.go b/tests/e2e/utils.go index cd7c1c90df9..c8ccf9c1990 100644 --- a/tests/e2e/utils.go +++ b/tests/e2e/utils.go @@ -15,9 +15,18 @@ package e2e import ( + "bytes" "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "fmt" + "math/big" + "net" + "os" "strings" "testing" "time" @@ -145,3 +154,120 @@ func patchArgs(args []string, flag, newValue string) error { } return fmt.Errorf("--%s flag not found", flag) } + +func generateCertsForIPs(tempDir string, ips []net.IP) (caFile string, certFiles []string, keyFiles []string, err error) { + ca := &x509.Certificate{ + SerialNumber: big.NewInt(1001), + Subject: pkix.Name{ + Organization: []string{"etcd"}, + OrganizationalUnit: []string{"etcd Security"}, + Locality: []string{"San Francisco"}, + Province: []string{"California"}, + Country: []string{"USA"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(0, 0, 1), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", nil, nil, err + } + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + if err != nil { + return "", nil, nil, err + } + + caFile, _, err = saveCertToFile(tempDir, caBytes, nil) + if err != nil { + return "", nil, nil, err + } + + for i, ip := range ips { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1001 + int64(i)), + Subject: pkix.Name{ + Organization: []string{"etcd"}, + OrganizationalUnit: []string{"etcd Security"}, + Locality: []string{"San Francisco"}, + Province: []string{"California"}, + Country: []string{"USA"}, + }, + IPAddresses: []net.IP{ip}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(0, 0, 1), + SubjectKeyId: []byte{1, 2, 3, 4, 5}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + certKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", nil, nil, err + } + certBytes, err := x509.CreateCertificate(rand.Reader, cert, ca, &certKey.PublicKey, caKey) + if err != nil { + return "", nil, nil, err + } + certFile, keyFile, err := saveCertToFile(tempDir, certBytes, certKey) + if err != nil { + return "", nil, nil, err + } + certFiles = append(certFiles, certFile) + keyFiles = append(keyFiles, keyFile) + } + + return caFile, certFiles, keyFiles, nil +} + +func saveCertToFile(tempDir string, certBytes []byte, key *rsa.PrivateKey) (certFile string, keyFile string, err error) { + certPEM := new(bytes.Buffer) + pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + cf, err := os.CreateTemp(tempDir, "*.crt") + if err != nil { + return "", "", err + } + defer cf.Close() + if _, err := cf.Write(certPEM.Bytes()); err != nil { + return "", "", err + } + + if key != nil { + certKeyPEM := new(bytes.Buffer) + pem.Encode(certKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) + + kf, err := os.CreateTemp(tempDir, "*.key.insecure") + if err != nil { + return "", "", err + } + defer kf.Close() + if _, err := kf.Write(certKeyPEM.Bytes()); err != nil { + return "", "", err + } + + return cf.Name(), kf.Name(), nil + } + + return cf.Name(), "", nil +} + +func getLocalIP() (string, error) { + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + return "", err + } + defer conn.Close() + + localAddress := conn.LocalAddr().(*net.UDPAddr) + + return localAddress.IP.String(), nil +}