From 1656c95bf55a01249e7efa6a5b5a5cbc51b5c92b Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Thu, 27 Apr 2017 06:01:10 -0700 Subject: [PATCH] integration: test TLS reload Signed-off-by: Gyu-Ho Lee --- integration/cluster.go | 48 ++++++++ integration/v3_grpc_test.go | 215 ++++++++++++++++++++++++++++++++++++ 2 files changed, 263 insertions(+) diff --git a/integration/cluster.go b/integration/cluster.go index fd7330a8533..f220a0e69a8 100644 --- a/integration/cluster.go +++ b/integration/cluster.go @@ -17,12 +17,14 @@ package integration import ( "crypto/tls" "fmt" + "io" "io/ioutil" "math/rand" "net" "net/http" "net/http/httptest" "os" + "path/filepath" "reflect" "sort" "strings" @@ -76,6 +78,13 @@ var ( ClientCertAuth: true, } + testTLSInfoExpired = transport.TLSInfo{ + KeyFile: "./fixtures-expired/server-key.pem", + CertFile: "./fixtures-expired/server.pem", + TrustedCAFile: "./fixtures-expired/etcd-root-ca.pem", + ClientCertAuth: true, + } + plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "integration") ) @@ -929,3 +938,42 @@ type grpcAPI struct { // Auth is the authentication API for the client's connection. Auth pb.AuthClient } + +// copyTLSFiles clones certs files to dst directory. +func copyTLSFiles(ti transport.TLSInfo, dst string) (transport.TLSInfo, error) { + ci := transport.TLSInfo{ + KeyFile: filepath.Join(dst, "server-key.pem"), + CertFile: filepath.Join(dst, "server.pem"), + TrustedCAFile: filepath.Join(dst, "etcd-root-ca.pem"), + ClientCertAuth: ti.ClientCertAuth, + } + if err := copyFile(ti.KeyFile, ci.KeyFile); err != nil { + return transport.TLSInfo{}, err + } + if err := copyFile(ti.CertFile, ci.CertFile); err != nil { + return transport.TLSInfo{}, err + } + if err := copyFile(ti.TrustedCAFile, ci.TrustedCAFile); err != nil { + return transport.TLSInfo{}, err + } + return ci, nil +} + +func copyFile(src, dst string) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + + w, err := os.Create(dst) + if err != nil { + return err + } + defer w.Close() + + if _, err = io.Copy(w, f); err != nil { + return err + } + return w.Sync() +} diff --git a/integration/v3_grpc_test.go b/integration/v3_grpc_test.go index 5113821def1..6aeedaf9886 100644 --- a/integration/v3_grpc_test.go +++ b/integration/v3_grpc_test.go @@ -16,17 +16,22 @@ package integration import ( "bytes" + "crypto/tls" "fmt" + "io/ioutil" "math/rand" "os" "reflect" + "strings" "testing" "time" + "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/etcdserver/api/v3rpc" "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" pb "github.com/coreos/etcd/etcdserver/etcdserverpb" "github.com/coreos/etcd/pkg/testutil" + "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -1374,6 +1379,216 @@ func TestTLSGRPCAcceptSecureAll(t *testing.T) { } } +// TestTLSReloadAtomicReplace ensures server reloads expired/valid certs +// when all certs are atomically replaced by directory renaming. +// And expects server to reject client requests, and vice versa. +func TestTLSReloadAtomicReplace(t *testing.T) { + defer testutil.AfterTest(t) + + // clone valid,expired certs to separate directories for atomic renaming + vDir, err := ioutil.TempDir(os.TempDir(), "fixtures-valid") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(vDir) + ts, err := copyTLSFiles(testTLSInfo, vDir) + if err != nil { + t.Fatal(err) + } + eDir, err := ioutil.TempDir(os.TempDir(), "fixtures-expired") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(eDir) + if _, err = copyTLSFiles(testTLSInfoExpired, eDir); err != nil { + t.Fatal(err) + } + + tDir, err := ioutil.TempDir(os.TempDir(), "fixtures") + if err != nil { + t.Fatal(err) + } + os.RemoveAll(tDir) + defer os.RemoveAll(tDir) + + // start with valid certs + clus := NewClusterV3(t, &ClusterConfig{Size: 1, PeerTLS: &ts, ClientTLS: &ts}) + defer clus.Terminate(t) + + // concurrent client dialing while certs transition from valid to expired + errc := make(chan error) + go func() { + for { + cc, err := ts.ClientConfig() + if err != nil { + if os.IsNotExist(err) { + // from concurrent renaming + continue + } + t.Fatal(err) + } + cli, cerr := clientv3.New(clientv3.Config{ + Endpoints: []string{clus.Members[0].GRPCAddr()}, + DialTimeout: 3 * time.Second, + TLS: cc, + }) + if cerr != nil { + errc <- cerr + return + } + cli.Close() + } + }() + + // replace certs directory with expired ones + if err = os.Rename(vDir, tDir); err != nil { + t.Fatal(err) + } + if err = os.Rename(eDir, vDir); err != nil { + t.Fatal(err) + } + + // after rename, + // 'vDir' contains expired certs + // 'tDir' contains valid certs + // 'eDir' does not exist + + select { + case err = <-errc: + if err != grpc.ErrClientConnTimeout { + t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err) + } + case <-time.After(7 * time.Second): + t.Fatal("expected dial timeout in 3 seconds, but never got it") + } + + // now, replace expired certs back with valid ones + if err = os.Rename(tDir, eDir); err != nil { + t.Fatal(err) + } + if err = os.Rename(vDir, tDir); err != nil { + t.Fatal(err) + } + if err = os.Rename(eDir, vDir); err != nil { + t.Fatal(err) + } + + // new incoming client request should trigger + // listener to reload valid certs + var tls *tls.Config + tls, err = ts.ClientConfig() + if err != nil { + t.Fatal(err) + } + var cl *clientv3.Client + cl, err = clientv3.New(clientv3.Config{ + Endpoints: []string{clus.Members[0].GRPCAddr()}, + DialTimeout: 3 * time.Second, + TLS: tls, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + cl.Close() +} + +// TestTLSReloadCopy ensures server reloads expired/valid certs +// when new certs are copied over, one by one. And expects server +// to reject client requests, and vice versa. +func TestTLSReloadCopy(t *testing.T) { + defer testutil.AfterTest(t) + + // clone certs directory, free to overwrite + cDir, err := ioutil.TempDir(os.TempDir(), "fixtures-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(cDir) + ts, err := copyTLSFiles(testTLSInfo, cDir) + if err != nil { + t.Fatal(err) + } + + // start with valid certs + clus := NewClusterV3(t, &ClusterConfig{Size: 1, PeerTLS: &ts, ClientTLS: &ts}) + defer clus.Terminate(t) + + // concurrent client dialing while certs transition from valid to expired + errc := make(chan error) + go func() { + for { + cc, err := ts.ClientConfig() + if err != nil { + if strings.Contains(err.Error(), "tls: private key does not match public key") { + // from concurrent certs overwriting + continue + } + t.Fatal(err) + } + cli, cerr := clientv3.New(clientv3.Config{ + Endpoints: []string{clus.Members[0].GRPCAddr()}, + DialTimeout: 3 * time.Second, + TLS: cc, + }) + if cerr != nil { + errc <- cerr + return + } + cli.Close() + } + }() + + // overwrite valid certs with expired ones + // (e.g. simulate cert expiration in practice) + if err = copyFile(testTLSInfoExpired.KeyFile, ts.KeyFile); err != nil { + t.Fatal(err) + } + if err = copyFile(testTLSInfoExpired.CertFile, ts.CertFile); err != nil { + t.Fatal(err) + } + if err = copyFile(testTLSInfoExpired.TrustedCAFile, ts.TrustedCAFile); err != nil { + t.Fatal(err) + } + + select { + case err = <-errc: + if err != grpc.ErrClientConnTimeout { + t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err) + } + case <-time.After(7 * time.Second): + t.Fatal("expected dial timeout in 3 seconds, but never got it") + } + + // now, replace expired certs back with valid ones + if err = copyFile(testTLSInfo.KeyFile, ts.KeyFile); err != nil { + t.Fatal(err) + } + if err = copyFile(testTLSInfo.CertFile, ts.CertFile); err != nil { + t.Fatal(err) + } + if err = copyFile(testTLSInfo.TrustedCAFile, ts.TrustedCAFile); err != nil { + t.Fatal(err) + } + + // new incoming client request should trigger + // listener to reload valid certs + var tls *tls.Config + tls, err = ts.ClientConfig() + if err != nil { + t.Fatal(err) + } + var cl *clientv3.Client + cl, err = clientv3.New(clientv3.Config{ + Endpoints: []string{clus.Members[0].GRPCAddr()}, + DialTimeout: 3 * time.Second, + TLS: tls, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + cl.Close() +} + func TestGRPCRequireLeader(t *testing.T) { defer testutil.AfterTest(t)