integration: test TLS reload

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>

integration/cluster.go

@@ -17,12 +17,14 @@ package integration
 import (
 	"crypto/tls"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
@@ -76,6 +78,13 @@ var (
 		ClientCertAuth: true,
 	}
 
+	testTLSInfoExpired = transport.TLSInfo{
+		KeyFile:        "./fixtures-expired/server-key.pem",
+		CertFile:       "./fixtures-expired/server.pem",
+		TrustedCAFile:  "./fixtures-expired/etcd-root-ca.pem",
+		ClientCertAuth: true,
+	}
+
 	plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "integration")
 )
 
@@ -929,3 +938,42 @@ type grpcAPI struct {
 	// Auth is the authentication API for the client's connection.
 	Auth pb.AuthClient
 }
+
+// copyTLSFiles clones certs files to dst directory.
+func copyTLSFiles(ti transport.TLSInfo, dst string) (transport.TLSInfo, error) {
+	ci := transport.TLSInfo{
+		KeyFile:        filepath.Join(dst, "server-key.pem"),
+		CertFile:       filepath.Join(dst, "server.pem"),
+		TrustedCAFile:  filepath.Join(dst, "etcd-root-ca.pem"),
+		ClientCertAuth: ti.ClientCertAuth,
+	}
+	if err := copyFile(ti.KeyFile, ci.KeyFile); err != nil {
+		return transport.TLSInfo{}, err
+	}
+	if err := copyFile(ti.CertFile, ci.CertFile); err != nil {
+		return transport.TLSInfo{}, err
+	}
+	if err := copyFile(ti.TrustedCAFile, ci.TrustedCAFile); err != nil {
+		return transport.TLSInfo{}, err
+	}
+	return ci, nil
+}
+
+func copyFile(src, dst string) error {
+	f, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	w, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer w.Close()
+
+	if _, err = io.Copy(w, f); err != nil {
+		return err
+	}
+	return w.Sync()
+}

integration/v3_grpc_test.go

@@ -16,17 +16,22 @@ package integration
 
 import (
 	"bytes"
+	"crypto/tls"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"os"
 	"reflect"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc"
 	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/testutil"
+
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
@@ -1374,6 +1379,216 @@ func TestTLSGRPCAcceptSecureAll(t *testing.T) {
 	}
 }
 
+// TestTLSReloadAtomicReplace ensures server reloads expired/valid certs
+// when all certs are atomically replaced by directory renaming.
+// And expects server to reject client requests, and vice versa.
+func TestTLSReloadAtomicReplace(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	// clone valid,expired certs to separate directories for atomic renaming
+	vDir, err := ioutil.TempDir(os.TempDir(), "fixtures-valid")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(vDir)
+	ts, err := copyTLSFiles(testTLSInfo, vDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	eDir, err := ioutil.TempDir(os.TempDir(), "fixtures-expired")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(eDir)
+	if _, err = copyTLSFiles(testTLSInfoExpired, eDir); err != nil {
+		t.Fatal(err)
+	}
+
+	tDir, err := ioutil.TempDir(os.TempDir(), "fixtures")
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.RemoveAll(tDir)
+	defer os.RemoveAll(tDir)
+
+	// start with valid certs
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1, PeerTLS: &ts, ClientTLS: &ts})
+	defer clus.Terminate(t)
+
+	// concurrent client dialing while certs transition from valid to expired
+	errc := make(chan error)
+	go func() {
+		for {
+			cc, err := ts.ClientConfig()
+			if err != nil {
+				if os.IsNotExist(err) {
+					// from concurrent renaming
+					continue
+				}
+				t.Fatal(err)
+			}
+			cli, cerr := clientv3.New(clientv3.Config{
+				Endpoints:   []string{clus.Members[0].GRPCAddr()},
+				DialTimeout: 3 * time.Second,
+				TLS:         cc,
+			})
+			if cerr != nil {
+				errc <- cerr
+				return
+			}
+			cli.Close()
+		}
+	}()
+
+	// replace certs directory with expired ones
+	if err = os.Rename(vDir, tDir); err != nil {
+		t.Fatal(err)
+	}
+	if err = os.Rename(eDir, vDir); err != nil {
+		t.Fatal(err)
+	}
+
+	// after rename,
+	// 'vDir' contains expired certs
+	// 'tDir' contains valid certs
+	// 'eDir' does not exist
+
+	select {
+	case err = <-errc:
+		if err != grpc.ErrClientConnTimeout {
+			t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err)
+		}
+	case <-time.After(7 * time.Second):
+		t.Fatal("expected dial timeout in 3 seconds, but never got it")
+	}
+
+	// now, replace expired certs back with valid ones
+	if err = os.Rename(tDir, eDir); err != nil {
+		t.Fatal(err)
+	}
+	if err = os.Rename(vDir, tDir); err != nil {
+		t.Fatal(err)
+	}
+	if err = os.Rename(eDir, vDir); err != nil {
+		t.Fatal(err)
+	}
+
+	// new incoming client request should trigger
+	// listener to reload valid certs
+	var tls *tls.Config
+	tls, err = ts.ClientConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var cl *clientv3.Client
+	cl, err = clientv3.New(clientv3.Config{
+		Endpoints:   []string{clus.Members[0].GRPCAddr()},
+		DialTimeout: 3 * time.Second,
+		TLS:         tls,
+	})
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	cl.Close()
+}
+
+// TestTLSReloadCopy ensures server reloads expired/valid certs
+// when new certs are copied over, one by one. And expects server
+// to reject client requests, and vice versa.
+func TestTLSReloadCopy(t *testing.T) {
+	defer testutil.AfterTest(t)
+
+	// clone certs directory, free to overwrite
+	cDir, err := ioutil.TempDir(os.TempDir(), "fixtures-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(cDir)
+	ts, err := copyTLSFiles(testTLSInfo, cDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// start with valid certs
+	clus := NewClusterV3(t, &ClusterConfig{Size: 1, PeerTLS: &ts, ClientTLS: &ts})
+	defer clus.Terminate(t)
+
+	// concurrent client dialing while certs transition from valid to expired
+	errc := make(chan error)
+	go func() {
+		for {
+			cc, err := ts.ClientConfig()
+			if err != nil {
+				if strings.Contains(err.Error(), "tls: private key does not match public key") {
+					// from concurrent certs overwriting
+					continue
+				}
+				t.Fatal(err)
+			}
+			cli, cerr := clientv3.New(clientv3.Config{
+				Endpoints:   []string{clus.Members[0].GRPCAddr()},
+				DialTimeout: 3 * time.Second,
+				TLS:         cc,
+			})
+			if cerr != nil {
+				errc <- cerr
+				return
+			}
+			cli.Close()
+		}
+	}()
+
+	// overwrite valid certs with expired ones
+	// (e.g. simulate cert expiration in practice)
+	if err = copyFile(testTLSInfoExpired.KeyFile, ts.KeyFile); err != nil {
+		t.Fatal(err)
+	}
+	if err = copyFile(testTLSInfoExpired.CertFile, ts.CertFile); err != nil {
+		t.Fatal(err)
+	}
+	if err = copyFile(testTLSInfoExpired.TrustedCAFile, ts.TrustedCAFile); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case err = <-errc:
+		if err != grpc.ErrClientConnTimeout {
+			t.Fatalf("expected %v, got %v", grpc.ErrClientConnTimeout, err)
+		}
+	case <-time.After(7 * time.Second):
+		t.Fatal("expected dial timeout in 3 seconds, but never got it")
+	}
+
+	// now, replace expired certs back with valid ones
+	if err = copyFile(testTLSInfo.KeyFile, ts.KeyFile); err != nil {
+		t.Fatal(err)
+	}
+	if err = copyFile(testTLSInfo.CertFile, ts.CertFile); err != nil {
+		t.Fatal(err)
+	}
+	if err = copyFile(testTLSInfo.TrustedCAFile, ts.TrustedCAFile); err != nil {
+		t.Fatal(err)
+	}
+
+	// new incoming client request should trigger
+	// listener to reload valid certs
+	var tls *tls.Config
+	tls, err = ts.ClientConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var cl *clientv3.Client
+	cl, err = clientv3.New(clientv3.Config{
+		Endpoints:   []string{clus.Members[0].GRPCAddr()},
+		DialTimeout: 3 * time.Second,
+		TLS:         tls,
+	})
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	cl.Close()
+}
+
 func TestGRPCRequireLeader(t *testing.T) {
 	defer testutil.AfterTest(t)