feat: split etcd certificates to peer/client

Changes: * Etcd peer port key usage: ServerAuth,ClientAuth * Etcd client port key usage: ServerAuth,ClientAuth * Talos etcd client key usage: ClientAuth * KubeAPI etcd client key usage: ClientAuth * List of etcd allowed ciphers Signed-off-by: Serge Logvinov <serge.logvinov@sinextra.dev> Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
siderolabs · Jun 23, 2021 · b52b206 · b52b206
1 parent 33119d2
commit b52b206
Show file tree

Hide file tree

Showing 11 changed files with 184 additions and 27 deletions.
diff --git a/go.mod b/go.mod
@@ -78,7 +78,7 @@ require (
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/viper v1.7.1 // indirect
 	github.com/stretchr/testify v1.7.0
-	github.com/talos-systems/crypto v0.3.1-0.20210615131117-6bc5bb50c527
+	github.com/talos-systems/crypto v0.3.1-0.20210617123329-d3cb77220384
 	github.com/talos-systems/go-blockdevice v0.2.1
 	github.com/talos-systems/go-cmd v0.0.0-20210216164758-68eb0067e0f0
 	github.com/talos-systems/go-debug v0.2.1-0.20210525175311-3d0a6e1bf5e3

diff --git a/go.sum b/go.sum
@@ -1112,8 +1112,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/talos-systems/crypto v0.3.1-0.20210615131117-6bc5bb50c527 h1:Ut3jI/0FPSyH0QpycKFKAueWKjzQ00OFmkeFG+0uO1A=
-github.com/talos-systems/crypto v0.3.1-0.20210615131117-6bc5bb50c527/go.mod h1:xaNCB2/Bxaj+qrkdeodhRv5eKQVvKOGBBMj58MrIPY8=
+github.com/talos-systems/crypto v0.3.1-0.20210617123329-d3cb77220384 h1:rfFjcSr+oza1k7gMcrYW9RCiVSiFvKUYMB1fcIrwMf0=
+github.com/talos-systems/crypto v0.3.1-0.20210617123329-d3cb77220384/go.mod h1:xaNCB2/Bxaj+qrkdeodhRv5eKQVvKOGBBMj58MrIPY8=
 github.com/talos-systems/go-blockdevice v0.2.1 h1:swoY5NcssuMgdCf/dlMngNDgEAasGp2jviPqAz9Epss=
 github.com/talos-systems/go-blockdevice v0.2.1/go.mod h1:qnn/zDc09I1DA2BUDDCOSA2D0P8pIDjN8pGiRoRaQig=
 github.com/talos-systems/go-cmd v0.0.0-20210216164758-68eb0067e0f0 h1:DI+BjK+fcrLBc70Fi50dZocQcaHosqsuWHrGHKp2NzE=

diff --git a/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go b/internal/app/machined/pkg/controllers/k8s/render_secrets_static_pod.go
@@ -149,7 +149,7 @@ func (ctrl *RenderSecretsStaticPodController) Run(ctx context.Context, r control
 						certFilename: "etcd-client-ca.crt",
 					},
 					{
-						getter:       func() *x509.PEMEncodedCertificateAndKey { return etcdSecrets.EtcdPeer },
+						getter:       func() *x509.PEMEncodedCertificateAndKey { return etcdSecrets.EtcdAPIServer },
 						certFilename: "etcd-client.crt",
 						keyFilename:  "etcd-client.key",
 					},

diff --git a/internal/app/machined/pkg/controllers/secrets/etcd.go b/internal/app/machined/pkg/controllers/secrets/etcd.go
@@ -130,9 +130,24 @@ func (ctrl *EtcdController) Run(ctx context.Context, r controller.Runtime, logge
 func (ctrl *EtcdController) updateSecrets(etcdRoot *secrets.RootEtcdSpec, etcdCerts *secrets.EtcdCertsSpec) error {
 	var err error
 
+	etcdCerts.Etcd, err = etcd.GenerateCert(etcdRoot.EtcdCA)
+	if err != nil {
+		return fmt.Errorf("error generating etcd client certs: %w", err)
+	}
+
 	etcdCerts.EtcdPeer, err = etcd.GeneratePeerCert(etcdRoot.EtcdCA)
 	if err != nil {
-		return fmt.Errorf("error generating etcd certs: %w", err)
+		return fmt.Errorf("error generating etcd peer certs: %w", err)
+	}
+
+	etcdCerts.EtcdAdmin, err = etcd.GenerateClientCert(etcdRoot.EtcdCA, "talos")
+	if err != nil {
+		return fmt.Errorf("error generating admin client certs: %w", err)
+	}
+
+	etcdCerts.EtcdAPIServer, err = etcd.GenerateClientCert(etcdRoot.EtcdCA, "kube-apiserver")
+	if err != nil {
+		return fmt.Errorf("error generating kube-apiserver etcd client certs: %w", err)
 	}
 
 	return nil

diff --git a/internal/app/machined/pkg/system/services/etcd.go b/internal/app/machined/pkg/system/services/etcd.go
@@ -19,7 +19,10 @@ import (
 	containerdapi "github.com/containerd/containerd"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/oci"
+	"github.com/cosi-project/runtime/pkg/resource"
+	"github.com/cosi-project/runtime/pkg/state"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/talos-systems/crypto/x509"
 	"github.com/talos-systems/go-retry/retry"
 	"github.com/talos-systems/net"
 	clientv3 "go.etcd.io/etcd/client/v3"
@@ -41,6 +44,7 @@ import (
 	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
 	"github.com/talos-systems/talos/pkg/resources/network"
+	"github.com/talos-systems/talos/pkg/resources/secrets"
 	timeresource "github.com/talos-systems/talos/pkg/resources/time"
 )
 
@@ -147,6 +151,8 @@ func (e *Etcd) Runner(r runtime.Runtime) (runner.Runner, error) {
 		env = append(env, "ETCD_UNSUPPORTED_ARCH=arm64")
 	}
 
+	env = append(env, "ETCD_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305") //nolint:lll
+
 	return restart.New(containerd.NewRunner(
 		r.Config().Debug(),
 		&args,
@@ -187,29 +193,73 @@ func (e *Etcd) HealthSettings(runtime.Runtime) *health.Settings {
 	}
 }
 
+//nolint:gocyclo
 func generatePKI(r runtime.Runtime) (err error) {
 	if err = os.MkdirAll(constants.EtcdPKIPath, 0o700); err != nil {
 		return err
 	}
 
-	if err = ioutil.WriteFile(constants.KubernetesEtcdCACert, r.Config().Cluster().Etcd().CA().Crt, 0o500); err != nil {
+	if err = ioutil.WriteFile(constants.KubernetesEtcdCACert, r.Config().Cluster().Etcd().CA().Crt, 0o400); err != nil {
 		return fmt.Errorf("failed to write CA certificate: %w", err)
 	}
 
-	if err = ioutil.WriteFile(constants.KubernetesEtcdCAKey, r.Config().Cluster().Etcd().CA().Key, 0o500); err != nil {
+	if err = ioutil.WriteFile(constants.KubernetesEtcdCAKey, r.Config().Cluster().Etcd().CA().Key, 0o400); err != nil {
 		return fmt.Errorf("failed to write CA key: %w", err)
 	}
 
-	certAndKey, err := etcd.GeneratePeerCert(r.Config().Cluster().Etcd().CA())
-	if err != nil {
+	// wait for etcd certificates to be generated in the controller
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	watchCh := make(chan state.Event)
+
+	if err = r.State().V1Alpha2().Resources().Watch(ctx, resource.NewMetadata(secrets.NamespaceName, secrets.EtcdType, secrets.EtcdID, resource.VersionUndefined), watchCh); err != nil {
 		return err
 	}
 
-	if err := ioutil.WriteFile(constants.KubernetesEtcdPeerKey, certAndKey.Key, 0o500); err != nil {
-		return err
+	var event state.Event
+
+	for {
+		event = <-watchCh
+
+		if event.Type == state.Created || event.Type == state.Updated {
+			break
+		}
 	}
 
-	return ioutil.WriteFile(constants.KubernetesEtcdPeerCert, certAndKey.Crt, 0o500)
+	etcdCerts := event.Resource.(*secrets.Etcd).Certs()
+
+	for _, keypair := range []struct {
+		getter   func() *x509.PEMEncodedCertificateAndKey
+		keyPath  string
+		certPath string
+	}{
+		{
+			getter:   func() *x509.PEMEncodedCertificateAndKey { return etcdCerts.Etcd },
+			keyPath:  constants.KubernetesEtcdKey,
+			certPath: constants.KubernetesEtcdCert,
+		},
+		{
+			getter:   func() *x509.PEMEncodedCertificateAndKey { return etcdCerts.EtcdPeer },
+			keyPath:  constants.KubernetesEtcdPeerKey,
+			certPath: constants.KubernetesEtcdPeerCert,
+		},
+		{
+			getter:   func() *x509.PEMEncodedCertificateAndKey { return etcdCerts.EtcdAdmin },
+			keyPath:  constants.KubernetesEtcdAdminKey,
+			certPath: constants.KubernetesEtcdAdminCert,
+		},
+	} {
+		if err = ioutil.WriteFile(keypair.keyPath, keypair.getter().Key, 0o400); err != nil {
+			return err
+		}
+
+		if err = ioutil.WriteFile(keypair.certPath, keypair.getter().Crt, 0o400); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func addMember(ctx context.Context, r runtime.Runtime, addrs []string, name string) (*clientv3.MemberListResponse, uint64, error) {
@@ -341,13 +391,14 @@ func (e *Etcd) argsForInit(ctx context.Context, r runtime.Runtime) error {
 		"data-dir":              constants.EtcdDataPath,
 		"listen-peer-urls":      "https://" + net.FormatAddress(listenAddress) + ":2380",
 		"listen-client-urls":    "https://" + net.FormatAddress(listenAddress) + ":2379",
-		"cert-file":             constants.KubernetesEtcdPeerCert,
-		"key-file":              constants.KubernetesEtcdPeerKey,
+		"client-cert-auth":      "true",
+		"cert-file":             constants.KubernetesEtcdCert,
+		"key-file":              constants.KubernetesEtcdKey,
 		"trusted-ca-file":       constants.KubernetesEtcdCACert,
 		"peer-client-cert-auth": "true",
 		"peer-cert-file":        constants.KubernetesEtcdPeerCert,
-		"peer-trusted-ca-file":  constants.KubernetesEtcdCACert,
 		"peer-key-file":         constants.KubernetesEtcdPeerKey,
+		"peer-trusted-ca-file":  constants.KubernetesEtcdCACert,
 	}
 
 	extraArgs := argsbuilder.Args(r.Config().Cluster().Etcd().ExtraArgs())
@@ -423,13 +474,14 @@ func (e *Etcd) argsForControlPlane(ctx context.Context, r runtime.Runtime) error
 		"data-dir":              constants.EtcdDataPath,
 		"listen-peer-urls":      "https://" + net.FormatAddress(listenAddress) + ":2380",
 		"listen-client-urls":    "https://" + net.FormatAddress(listenAddress) + ":2379",
+		"client-cert-auth":      "true",
 		"cert-file":             constants.KubernetesEtcdPeerCert,
 		"key-file":              constants.KubernetesEtcdPeerKey,
 		"trusted-ca-file":       constants.KubernetesEtcdCACert,
 		"peer-client-cert-auth": "true",
 		"peer-cert-file":        constants.KubernetesEtcdPeerCert,
-		"peer-trusted-ca-file":  constants.KubernetesEtcdCACert,
 		"peer-key-file":         constants.KubernetesEtcdPeerKey,
+		"peer-trusted-ca-file":  constants.KubernetesEtcdCACert,
 	}
 
 	extraArgs := argsbuilder.Args(r.Config().Cluster().Etcd().ExtraArgs())

diff --git a/internal/pkg/etcd/certs.go b/internal/pkg/etcd/certs.go
@@ -5,6 +5,7 @@
 package etcd
 
 import (
+	stdlibx509 "crypto/x509"
 	"fmt"
 	stdlibnet "net"
 	"os"
@@ -14,8 +15,8 @@ import (
 	"github.com/talos-systems/net"
 )
 
-// GeneratePeerCert generates etcd peer certificate and key from etcd CA.
-func GeneratePeerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncodedCertificateAndKey, error) {
+// NewCommonOptions set common certificate options.
+func NewCommonOptions() ([]x509.Option, error) {
 	ips, err := net.IPAddrs()
 	if err != nil {
 		return nil, fmt.Errorf("failed to discover IP addresses: %w", err)
@@ -38,13 +39,31 @@ func GeneratePeerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncode
 
 	dnsNames = append(dnsNames, "localhost")
 
-	opts := []x509.Option{
+	return []x509.Option{
 		x509.CommonName(hostname),
 		x509.DNSNames(dnsNames),
 		x509.IPAddresses(ips),
 		x509.NotAfter(time.Now().Add(87600 * time.Hour)),
+		x509.KeyUsage(stdlibx509.KeyUsageDigitalSignature | stdlibx509.KeyUsageKeyEncipherment),
+	}, nil
+}
+
+// GeneratePeerCert generates etcd peer certificate and key from etcd CA.
+//
+//nolint:dupl
+func GeneratePeerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncodedCertificateAndKey, error) {
+	opts, err := NewCommonOptions()
+	if err != nil {
+		return nil, err
 	}
 
+	opts = append(opts,
+		x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{
+			stdlibx509.ExtKeyUsageServerAuth,
+			stdlibx509.ExtKeyUsageClientAuth,
+		}),
+	)
+
 	ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(etcdCA)
 	if err != nil {
 		return nil, fmt.Errorf("failed loading CA from config: %w", err)
@@ -57,3 +76,59 @@ func GeneratePeerCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncode
 
 	return x509.NewCertificateAndKeyFromKeyPair(keyPair), nil
 }
+
+// GenerateCert generates etcd certificate and key from etcd CA.
+//
+//nolint:dupl
+func GenerateCert(etcdCA *x509.PEMEncodedCertificateAndKey) (*x509.PEMEncodedCertificateAndKey, error) {
+	opts, err := NewCommonOptions()
+	if err != nil {
+		return nil, err
+	}
+
+	opts = append(opts,
+		x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{
+			stdlibx509.ExtKeyUsageServerAuth,
+			stdlibx509.ExtKeyUsageClientAuth,
+		}),
+	)
+
+	ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(etcdCA)
+	if err != nil {
+		return nil, fmt.Errorf("failed loading CA from config: %w", err)
+	}
+
+	keyPair, err := x509.NewKeyPair(ca, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed generating client key pair: %w", err)
+	}
+
+	return x509.NewCertificateAndKeyFromKeyPair(keyPair), nil
+}
+
+// GenerateClientCert generates client certificate and key from etcd CA.
+func GenerateClientCert(etcdCA *x509.PEMEncodedCertificateAndKey, commonName string) (*x509.PEMEncodedCertificateAndKey, error) {
+	opts, err := NewCommonOptions()
+	if err != nil {
+		return nil, err
+	}
+
+	opts = append(opts, x509.CommonName(commonName))
+	opts = append(opts,
+		x509.ExtKeyUsage([]stdlibx509.ExtKeyUsage{
+			stdlibx509.ExtKeyUsageClientAuth,
+		}),
+	)
+
+	ca, err := x509.NewCertificateAuthorityFromCertificateAndKey(etcdCA)
+	if err != nil {
+		return nil, fmt.Errorf("failed loading CA from config: %w", err)
+	}
+
+	keyPair, err := x509.NewKeyPair(ca, opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed generating client key pair: %w", err)
+	}
+
+	return x509.NewCertificateAndKeyFromKeyPair(keyPair), nil
+}
diff --git a/internal/pkg/etcd/etcd.go b/internal/pkg/etcd/etcd.go
@@ -40,8 +40,8 @@ type Client struct {
 // a list of endpoints.
 func NewClient(endpoints []string) (client *Client, err error) {
 	tlsInfo := transport.TLSInfo{
-		CertFile:      constants.KubernetesEtcdPeerCert,
-		KeyFile:       constants.KubernetesEtcdPeerKey,
+		CertFile:      constants.KubernetesEtcdAdminCert,
+		KeyFile:       constants.KubernetesEtcdAdminKey,
 		TrustedCAFile: constants.KubernetesEtcdCACert,
 	}
 

diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go
@@ -141,12 +141,24 @@ const (
 	// KubernetesEtcdCAKey is the path to the etcd CA private key.
 	KubernetesEtcdCAKey = EtcdPKIPath + "/" + "ca.key"
 
-	// KubernetesEtcdPeerCert is the path to the etcd CA certificate.
+	// KubernetesEtcdCert is the path to the etcd server certificate.
+	KubernetesEtcdCert = EtcdPKIPath + "/" + "server.crt"
+
+	// KubernetesEtcdKey is the path to the etcd server private key.
+	KubernetesEtcdKey = EtcdPKIPath + "/" + "server.key"
+
+	// KubernetesEtcdPeerCert is the path to the etcd peer certificate.
 	KubernetesEtcdPeerCert = EtcdPKIPath + "/" + "peer.crt"
 
-	// KubernetesEtcdPeerKey is the path to the etcd CA private key.
+	// KubernetesEtcdPeerKey is the path to the etcd peer private key.
 	KubernetesEtcdPeerKey = EtcdPKIPath + "/" + "peer.key"
 
+	// KubernetesEtcdAdminCert is the path to the talos client certificate.
+	KubernetesEtcdAdminCert = EtcdPKIPath + "/" + "admin.crt"
+
+	// KubernetesEtcdAdminKey is the path to the talos client private key.
+	KubernetesEtcdAdminKey = EtcdPKIPath + "/" + "admin.key"
+
 	// KubernetesEtcdListenClientPort defines the port etcd listen on for client traffic.
 	KubernetesEtcdListenClientPort = "2379"
 

diff --git a/pkg/machinery/go.mod b/pkg/machinery/go.mod
@@ -24,7 +24,7 @@ require (
 	github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.7.0
-	github.com/talos-systems/crypto v0.3.1-0.20210615131117-6bc5bb50c527
+	github.com/talos-systems/crypto v0.3.1-0.20210617123329-d3cb77220384
 	github.com/talos-systems/go-blockdevice v0.2.1
 	github.com/talos-systems/net v0.3.0
 	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22

diff --git a/pkg/machinery/go.sum b/pkg/machinery/go.sum
@@ -144,8 +144,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/talos-systems/crypto v0.3.1-0.20210615131117-6bc5bb50c527 h1:Ut3jI/0FPSyH0QpycKFKAueWKjzQ00OFmkeFG+0uO1A=
-github.com/talos-systems/crypto v0.3.1-0.20210615131117-6bc5bb50c527/go.mod h1:xaNCB2/Bxaj+qrkdeodhRv5eKQVvKOGBBMj58MrIPY8=
+github.com/talos-systems/crypto v0.3.1-0.20210617123329-d3cb77220384 h1:rfFjcSr+oza1k7gMcrYW9RCiVSiFvKUYMB1fcIrwMf0=
+github.com/talos-systems/crypto v0.3.1-0.20210617123329-d3cb77220384/go.mod h1:xaNCB2/Bxaj+qrkdeodhRv5eKQVvKOGBBMj58MrIPY8=
 github.com/talos-systems/go-blockdevice v0.2.1 h1:swoY5NcssuMgdCf/dlMngNDgEAasGp2jviPqAz9Epss=
 github.com/talos-systems/go-blockdevice v0.2.1/go.mod h1:qnn/zDc09I1DA2BUDDCOSA2D0P8pIDjN8pGiRoRaQig=
 github.com/talos-systems/go-cmd v0.0.0-20210216164758-68eb0067e0f0/go.mod h1:kf+rZzTEmlDiYQ6ulslvRONnKLQH8x83TowltGMhO+k=

diff --git a/pkg/resources/secrets/etcd.go b/pkg/resources/secrets/etcd.go
@@ -26,7 +26,10 @@ type Etcd struct {
 
 // EtcdCertsSpec describes etcd certs secrets.
 type EtcdCertsSpec struct {
-	EtcdPeer *x509.PEMEncodedCertificateAndKey `yaml:"etcdPeer"`
+	Etcd          *x509.PEMEncodedCertificateAndKey `yaml:"etcd"`
+	EtcdPeer      *x509.PEMEncodedCertificateAndKey `yaml:"etcdPeer"`
+	EtcdAdmin     *x509.PEMEncodedCertificateAndKey `yaml:"etcdAdmin"`
+	EtcdAPIServer *x509.PEMEncodedCertificateAndKey `yaml:"etcdAPIServer"`
 }
 
 // NewEtcd initializes a Etc resource.