From d9bc6cf6990639f68b4ba591fe84057176ba1572 Mon Sep 17 00:00:00 2001 From: jannfis Date: Mon, 26 Apr 2021 19:56:09 +0200 Subject: [PATCH] feat: Externalize TLS cert for argocd-server (#6071) * feat: Externalize TLS cert for argocd-server Signed-off-by: jannfis * Update docs for mTLS scenarios Signed-off-by: jannfis * Inline documentation Signed-off-by: jannfis --- docs/operator-manual/tls.md | 164 ++++++++++++++++++++++++++++ mkdocs.yml | 1 + util/settings/settings.go | 63 +++++++++-- util/settings/settings_test.go | 188 +++++++++++++++++++++++++++++++++ 4 files changed, 406 insertions(+), 10 deletions(-) create mode 100644 docs/operator-manual/tls.md diff --git a/docs/operator-manual/tls.md b/docs/operator-manual/tls.md new file mode 100644 index 0000000000000..4014f08ddef9c --- /dev/null +++ b/docs/operator-manual/tls.md @@ -0,0 +1,164 @@ +# TLS configuration + +Argo CD provides two inbound TLS endpoints that can be configured: + +* The user-facing endpoint of the `argocd-server` workload which serves the UI + and the API +* The endpoint of the `argocd-repo-server`, which is accessed by `argocd-server` + and `argocd-application-controller` workloads to request repository + operations. + +By default, and without further configuration, both of these endpoints will be +set-up to use an automatically generated, self-signed certificate. However, +most users will want to explicitly configure the certificates for these TLS +endpoints, possibly using automated means such as `cert-manager` or using +their own dedicated Certificate Authority. + +## Configuring TLS for argocd-server + +### Inbound TLS options for argocd-server + +You can configure certain TLS options for the `argocd-server` workload by +setting command line parameters. The following parameters are available: + +|Parameter|Default|Description| +|---------|-------|-----------| +|`--insecure`|`false`|Disables TLS completely| +|`--tlsminversion`|`1.2`|The minimum TLS version to be offered to clients| +|`--tlsmaxversion`|`1.3`|The maximum TLS version to be offered to clients| +|`--tlsciphers`|`TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_256_GCM_SHA384`|A colon separated list of TLS cipher suites to be offered to clients| + +### TLS certificates used by argocd-sever + +There are two ways to configure the TLS certificates used by `argocd-server`: + +* Setting the `tls.crt` and `tls.key` keys in the `argocd-server-tls` secret + to hold PEM data of the certificate and the corresponding private key. The + `argocd-server-tls` secret may be of type `tls`, but does not have to be. +* Setting the `tls.crt` and `tls.key` keys in the `argocd-secret` secret to + hold PEM data of the certificate and the corresponding private key. This + method is considered deprecated, and only exists for purposes of backwards + compatiblity. Changing `argocd-secret` should not be used to override the + TLS certificate anymore. + +Argo CD decides which TLS certificate to use for the endpoint of +`argocd-server` as follows: + +* If the `argocd-server-tls` secret exists and contains a valid key pair in the + `tls.crt` and `tls.key` keys, this will be used for the certificate of the + endpoint of `argocd-server`. +* Otherwise, if the `argocd-secret` secret contains a valid key pair in the + `tls.crt` and `tls.key` keys, this will be used as certificate for the + endpoint of `argocd-server`. +* If no `tls.crt` and `tls.key` keys are found in neither of the two mentioned + secrets, Argo CD will generate a self-signed certificate and persist it in + the `argocd-secret` secret. + +The `argocd-server-tls` secret contains only information for TLS configuration +to be used by `argocd-server` and is safe to be managed via third-party tools +such as `cert-manager` or `SealedSecrets` + +To create this secret manually from an existing key pair, you can use `kubectl`: + +```shell +kubectl create -n argocd secret tls argocd-server-tls \ + --cert=/path/to/cert.pem \ + --key=/path/to/key.pem +``` + +Argo CD will pick up changes to the `argocd-server-tls` secret automatically +and will not require restart of the pods to use a renewed certificate. + +## Configuring inbound TLS for argocd-repo-server + +### Inbound TLS options for argocd-repo-server + +You can configure certain TLS options for the `argocd-repo-server` workload by +setting command line parameters. The following parameters are available: + +|Parameter|Default|Description| +|---------|-------|-----------| +|`--disable-tls`|`false`|Disables TLS completely| +|`--tlsminversion`|`1.2`|The minimum TLS version to be offered to clients| +|`--tlsmaxversion`|`1.3`|The maximum TLS version to be offered to clients| +|`--tlsciphers`|`TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_256_GCM_SHA384`|A colon separated list of TLS cipher suites to be offered to clients| + +### Inbound TLS certificates used by argocd-sever + +To configure the TLS certificate used by the `argocd-repo-server` workload, +create a secret named `argocd-repo-server-tls` in the namespace where Argo CD +is running in with the certificate's key pair stored in `tls.crt` and +`tls.key` keys. If this secret does not exist, `argocd-repo-server` will +generate and use a self-signed certificate. + +To create this secret, you can use `kubectl`: + +```shell +kubectl create -n argocd secret tls argocd-repo-server-tls \ + --cert=/path/to/cert.pem \ + --key=/path/to/key.pem +``` + +Please note, that as opposed to `argocd-server`, the `argocd-repo-server` is +not able to pick up changes to this secret automatically. If you create (or +update) this secret, the `argocd-repo-server` pods need to be restarted. + +Also note, that the certificate should be issued with the correct SAN entries +for the `argocd-repo-server`, containing at least the entries for +`DNS:argocd-repo-server` and `DNS:argocd-repo-server.argo-cd.svc` depending +on how your workloads connect to the repository server. + +## Configuring TLS between Argo CD components + +### Configuring TLS to argocd-repo-server + +Both `argocd-server` and `argocd-application-controller` communicate with the +`argocd-repo-server` using a gRPC API over TLS. By default, +`argocd-repo-server` generates a non-persistant, self signed certificate +to use for its gRPC endpoint on startup. Because the `argocd-repo-server` has +no means to connect to the K8s control plane API, this certificate is not +being available to outside consumers for verification. Both, the +`argocd-server` and `argocd-application-server` will use a non-validating +connection to the `argocd-repo-server` for this reason. + +To change this behavior to be more secure by having the `argocd-server` and +`argocd-application-controller` validate the TLS certificate of the +`argocd-repo-server` endpoint, the following steps need to be performed: + +* Create a persistent TLS certificate to be used by `argocd-repo-server`, as + shown above +* Restart the `argocd-repo-server` pod(s) +* Modify the pod startup parameters for `argocd-server` and + `argocd-application-controller` to include the `--repo-server-strict-tls` + parameter. + +The `argocd-server` and `argocd-application-controller` workloads will now +validate the TLS certificate of the `argocd-repo-server` by using the +certificate stored in the `argocd-repo-server-tls` secret. + +!!!note "Certificate expiry" + Please make sure that the certificate has a proper life time. Keep in + mind that when you have to replace the certificate, all workloads have + to be restarted in order to properly work again. + +### Disabling TLS to argocd-repo-server + +In some scenarios where mTLS through side-car proxies is involved (e.g. +in a service mesh), you may want configure the connections between the +`argocd-server` and `argocd-application-controller` to `argocd-repo-server` +to not use TLS at all. + +In this case, you will need to: + +* Configure `argocd-repo-server` with TLS on the gRPC API disabled by specifying + the `--disable-tls` parameter to the pod container's startup arguments +* Configure `argocd-server` and `argocd-application-controller` to not use TLS + for connections to the `argocd-repo-server` by specifying the parameter + `--repo-server-plaintext` to the pod container's startup arguments +* Configure `argocd-server` and `argocd-application-controller` to connect to + the side-car instead of directly to the `argocd-repo-server` service by + specifying its address via the `--repo-server
` parameter + +After this change, the `argocd-server` and `argocd-application-controller` will +use a plain text connection to the side-car proxy, that will handle all aspects +of TLS to the `argocd-repo-server`'s TLS side-car proxy. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 94f083b8a2103..81795c19c1a9e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,6 +44,7 @@ nav: - operator-manual/user-management/google.md - operator-manual/rbac.md - operator-manual/security.md + - operator-manual/tls.md - operator-manual/cluster-bootstrapping.md - operator-manual/secret-management.md - operator-manual/high_availability.md diff --git a/util/settings/settings.go b/util/settings/settings.go index f23eaa9a689dd..a222d7b3bb028 100644 --- a/util/settings/settings.go +++ b/util/settings/settings.go @@ -53,6 +53,8 @@ type ArgoCDSettings struct { // Certificate holds the certificate/private key for the Argo CD API server. // If nil, will run insecure without TLS. Certificate *tls.Certificate `json:"-"` + // CertificateIsExternal indicates whether Certificate was loaded from external secret + CertificateIsExternal bool `json:"-"` // WebhookGitLabSecret holds the shared secret for authenticating GitHub webhook events WebhookGitHubSecret string `json:"webhookGitHubSecret,omitempty"` // WebhookGitLabSecret holds the shared secret for authenticating GitLab webhook events @@ -298,6 +300,8 @@ const ( initialPasswordSecretField = "password" // initialPasswordLength defines the length of the generated initial password initialPasswordLength = 16 + // externalServerTLSSecretName defines the name of the external secret holding the server's TLS certificate + externalServerTLSSecretName = "argocd-server-tls" ) // SettingsManager holds config info for a new manager with which to access Kubernetes ConfigMaps. @@ -872,7 +876,7 @@ func (mgr *SettingsManager) GetSettings() (*ArgoCDSettings, error) { var settings ArgoCDSettings var errs []error updateSettingsFromConfigMap(&settings, argoCDCM) - if err := updateSettingsFromSecret(&settings, argoCDSecret); err != nil { + if err := mgr.updateSettingsFromSecret(&settings, argoCDSecret); err != nil { errs = append(errs, err) } if len(errs) > 0 { @@ -1015,7 +1019,7 @@ func validateExternalURL(u string) error { } // updateSettingsFromSecret transfers settings from a Kubernetes secret into an ArgoCDSettings struct. -func updateSettingsFromSecret(settings *ArgoCDSettings, argoCDSecret *apiv1.Secret) error { +func (mgr *SettingsManager) updateSettingsFromSecret(settings *ArgoCDSettings, argoCDSecret *apiv1.Secret) error { var errs []error secretKey, ok := argoCDSecret.Data[settingServerSignatureKey] if ok { @@ -1039,14 +1043,29 @@ func updateSettingsFromSecret(settings *ArgoCDSettings, argoCDSecret *apiv1.Secr settings.WebhookGogsSecret = string(gogsWebhookSecret) } - serverCert, certOk := argoCDSecret.Data[settingServerCertificate] - serverKey, keyOk := argoCDSecret.Data[settingServerPrivateKey] - if certOk && keyOk { - cert, err := tls.X509KeyPair(serverCert, serverKey) - if err != nil { - errs = append(errs, &incompleteSettingsError{message: fmt.Sprintf("invalid x509 key pair %s/%s in secret: %s", settingServerCertificate, settingServerPrivateKey, err)}) + // The TLS certificate may be externally managed. We try to load it from an + // external secret first. If the external secret doesn't exist, we either + // load it from argocd-secret or generate (and persist) a self-signed one. + cert, err := mgr.externalServerTLSCertificate() + if err != nil { + errs = append(errs, &incompleteSettingsError{message: fmt.Sprintf("could not read from secret %s/%s: %v", mgr.namespace, externalServerTLSSecretName, err)}) + } else { + if cert != nil { + settings.Certificate = cert + settings.CertificateIsExternal = true + log.Infof("Loading TLS configuration from secret %s/%s", mgr.namespace, externalServerTLSSecretName) } else { - settings.Certificate = &cert + serverCert, certOk := argoCDSecret.Data[settingServerCertificate] + serverKey, keyOk := argoCDSecret.Data[settingServerPrivateKey] + if certOk && keyOk { + cert, err := tls.X509KeyPair(serverCert, serverKey) + if err != nil { + errs = append(errs, &incompleteSettingsError{message: fmt.Sprintf("invalid x509 key pair %s/%s in secret: %s", settingServerCertificate, settingServerPrivateKey, err)}) + } else { + settings.Certificate = &cert + settings.CertificateIsExternal = false + } + } } } secretValues := make(map[string]string, len(argoCDSecret.Data)) @@ -1060,6 +1079,28 @@ func updateSettingsFromSecret(settings *ArgoCDSettings, argoCDSecret *apiv1.Secr return nil } +// externalServerTLSCertificate will try and load a TLS certificate from an +// external secret, instead of tls.crt and tls.key in argocd-secret. If both +// return values are nil, no external secret has been configured. +func (mgr *SettingsManager) externalServerTLSCertificate() (*tls.Certificate, error) { + var cert tls.Certificate + secret, err := mgr.clientset.CoreV1().Secrets(mgr.namespace).Get(mgr.ctx, externalServerTLSSecretName, metav1.GetOptions{}) + if err != nil { + if apierr.IsNotFound(err) { + return nil, nil + } + } + tlsCert, certOK := secret.Data[settingServerCertificate] + tlsKey, keyOK := secret.Data[settingServerPrivateKey] + if certOK && keyOK { + cert, err = tls.X509KeyPair(tlsCert, tlsKey) + if err != nil { + return nil, err + } + } + return &cert, nil +} + // SaveSettings serializes ArgoCDSettings and upserts it into K8s secret/configmap func (mgr *SettingsManager) SaveSettings(settings *ArgoCDSettings) error { err := mgr.updateConfigMap(func(argoCDCM *apiv1.ConfigMap) error { @@ -1115,7 +1156,9 @@ func (mgr *SettingsManager) SaveSettings(settings *ArgoCDSettings) error { if settings.WebhookGogsSecret != "" { argoCDSecret.Data[settingsWebhookGogsSecretKey] = []byte(settings.WebhookGogsSecret) } - if settings.Certificate != nil { + // we only write the certificate to the secret if it's not externally + // managed. + if settings.Certificate != nil && !settings.CertificateIsExternal { cert, key := tlsutil.EncodeX509KeyPair(*settings.Certificate) argoCDSecret.Data[settingServerCertificate] = cert argoCDSecret.Data[settingServerPrivateKey] = key diff --git a/util/settings/settings_test.go b/util/settings/settings_test.go index 2510b167d5f09..3fb6f8e338142 100644 --- a/util/settings/settings_test.go +++ b/util/settings/settings_test.go @@ -2,11 +2,14 @@ package settings import ( "context" + "crypto/tls" + "crypto/x509" "sort" "testing" "github.com/argoproj/argo-cd/v2/common" "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + testutil "github.com/argoproj/argo-cd/v2/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" @@ -681,3 +684,188 @@ func TestGetOIDCSecretTrim(t *testing.T) { assert.NotNil(t, oidcConfig) assert.Equal(t, "test-secret", oidcConfig.ClientSecret) } + +func getCNFromCertificate(cert *tls.Certificate) string { + c, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return "" + } + return c.Subject.CommonName +} + +func Test_GetTLSConfiguration(t *testing.T) { + t.Run("Valid external TLS secret with success", func(t *testing.T) { + kubeClient := fake.NewSimpleClientset( + &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDConfigMapName, + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string]string{ + "oidc.config": "\n name: Okta\n clientSecret: test-secret\r\n \n clientID: aaaabbbbccccddddeee\n", + }, + }, + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDSecretName, + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string][]byte{ + "admin.password": nil, + "server.secretkey": nil, + }, + }, + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: externalServerTLSSecretName, + Namespace: "default", + }, + Data: map[string][]byte{ + "tls.crt": []byte(testutil.MustLoadFileToString("../../test/fixture/certs/argocd-test-server.crt")), + "tls.key": []byte(testutil.MustLoadFileToString("../../test/fixture/certs/argocd-test-server.key")), + }, + }, + ) + settingsManager := NewSettingsManager(context.Background(), kubeClient, "default") + settings, err := settingsManager.GetSettings() + assert.NoError(t, err) + assert.True(t, settings.CertificateIsExternal) + assert.NotNil(t, settings.Certificate) + assert.Contains(t, getCNFromCertificate(settings.Certificate), "localhost") + }) + + t.Run("Valid external TLS secret overrides argocd-secret", func(t *testing.T) { + kubeClient := fake.NewSimpleClientset( + &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDConfigMapName, + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string]string{ + "oidc.config": "\n name: Okta\n clientSecret: test-secret\r\n \n clientID: aaaabbbbccccddddeee\n", + }, + }, + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDSecretName, + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string][]byte{ + "admin.password": nil, + "server.secretkey": nil, + "tls.crt": []byte(testutil.MustLoadFileToString("../../test/fixture/certs/argocd-e2e-server.crt")), + "tls.key": []byte(testutil.MustLoadFileToString("../../test/fixture/certs/argocd-e2e-server.key")), + }, + }, + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: externalServerTLSSecretName, + Namespace: "default", + }, + Data: map[string][]byte{ + "tls.crt": []byte(testutil.MustLoadFileToString("../../test/fixture/certs/argocd-test-server.crt")), + "tls.key": []byte(testutil.MustLoadFileToString("../../test/fixture/certs/argocd-test-server.key")), + }, + }, + ) + settingsManager := NewSettingsManager(context.Background(), kubeClient, "default") + settings, err := settingsManager.GetSettings() + assert.NoError(t, err) + assert.True(t, settings.CertificateIsExternal) + assert.NotNil(t, settings.Certificate) + assert.Contains(t, getCNFromCertificate(settings.Certificate), "localhost") + }) + t.Run("Invalid external TLS secret", func(t *testing.T) { + kubeClient := fake.NewSimpleClientset( + &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDConfigMapName, + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string]string{ + "oidc.config": "\n name: Okta\n clientSecret: test-secret\r\n \n clientID: aaaabbbbccccddddeee\n", + }, + }, + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDSecretName, + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string][]byte{ + "admin.password": nil, + "server.secretkey": nil, + }, + }, + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: externalServerTLSSecretName, + Namespace: "default", + }, + Data: map[string][]byte{ + "tls.crt": []byte(""), + "tls.key": []byte(""), + }, + }, + ) + settingsManager := NewSettingsManager(context.Background(), kubeClient, "default") + settings, err := settingsManager.GetSettings() + assert.Error(t, err) + assert.Contains(t, err.Error(), "could not read from secret") + assert.NotNil(t, settings) + }) + t.Run("No external TLS secret", func(t *testing.T) { + kubeClient := fake.NewSimpleClientset( + &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDConfigMapName, + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string]string{ + "oidc.config": "\n name: Okta\n clientSecret: test-secret\r\n \n clientID: aaaabbbbccccddddeee\n", + }, + }, + &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ArgoCDSecretName, + Namespace: "default", + Labels: map[string]string{ + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string][]byte{ + "admin.password": nil, + "server.secretkey": nil, + "tls.crt": []byte(testutil.MustLoadFileToString("../../test/fixture/certs/argocd-e2e-server.crt")), + "tls.key": []byte(testutil.MustLoadFileToString("../../test/fixture/certs/argocd-e2e-server.key")), + }, + }, + ) + settingsManager := NewSettingsManager(context.Background(), kubeClient, "default") + settings, err := settingsManager.GetSettings() + assert.NoError(t, err) + assert.False(t, settings.CertificateIsExternal) + assert.NotNil(t, settings.Certificate) + assert.Contains(t, getCNFromCertificate(settings.Certificate), "Argo CD E2E") + }) +}