diff --git a/go.mod b/go.mod index 11ef406e143..e908c993958 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/lib/pq v1.10.6 github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a github.com/onsi/gomega v1.30.0 + github.com/open-policy-agent/cert-controller v0.10.1 github.com/prometheus/client_golang v1.18.0 github.com/shirou/gopsutil/v3 v3.22.5 github.com/spf13/viper v1.9.0 @@ -123,6 +124,7 @@ require ( github.com/tklauser/numcpus v0.4.0 // indirect github.com/vbatts/tar-split v0.11.3 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.21.0 // indirect diff --git a/go.sum b/go.sum index 5d756af16fc..f0f3ee30e17 100644 --- a/go.sum +++ b/go.sum @@ -540,6 +540,10 @@ github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= +github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4/go.mod h1:54/KzLMvA5ndBVpm7B1OjLeV0cUtTLTz2bZ2OtydLpU= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= @@ -687,6 +691,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1248,6 +1254,8 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/kube-aggregator v0.28.1 h1:rvG4llYnQKHjj6YjjoBPEJxfD1uH0DJwkrJTNKGAaCs= +k8s.io/kube-aggregator v0.28.1/go.mod h1:JaLizMe+AECSpO2OmrWVsvnG0V3dX1RpW+Wq/QHbu18= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= diff --git a/manifests/v1beta1/components/controller/rbac.yaml b/manifests/v1beta1/components/controller/rbac.yaml index 3118ea7dedf..e93fbd031cf 100644 --- a/manifests/v1beta1/components/controller/rbac.yaml +++ b/manifests/v1beta1/components/controller/rbac.yaml @@ -58,6 +58,7 @@ rules: - "list" - "watch" - "patch" + - "update" - apiGroups: - apps resources: @@ -127,6 +128,7 @@ rules: - "watch" - "list" - "patch" + - "update" --- apiVersion: v1 kind: ServiceAccount diff --git a/pkg/certgenerator/v1beta1/certificate.go b/pkg/certgenerator/v1beta1/certificate.go deleted file mode 100644 index 19d07615f67..00000000000 --- a/pkg/certgenerator/v1beta1/certificate.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2022 The Kubeflow Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certgenerator - -import ( - "bytes" - "crypto/rsa" - "crypto/x509" - "encoding/pem" -) - -// certificates contains all certificates for katib-webhook-cert. -type certificates struct { - certPem []byte - keyPem []byte - cert *x509.Certificate - key *rsa.PrivateKey -} - -// encode creates PEM key and convert DER to CRT. -func encode(rawKey *rsa.PrivateKey, der []byte) (*certificates, error) { - keyPem := &bytes.Buffer{} - if err := pem.Encode(keyPem, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rawKey)}); err != nil { - return nil, err - } - - certPem := &bytes.Buffer{} - if err := pem.Encode(certPem, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil { - return nil, err - } - - cert, err := x509.ParseCertificate(der) - if err != nil { - return nil, err - } - - return &certificates{ - certPem: certPem.Bytes(), - keyPem: keyPem.Bytes(), - cert: cert, - key: rawKey, - }, nil -} diff --git a/pkg/certgenerator/v1beta1/const.go b/pkg/certgenerator/v1beta1/const.go deleted file mode 100644 index 09aac6f6669..00000000000 --- a/pkg/certgenerator/v1beta1/const.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright 2022 The Kubeflow Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certgenerator - -const ( - Webhook = "katib.kubeflow.org" - serverKeyName = "tls.key" - serverCertName = "tls.crt" - ssaFieldOwnerName = "cert-generator" -) diff --git a/pkg/certgenerator/v1beta1/generator.go b/pkg/certgenerator/v1beta1/generator.go index 0784a625ca7..3a61bf7ac45 100644 --- a/pkg/certgenerator/v1beta1/generator.go +++ b/pkg/certgenerator/v1beta1/generator.go @@ -17,276 +17,41 @@ limitations under the License. package certgenerator import ( - "bytes" - "context" - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "crypto/x509/pkix" - "errors" "fmt" - "math/big" - "os" - "path/filepath" - "strings" - "time" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" + cert "github.com/open-policy-agent/cert-controller/pkg/rotator" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/manager" configv1beta1 "github.com/kubeflow/katib/pkg/apis/config/v1beta1" "github.com/kubeflow/katib/pkg/controller.v1beta1/consts" ) -var ( - log = logf.Log.WithName("cert-generator") +const Webhook = "katib.kubeflow.org" - errServiceNotFound = errors.New("unable to locate controller service") - errCertCheckFail = errors.New("failed to check if certs already exist") - errCreateCertFail = errors.New("failed to create certs") - errCreateCertSecretFail = errors.New("failed to create secret embedded certs") - errInjectCertError = errors.New("failed to inject certs into WebhookConfigurations") -) - -// CertGenerator is the manager to generate certs. -type CertGenerator struct { - namespace string - webhookServiceName string - webhookSecretName string - fullServiceDomain string - kubeClient client.Client - certsReady chan struct{} - - certs *certificates -} - -var _ manager.Runnable = &CertGenerator{} -var _ manager.LeaderElectionRunnable = &CertGenerator{} - -func (c *CertGenerator) Start(ctx context.Context) error { - if err := c.generate(ctx); err != nil { - return err - } - log.Info("Waiting for certs to get ready.") - if err := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ - Duration: time.Second, - Factor: 2, - Jitter: 1, - Steps: 10, - Cap: time.Minute * 5, - }, ensureCertMounted(time.Now())); err != nil { - return err - } - // Sending an empty data to a certsReady means it starts to register controllers to the manager. - c.certsReady <- struct{}{} - return nil -} - -// ensureCertMounted ensures that the generated certs are mounted inside the container. -func ensureCertMounted(start time.Time) func(context.Context) (bool, error) { - return func(ctx context.Context) (bool, error) { - now := time.Now() - outputLog := false - if now.Sub(start) >= 15*time.Second { - start = now - outputLog = true - } - - certFile := filepath.Join(consts.CertDir, serverCertName) - if _, err := os.Stat(certFile); err != nil { - if outputLog { - log.Info("Public key file doesn't exist in the container yet.", "publicKeyFile", certFile) - } - return false, nil - } - keyFile := filepath.Join(consts.CertDir, serverKeyName) - if _, err := os.Stat(keyFile); err != nil { - if outputLog { - log.Info("Private key file doesn't exist in the container yet.", "privateKeyFile", keyFile) - } - return false, nil - } - log.Info("Succeeded to be mounted certs inside the container.") - return true, nil - } -} - -func (c *CertGenerator) NeedLeaderElection() bool { - return false -} +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=get;list;watch;update // AddToManager adds the cert-generator to the manager. -func AddToManager(mgr manager.Manager, config configv1beta1.CertGeneratorConfig, certsReady chan struct{}) error { - return mgr.Add(&CertGenerator{ - namespace: consts.DefaultKatibNamespace, - webhookServiceName: config.WebhookServiceName, - webhookSecretName: config.WebhookSecretName, - fullServiceDomain: strings.Join([]string{ - config.WebhookServiceName, - consts.DefaultKatibNamespace, - "svc", - }, "."), - kubeClient: mgr.GetClient(), - certsReady: certsReady, - }) -} - -// generate generates certificates for the admission webhooks. -func (c *CertGenerator) generate(ctx context.Context) error { - controllerService := &corev1.Service{} - if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: c.webhookServiceName, Namespace: c.namespace}, controllerService); err != nil { - return fmt.Errorf("%w: %v", errServiceNotFound, err) - } - - certExist, err := c.isCertExist(ctx) - if err != nil { - return fmt.Errorf("%w: %v", errCertCheckFail, err) - } - if !certExist { - if err = c.createCert(); err != nil { - return fmt.Errorf("%w: %v", errCreateCertFail, err) - } - if err = c.updateCertSecret(ctx); err != nil { - return fmt.Errorf("%w: %v", errCreateCertSecretFail, err) - } - } - if err = c.injectCert(ctx); err != nil { - return fmt.Errorf("%w: %v", errInjectCertError, err) - } - return nil -} - -// isCertExist checks if a secret embedded certs already exists. -// For example, it will return true if the katib-controller is created with enabled leader-election -// since another controller pod will create the secret. -func (c *CertGenerator) isCertExist(ctx context.Context) (bool, error) { - secret := &corev1.Secret{} - if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: c.webhookSecretName, Namespace: c.namespace}, secret); err != nil { - return false, err - } - key := secret.Data[serverKeyName] - cert := secret.Data[serverCertName] - if len(key) != 0 && len(cert) != 0 { - c.certs = &certificates{ - keyPem: key, - certPem: cert, - } - return true, nil - } - return false, nil -} - -// createCert creates the self-signed certificate and private key. -func (c *CertGenerator) createCert() error { - now := time.Now() - template := &x509.Certificate{ - SerialNumber: big.NewInt(0), - Subject: pkix.Name{ - CommonName: c.fullServiceDomain, +func AddToManager(mgr manager.Manager, cfg configv1beta1.CertGeneratorConfig, certsReady chan struct{}) error { + return cert.AddRotator(mgr, &cert.CertRotator{ + SecretKey: types.NamespacedName{ + Namespace: consts.DefaultKatibNamespace, + Name: cfg.WebhookSecretName, }, - DNSNames: []string{ - c.fullServiceDomain, + CertDir: consts.CertDir, + CAName: "katib-ca", + CAOrganization: "katib", + DNSName: fmt.Sprintf("%s.%s.svc", cfg.WebhookServiceName, consts.DefaultKatibNamespace), + IsReady: certsReady, + Webhooks: []cert.WebhookInfo{ + {Name: Webhook, Type: cert.Validating}, + {Name: Webhook, Type: cert.Mutating}, }, - NotBefore: now, - NotAfter: now.Add(24 * time.Hour * 365 * 10), - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - } - - log.Info("Generating self-signed public certificate and private key.") - rawKey, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - return err - } - - der, err := x509.CreateCertificate(rand.Reader, template, template, rawKey.Public(), rawKey) - if err != nil { - return err - } - if c.certs, err = encode(rawKey, der); err != nil { - return err - } - return nil -} - -// updateCertSecret updates Secret embedded tls.key and tls.crt. -func (c *CertGenerator) updateCertSecret(ctx context.Context) error { - secret := &corev1.Secret{} - if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: c.webhookSecretName, Namespace: c.namespace}, secret); err != nil { - return err - } - newSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - UID: secret.UID, - Name: secret.Name, - Namespace: secret.Namespace, - }, - TypeMeta: secret.TypeMeta, - } - newSecret.Data = map[string][]byte{ - serverKeyName: c.certs.keyPem, - serverCertName: c.certs.certPem, - } - return c.kubeClient.Patch(ctx, newSecret, client.Apply, client.FieldOwner(ssaFieldOwnerName), client.ForceOwnership) -} - -// injectCert applies patch to ValidatingWebhookConfiguration and MutatingWebhookConfiguration. -func (c *CertGenerator) injectCert(ctx context.Context) error { - vWebhookConfig := &admissionregistrationv1.ValidatingWebhookConfiguration{} - if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: Webhook}, vWebhookConfig); err != nil { - return err - } - if !bytes.Equal(vWebhookConfig.Webhooks[0].ClientConfig.CABundle, c.certs.certPem) { - newVWebhookConfig := &admissionregistrationv1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - UID: vWebhookConfig.UID, - Name: vWebhookConfig.Name, - Namespace: vWebhookConfig.Namespace, - Generation: vWebhookConfig.Generation, - }, - TypeMeta: vWebhookConfig.TypeMeta, - } - newVWebhookConfig.Webhooks = vWebhookConfig.Webhooks - newVWebhookConfig.Webhooks[0].ClientConfig.CABundle = c.certs.certPem - - log.Info("Trying to patch ValidatingWebhookConfiguration adding the caBundle.") - err := c.kubeClient.Patch(ctx, newVWebhookConfig, client.Apply, client.FieldOwner(ssaFieldOwnerName), client.ForceOwnership) - if err != nil { - log.Error(err, "Unable to patch ValidatingWebhookConfiguration", "ValidatingWebhookConfiguration", Webhook) - return err - } - } - - mWebhookConfig := &admissionregistrationv1.MutatingWebhookConfiguration{} - if err := c.kubeClient.Get(ctx, client.ObjectKey{Name: Webhook}, mWebhookConfig); err != nil { - return err - } - if !bytes.Equal(mWebhookConfig.Webhooks[0].ClientConfig.CABundle, c.certs.certPem) || - !bytes.Equal(mWebhookConfig.Webhooks[1].ClientConfig.CABundle, c.certs.certPem) { - newMWebhookConfig := &admissionregistrationv1.MutatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - UID: mWebhookConfig.UID, - Name: mWebhookConfig.Name, - Namespace: mWebhookConfig.Namespace, - Generation: mWebhookConfig.Generation, - }, - TypeMeta: mWebhookConfig.TypeMeta, - } - newMWebhookConfig.Webhooks = mWebhookConfig.Webhooks - newMWebhookConfig.Webhooks[0].ClientConfig.CABundle = c.certs.certPem - newMWebhookConfig.Webhooks[1].ClientConfig.CABundle = c.certs.certPem - - log.Info("Trying to patch MutatingWebhookConfiguration adding the caBundle.") - err := c.kubeClient.Patch(ctx, newMWebhookConfig, client.Apply, client.FieldOwner(ssaFieldOwnerName), client.ForceOwnership) - if err != nil { - log.Error(err, "Unable to patch MutatingWebhookConfiguration", "MutatingWebhookConfiguration", Webhook) - return err - } - } - return nil + FieldOwner: "cert-generator", + // When the Katib is running in the leader election mode, + // we expect webhook server will run in primary and secondary instance + RequireLeaderElection: false, + }) } diff --git a/pkg/certgenerator/v1beta1/generator_test.go b/pkg/certgenerator/v1beta1/generator_test.go deleted file mode 100644 index 05dc5600749..00000000000 --- a/pkg/certgenerator/v1beta1/generator_test.go +++ /dev/null @@ -1,306 +0,0 @@ -/* -Copyright 2022 The Kubeflow Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package certgenerator - -import ( - "context" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - admissionregistration "k8s.io/api/admissionregistration/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - - configv1beta1 "github.com/kubeflow/katib/pkg/apis/config/v1beta1" - "github.com/kubeflow/katib/pkg/controller.v1beta1/consts" -) - -func TestGenerate(t *testing.T) { - const testNamespace = "test" - - emptyVWebhookConfig := &admissionregistration.ValidatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: admissionregistration.SchemeGroupVersion.String(), - Kind: "ValidatingWebhookConfiguration", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: Webhook, - }, - Webhooks: []admissionregistration.ValidatingWebhook{ - { - Name: strings.Join([]string{"validator.experiment", Webhook}, "."), - ClientConfig: admissionregistration.WebhookClientConfig{}, - }, - }, - } - emptyMWebhookConfig := &admissionregistration.MutatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: admissionregistration.SchemeGroupVersion.String(), - Kind: "MutatingWebhookConfiguration", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: Webhook, - }, - Webhooks: []admissionregistration.MutatingWebhook{ - { - Name: strings.Join([]string{"defaulter.experiment", Webhook}, "."), - ClientConfig: admissionregistration.WebhookClientConfig{}, - }, - { - Name: strings.Join([]string{"mutator.pod", Webhook}, "."), - ClientConfig: admissionregistration.WebhookClientConfig{}, - }, - }, - } - webhookSecret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "katib-test-secret", - Namespace: testNamespace, - }, - } - controllerService := &corev1.Service{ - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: configv1beta1.DefaultWebhookServiceName, - Namespace: testNamespace, - }, - } - - tests := map[string]struct { - objects []client.Object - opts *CertGenerator - wantError error - }{ - "Generate successfully": { - opts: &CertGenerator{ - namespace: testNamespace, - webhookServiceName: "katib-controller", - webhookSecretName: "katib-test-secret", - }, - objects: []client.Object{ - emptyVWebhookConfig, - emptyMWebhookConfig, - controllerService, - webhookSecret, - }, - }, - "There is not ValidatingWebhookConfiguration": { - opts: &CertGenerator{ - namespace: testNamespace, - webhookServiceName: "katib-controller", - webhookSecretName: "katib-test-secret", - }, - objects: []client.Object{ - emptyMWebhookConfig, - controllerService, - webhookSecret, - }, - wantError: errInjectCertError, - }, - "There is not MutatingWebhookConfiguration": { - opts: &CertGenerator{ - namespace: testNamespace, - webhookServiceName: "katib-controller", - webhookSecretName: "katib-test-secret", - }, - objects: []client.Object{ - emptyVWebhookConfig, - controllerService, - webhookSecret, - }, - wantError: errInjectCertError, - }, - "There is not Service katib-controller": { - opts: &CertGenerator{ - namespace: testNamespace, - webhookServiceName: "katib-controller", - webhookSecretName: "katib-test-secret", - }, - objects: []client.Object{ - emptyVWebhookConfig, - emptyMWebhookConfig, - webhookSecret, - }, - wantError: errServiceNotFound, - }, - "There is not Secret katib-webhook-cert": { - opts: &CertGenerator{ - namespace: testNamespace, - webhookServiceName: "katib-controller", - webhookSecretName: "katib-test-secret", - }, - objects: []client.Object{ - emptyVWebhookConfig, - emptyMWebhookConfig, - controllerService, - }, - wantError: errCertCheckFail, - }, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - kc := buildFakeClient(tc.objects) - tc.opts.kubeClient = kc - err := tc.opts.generate(context.Background()) - if diff := cmp.Diff(tc.wantError, err, cmpopts.EquateErrors()); len(diff) != 0 { - t.Errorf("Unexpected error from generate() (-want,+got):\n%s", diff) - } - - if tc.wantError == nil { - secret := &corev1.Secret{} - if err = kc.Get(context.Background(), client.ObjectKey{Name: tc.opts.webhookSecretName, Namespace: testNamespace}, secret); err != nil { - t.Fatalf("Failed to get a webhookSecret: %v", err) - } - if len(secret.Data[serverKeyName]) == 0 { - t.Errorf("Unexpected tls.key embedded in secret: %v", secret.Data) - } - if len(secret.Data[serverCertName]) == 0 { - t.Errorf("Unexpected tls.crt embedded in secret: %v", secret.Data) - } - - vConfig := &admissionregistration.ValidatingWebhookConfiguration{} - if err = kc.Get(context.Background(), client.ObjectKey{Name: Webhook}, vConfig); err != nil { - t.Fatalf("Failed to get a ValidatingWebhookConfiguration: %v", err) - } - if len(vConfig.Webhooks[0].ClientConfig.CABundle) == 0 { - t.Errorf("Unexpected tls.crt embedded in ValidatingWebhookConfiguration: %v", vConfig.Webhooks) - } - - mConfig := &admissionregistration.MutatingWebhookConfiguration{} - if err = kc.Get(context.Background(), client.ObjectKey{Name: Webhook}, mConfig); err != nil { - t.Fatalf("Failed to get a MutatingWebhookConfiguration: %v", err) - } - if len(mConfig.Webhooks[0].ClientConfig.CABundle) == 0 || len(mConfig.Webhooks[1].ClientConfig.CABundle) == 0 { - t.Errorf("Unexpected tls.crt embedded in MutatingWebhookConfiguration: %v", mConfig.Webhooks) - } - } - }) - } -} - -func buildFakeClient(kubeResources []client.Object) client.Client { - fakeClientBuilder := fake.NewClientBuilder(). - WithScheme(scheme.Scheme). - WithInterceptorFuncs(interceptor.Funcs{Patch: ssaAsStrategicMergePatchFunc}) - if len(kubeResources) > 0 { - fakeClientBuilder.WithObjects(kubeResources...) - } - return fakeClientBuilder.Build() -} - -type ssaPatchAsStrategicMerge struct { - client.Patch -} - -func (*ssaPatchAsStrategicMerge) Type() types.PatchType { - return types.StrategicMergePatchType -} - -func wrapSSAPatch(patch client.Patch) client.Patch { - if patch.Type() == types.ApplyPatchType { - return &ssaPatchAsStrategicMerge{Patch: patch} - } - return patch -} - -// ssaAsStrategicMergePatchFunc returns the patch interceptor. -// TODO (tenzen-y): Once the fake client supports server-side apply, we should remove these interceptor. -// REF: https://github.com/kubernetes/kubernetes/issues/115598 -func ssaAsStrategicMergePatchFunc( - ctx context.Context, - cli client.WithWatch, - obj client.Object, - patch client.Patch, - opts ...client.PatchOption, -) error { - return cli.Patch(ctx, obj, wrapSSAPatch(patch), opts...) -} - -func TestEnsureCertMounted(t *testing.T) { - tests := map[string]struct { - keyExist bool - certExist bool - wantExist bool - }{ - "key and cert exist": { - keyExist: true, - certExist: true, - wantExist: true, - }, - "key doesn't exist": { - keyExist: false, - certExist: true, - wantExist: false, - }, - "cert doesn't exist": { - keyExist: true, - certExist: false, - wantExist: false, - }, - "all files doesn't exist": { - keyExist: false, - certExist: false, - wantExist: false, - }, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - if tc.keyExist || tc.certExist { - if err := os.MkdirAll(consts.CertDir, 0760); err != nil { - t.Fatalf("Failed to set up directory: %v", err) - } - defer func() { - if err := os.RemoveAll(consts.CertDir); err != nil { - t.Fatalf("Failed to clean up directory: %v", err) - } - }() - } - if tc.keyExist { - if _, err := os.Create(filepath.Join(consts.CertDir, serverKeyName)); err != nil { - t.Fatalf("Failed to create tls.key: %v", err) - } - } - if tc.certExist { - if _, err := os.Create(filepath.Join(consts.CertDir, serverCertName)); err != nil { - t.Fatalf("Failed to create tls.crt: %v", err) - } - } - ensureFunc := ensureCertMounted(time.Now()) - got, _ := ensureFunc(context.Background()) - if tc.wantExist != got { - t.Errorf("Unexpected value from ensureCertMounted: \n(want: %v, got: %v)\n", tc.wantExist, got) - } - }) - } -}