diff --git a/test/e2e/utils/test_context.go b/test/e2e/utils/test_context.go index 012a4e5af3d..60ae5c4bbaa 100644 --- a/test/e2e/utils/test_context.go +++ b/test/e2e/utils/test_context.go @@ -270,14 +270,21 @@ func (t *TestContext) CreateManagerNamespace() error { return err } -// LabelAllNamespacesToWarnAboutRestricted will label all namespaces so that we can verify +// LabelNamespacesToWarnAboutRestricted will label all namespaces so that we can verify // if a warning with `Warning: would violate PodSecurity` will be raised when the manifests are applied -func (t *TestContext) LabelAllNamespacesToWarnAboutRestricted() error { - _, err := t.Kubectl.Command("label", "--overwrite", "ns", "--all", +func (t *TestContext) LabelNamespacesToWarnAboutRestricted() error { + _, err := t.Kubectl.Command("label", "--overwrite", "ns", t.Kubectl.Namespace, "pod-security.kubernetes.io/warn=restricted") return err } +// RemoveNamespaceLabelToWarnAboutRestricted will remove the `pod-security.kubernetes.io/warn` label +// from the specified namespace +func (t *TestContext) RemoveNamespaceLabelToWarnAboutRestricted() error { + _, err := t.Kubectl.Command("label", "ns", t.Kubectl.Namespace, "pod-security.kubernetes.io/warn-") + return err +} + // LoadImageToKindCluster loads a local docker image to the kind cluster func (t *TestContext) LoadImageToKindCluster() error { cluster := "kind" diff --git a/test/e2e/v4/plugin_cluster_test.go b/test/e2e/v4/plugin_cluster_test.go index 4a2d1a109c4..0cc32c5a934 100644 --- a/test/e2e/v4/plugin_cluster_test.go +++ b/test/e2e/v4/plugin_cluster_test.go @@ -25,12 +25,10 @@ import ( "strings" "time" - "sigs.k8s.io/kubebuilder/v4/pkg/plugin/util" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + "sigs.k8s.io/kubebuilder/v4/pkg/plugin/util" "sigs.k8s.io/kubebuilder/v4/test/e2e/utils" ) @@ -43,9 +41,38 @@ var _ = Describe("kubebuilder", func() { kbc, err = utils.NewTestContext(util.KubebuilderBinName, "GO111MODULE=on") Expect(err).NotTo(HaveOccurred()) Expect(kbc.Prepare()).To(Succeed()) + + By("creating manager namespace") + err = kbc.CreateManagerNamespace() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + if kbc.IsRestricted { + By("labeling all namespaces to warn about restricted") + err = kbc.LabelNamespacesToWarnAboutRestricted() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("updating the go.mod") + err = kbc.Tidy() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("run make all") + err = kbc.Make("all") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("building the controller image") + err = kbc.Make("docker-build", "IMG="+kbc.ImageName) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the controller docker image into the kind cluster") + err = kbc.LoadImageToKindCluster() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) AfterEach(func() { + By("By removing restricted namespace label") + _ = kbc.RemoveNamespaceLabelToWarnAboutRestricted() + By("clean up API objects created during the test") kbc.CleanupManifests(filepath.Join("config", "default")) @@ -80,42 +107,16 @@ var _ = Describe("kubebuilder", func() { func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool) { var controllerPodName string var err error - - By("creating manager namespace") - err = kbc.CreateManagerNamespace() - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("labeling all namespaces to warn about restricted") - err = kbc.LabelAllNamespacesToWarnAboutRestricted() - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("updating the go.mod") - err = kbc.Tidy() - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("run make manifests") - err = kbc.Make("manifests") - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("run make generate") - err = kbc.Make("generate") - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("building the controller image") - err = kbc.Make("docker-build", "IMG="+kbc.ImageName) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the controller docker image into the kind cluster") - err = kbc.LoadImageToKindCluster() - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - var output []byte + if !isToUseInstaller { By("deploying the controller-manager") cmd := exec.Command("make", "deploy", "IMG="+kbc.ImageName) output, err = kbc.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - } else { + } + + if isToUseInstaller { By("building the installer") err = kbc.Make("build-installer", "IMG="+kbc.ImageName) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -130,38 +131,8 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool) ExpectWithOffset(1, output).NotTo(ContainSubstring("Warning: would violate PodSecurity")) } - By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { - // Get pod name - podOutput, err := kbc.Kubectl.Get( - true, - "pods", "-l", "control-plane=controller-manager", - "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}") - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - podNames := util.GetNonEmptyLines(podOutput) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) - } - controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) - - // Validate pod status - status, err := kbc.Kubectl.Get( - true, - "pods", controllerPodName, "-o", "jsonpath={.status.phase}") - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - if status != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil - } - defer func() { - out, err := kbc.Kubectl.CommandInNamespace("describe", "all") - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - fmt.Fprintln(GinkgoWriter, out) - }() - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + By("Checking controllerManager and getting the name of the Pod") + controllerPodName = getControllerName(kbc) By("Checking if all flags are applied to the manager pod") podOutput, err := kbc.Kubectl.Get( @@ -175,9 +146,6 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool) ExpectWithOffset(1, podOutput).To(ContainSubstring("health-probe-bind-address"), "Expected manager pod to have --health-probe-bind-address flag") - By("validating the metrics endpoint") - _ = curlMetrics(kbc, hasMetrics) - if hasWebhook { By("validating that cert-manager has provisioned the certificate Secret") EventuallyWithOffset(1, func() error { @@ -233,18 +201,15 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool) // we can change it to probe the readiness endpoint after CR supports it. sampleFile := filepath.Join("config", "samples", fmt.Sprintf("%s_%s_%s.yaml", kbc.Group, kbc.Version, strings.ToLower(kbc.Kind))) - sampleFilePath, err := filepath.Abs(filepath.Join(fmt.Sprintf("e2e-%s", kbc.TestSuffix), sampleFile)) Expect(err).To(Not(HaveOccurred())) f, err := os.OpenFile(sampleFilePath, os.O_APPEND|os.O_WRONLY, 0o644) Expect(err).To(Not(HaveOccurred())) - defer func() { err = f.Close() Expect(err).To(Not(HaveOccurred())) }() - _, err = f.WriteString(" foo: bar") Expect(err).To(Not(HaveOccurred())) @@ -255,13 +220,18 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool) if hasMetrics { By("checking the metrics values to validate that the created resource object gets reconciled") - metricsOutput := curlMetrics(kbc, hasMetrics) + metricsOutput := getMetricsOutput(kbc) ExpectWithOffset(1, metricsOutput).To(ContainSubstring(fmt.Sprintf( `controller_runtime_reconcile_total{controller="%s",result="success"} 1`, strings.ToLower(kbc.Kind), ))) } + if !hasMetrics { + By("validating the metrics endpoint is not working as expected") + metricsShouldBeUnavailable(kbc) + } + if hasWebhook { By("validating that mutating and validating webhooks are working fine") cnt, err := kbc.Kubectl.Get( @@ -273,112 +243,148 @@ func Run(kbc *utils.TestContext, hasWebhook, isToUseInstaller, hasMetrics bool) ExpectWithOffset(1, err).NotTo(HaveOccurred()) ExpectWithOffset(1, count).To(BeNumerically("==", 5)) } +} + +func getControllerName(kbc *utils.TestContext) string { + By("validating that the controller-manager pod is running as expected") + var controllerPodName string + verifyControllerUp := func() error { + // Get pod name + podOutput, err := kbc.Kubectl.Get( + true, + "pods", "-l", "control-plane=controller-manager", + "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}") + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + podNames := util.GetNonEmptyLines(podOutput) + if len(podNames) != 1 { + return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) + } + controllerPodName = podNames[0] + ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + // Validate pod status + status, err := kbc.Kubectl.Get( + true, + "pods", controllerPodName, "-o", "jsonpath={.status.phase}") + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + if status != "Running" { + return fmt.Errorf("controller pod in %s status", status) + } + return nil + } + defer func() { + out, err := kbc.Kubectl.CommandInNamespace("describe", "all") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + fmt.Fprintln(GinkgoWriter, out) + }() + EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + return controllerPodName } -// curlMetrics curl's the /metrics endpoint, returning all logs once a 200 status is returned. -func curlMetrics(kbc *utils.TestContext, hasMetrics bool) string { +// getMetricsOutput return the metrics output from curl pod +func getMetricsOutput(kbc *utils.TestContext) string { var metricsOutput string - if hasMetrics { - By("validating that the controller-manager service is available") - _, err := kbc.Kubectl.Get( + By("validating that the controller-manager service is available") + _, err := kbc.Kubectl.Get( + true, + "service", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), + ) + ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Controller-manager service should exist") + + By("ensuring the service endpoint is ready") + eventuallyCheckServiceEndpoint := func() error { + output, err := kbc.Kubectl.Get( true, - "service", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), + "endpoints", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), + "-o", "jsonpath={.subsets[*].addresses[*].ip}", ) - ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Controller-manager service should exist") - - By("ensuring the service endpoint is ready") - eventuallyCheckServiceEndpoint := func() error { - output, err := kbc.Kubectl.Get( - true, - "endpoints", fmt.Sprintf("e2e-%s-controller-manager-metrics-service", kbc.TestSuffix), - "-o", "jsonpath={.subsets[*].addresses[*].ip}", - ) - if err != nil { - return err - } - if output == "" { - return fmt.Errorf("no endpoints found") - } - return nil + if err != nil { + return err } - EventuallyWithOffset(2, eventuallyCheckServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(), - "Service endpoint should be ready") - - By("creating a curl pod to access the metrics endpoint") - // nolint:lll - cmdOpts := []string{ - "run", "curl", - "--restart=Never", - "--namespace", kbc.Kubectl.Namespace, - "--image=curlimages/curl:7.78.0", - "--", - "/bin/sh", "-c", fmt.Sprintf("curl -v -k http://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8080/metrics", - kbc.TestSuffix, kbc.Kubectl.Namespace), + if output == "" { + return fmt.Errorf("no endpoints found") } - _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) + return nil + } + EventuallyWithOffset(2, eventuallyCheckServiceEndpoint, 2*time.Minute, time.Second).Should(Succeed(), + "Service endpoint should be ready") - By("validating that the curl pod is running as expected") - verifyCurlUp := func() error { - status, err := kbc.Kubectl.Get( - true, - "pods", "curl", "-o", "jsonpath={.status.phase}") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) - if status != "Succeeded" { - return fmt.Errorf("curl pod in %s status", status) - } - return nil - } - EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + By("creating a curl pod to access the metrics endpoint") + cmdOpts := cmdOptsToCreateCurlPod(kbc) + _, err = kbc.Kubectl.CommandInNamespace(cmdOpts...) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) - By("validating that the metrics endpoint is serving as expected") - getCurlLogs := func() string { - metricsOutput, err = kbc.Kubectl.Logs("curl") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) - return metricsOutput - } - EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/1.1 200 OK")) - } else { - By("creating a curl pod to access the metrics endpoint") - // nolint:lll - cmdOpts := []string{ - "run", "curl", - "--restart=Never", - "--namespace", kbc.Kubectl.Namespace, - "--image=curlimages/curl:7.78.0", - "--", - "/bin/sh", "-c", fmt.Sprintf("curl -v -k http://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8080/metrics", - kbc.TestSuffix, kbc.Kubectl.Namespace), + By("validating that the curl pod is running as expected") + verifyCurlUp := func() error { + status, err := kbc.Kubectl.Get( + true, + "pods", "curl", "-o", "jsonpath={.status.phase}") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if status != "Succeeded" { + return fmt.Errorf("curl pod in %s status", status) } - _, err := kbc.Kubectl.CommandInNamespace(cmdOpts...) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) + return nil + } + EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) - By("validating that the curl pod fail as expected") - verifyCurlUp := func() error { - status, err := kbc.Kubectl.Get( - true, - "pods", "curl", "-o", "jsonpath={.status.phase}") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) - if status != "Failed" { - return fmt.Errorf( - "curl pod in %s status when should fail with an error", status) - } - return nil - } - EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + By("validating that the metrics endpoint is serving as expected") + getCurlLogs := func() string { + metricsOutput, err = kbc.Kubectl.Logs("curl") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + return metricsOutput + } + EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("< HTTP/1.1 200 OK")) + removeCurlPod(kbc) + return metricsOutput +} - By("validating that the metrics endpoint is not working as expected") - getCurlLogs := func() string { - metricsOutput, err := kbc.Kubectl.Logs("curl") - ExpectWithOffset(3, err).NotTo(HaveOccurred()) - return metricsOutput +func metricsShouldBeUnavailable(kbc *utils.TestContext) { + By("creating a curl pod to access the metrics endpoint") + cmdOpts := cmdOptsToCreateCurlPod(kbc) + _, err := kbc.Kubectl.CommandInNamespace(cmdOpts...) + ExpectWithOffset(2, err).NotTo(HaveOccurred()) + + By("validating that the curl pod fail as expected") + verifyCurlUp := func() error { + status, err := kbc.Kubectl.Get( + true, + "pods", "curl", "-o", "jsonpath={.status.phase}") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + if status != "Failed" { + return fmt.Errorf( + "curl pod in %s status when should fail with an error", status) } - EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("Could not resolve host")) + return nil } + EventuallyWithOffset(2, verifyCurlUp, 240*time.Second, time.Second).Should(Succeed()) + + By("validating that the metrics endpoint is not working as expected") + getCurlLogs := func() string { + metricsOutput, err := kbc.Kubectl.Logs("curl") + ExpectWithOffset(3, err).NotTo(HaveOccurred()) + return metricsOutput + } + EventuallyWithOffset(2, getCurlLogs, 10*time.Second, time.Second).Should(ContainSubstring("Could not resolve host")) + removeCurlPod(kbc) +} + +func cmdOptsToCreateCurlPod(kbc *utils.TestContext) []string { + // nolint:lll + cmdOpts := []string{ + "run", "curl", + "--restart=Never", + "--namespace", kbc.Kubectl.Namespace, + "--image=curlimages/curl:7.78.0", + "--", + "/bin/sh", "-c", fmt.Sprintf("curl -v -k http://e2e-%s-controller-manager-metrics-service.%s.svc.cluster.local:8080/metrics", + kbc.TestSuffix, kbc.Kubectl.Namespace), + } + return cmdOpts +} + +func removeCurlPod(kbc *utils.TestContext) { By("cleaning up the curl pod") _, err := kbc.Kubectl.Delete(true, "pods/curl") ExpectWithOffset(3, err).NotTo(HaveOccurred()) - - return metricsOutput }