diff --git a/test/e2e/kind-config-v1.18.yaml b/test/e2e/kind-config-v1.18.yaml new file mode 100644 index 00000000000..1e23fe5e648 --- /dev/null +++ b/test/e2e/kind-config-v1.18.yaml @@ -0,0 +1,30 @@ +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane +kubeadmConfigPatches: +- | + kind: ClusterConfiguration + metadata: + name: config + apiServer: + extraArgs: + "service-account-signing-key-file": /etc/kubernetes/pki/sa.key + "service-account-key-file": /etc/kubernetes/pki/sa.pub + "service-account-issuer": api + "service-account-api-audiences": api,vault,factors + diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index ce4cb2f099e..5d02e89037f 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -27,10 +27,16 @@ install_kind # export KIND_CLUSTER= # create_cluster function create_cluster { + KIND_VERSION=$1 : ${KIND_CLUSTER:?"KIND_CLUSTER must be set"} : ${1:?"k8s version must be set as arg 1"} if ! kind get clusters | grep -q $KIND_CLUSTER ; then - kind create cluster -v 4 --name $KIND_CLUSTER --retain --wait=1m --config $(dirname "$0")/kind-config.yaml --image=kindest/node:$1 + version_prefix="${KIND_VERSION%.*}" + kind_config=$(dirname "$0")/kind-config.yaml + if test -f $(dirname "$0")/kind-config-${version_prefix}.yaml; then + kind_config=$(dirname "$0")/kind-config-${version_prefix}.yaml + fi + kind create cluster -v 4 --name $KIND_CLUSTER --retain --wait=1m --config ${kind_config} --image=kindest/node:$1 fi } diff --git a/test/e2e/v3/plugin_cluster_test.go b/test/e2e/v3/plugin_cluster_test.go index 9a1ac102f48..c386354fbfe 100644 --- a/test/e2e/v3/plugin_cluster_test.go +++ b/test/e2e/v3/plugin_cluster_test.go @@ -17,7 +17,7 @@ limitations under the License. package v3 import ( - "encoding/base64" + "encoding/json" "fmt" "os" "path/filepath" @@ -38,11 +38,22 @@ import ( "sigs.k8s.io/kubebuilder/v3/test/e2e/utils" ) +const ( + tokenRequestRawString = `{"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest"}` +) + +// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type +// that we want to use for extracting the token. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} + var _ = Describe("kubebuilder", func() { Context("project version 3", func() { var ( kbc *utils.TestContext - sat bool ) BeforeEach(func() { @@ -53,8 +64,6 @@ var _ = Describe("kubebuilder", func() { By("installing the Prometheus operator") Expect(kbc.InstallPrometheusOperManager()).To(Succeed()) - - sat = false }) AfterEach(func() { @@ -94,7 +103,7 @@ var _ = Describe("kubebuilder", func() { kbc.Kubectl.ServiceAccount = "default" defer func() { kbc.Kubectl.ServiceAccount = tmp }() GenerateV2(kbc) - Run(kbc, sat) + Run(kbc) }) }) @@ -119,15 +128,7 @@ var _ = Describe("kubebuilder", func() { } GenerateV3(kbc, "v1") - - // only if running on Kubernetes >= 1.24 do we need to generate the ServiceAccount token Secret - // TODO: Remove this once a better implementation using something like the TokenRequest API - // is used in the e2e tests - if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() == 1 && srvVer.GetMinorInt() >= 24 { - sat = true - } - - Run(kbc, sat) + Run(kbc) }) It("should generate a runnable project with the golang base plugin v3 and kustomize v4-alpha", func() { // Skip if cluster version < 1.16, when v1 CRDs and webhooks did not exist. @@ -139,15 +140,7 @@ var _ = Describe("kubebuilder", func() { } GenerateV3WithKustomizeV2(kbc, "v1") - - // only if running on Kubernetes >= 1.24 do we need to generate the ServiceAccount token Secret - // TODO: Remove this once a better implementation using something like the TokenRequest API - // is used in the e2e tests - if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() == 1 && srvVer.GetMinorInt() >= 24 { - sat = true - } - - Run(kbc, sat) + Run(kbc) }) It("should generate a runnable project with v1beta1 CRDs and Webhooks", func() { // Skip if cluster version < 1.15, when `.spec.preserveUnknownFields` was not a v1beta1 CRD field. @@ -161,14 +154,14 @@ var _ = Describe("kubebuilder", func() { } GenerateV3(kbc, "v1beta1") - Run(kbc, sat) + Run(kbc) }) }) }) }) // Run runs a set of e2e tests for a scaffolded project defined by a TestContext. -func Run(kbc *utils.TestContext, sat bool) { +func Run(kbc *utils.TestContext) { var controllerPodName string var err error @@ -233,10 +226,6 @@ func Run(kbc *utils.TestContext, sat bool) { fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount)) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - if sat { - ServiceAccountToken(kbc) - } - _ = curlMetrics(kbc) By("validating that cert-manager has provisioned the certificate Secret") @@ -347,21 +336,14 @@ func Run(kbc *utils.TestContext, sat bool) { func curlMetrics(kbc *utils.TestContext) string { By("reading the metrics token") // Filter token query by service account in case more than one exists in a namespace. - // TODO: Parsing a token this way is not ideal or best practice and should not be used. - // Instead, a TokenRequest should be created to get a token to use for this step. - query := fmt.Sprintf(`{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="%s")].data.token}`, - kbc.Kubectl.ServiceAccount, - ) - b64Token, err := kbc.Kubectl.Get(true, "secrets", "-o=jsonpath="+query) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - token, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Token)) + token, err := ServiceAccountToken(kbc) ExpectWithOffset(2, err).NotTo(HaveOccurred()) ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0)) By("creating a curl pod") cmdOpts := []string{ "run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--", - "curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token), + "curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, strings.TrimSpace(token)), fmt.Sprintf("https://e2e-%s-controller-manager-metrics-service.%s.svc:8443/metrics", kbc.TestSuffix, kbc.Kubectl.Namespace), } @@ -398,43 +380,38 @@ func curlMetrics(kbc *utils.TestContext) string { return metricsOutput } -// TODO: Remove this when and other related functions when -// tests have been changed to use a better implementation. -// ServiceAccountToken creates the ServiceAccount token Secret. -// This is to be used when Kubernetes cluster version is >= 1.24. -// In k8s 1.24+ a ServiceAccount token Secret is no longer -// automatically generated -func ServiceAccountToken(kbc *utils.TestContext) { - // As of Kubernetes 1.24 a ServiceAccount no longer has a ServiceAccount token - // secret autogenerated. We have to create it manually here +// ServiceAccountToken provides a helper function that can provide you with a service account +// token that you can use to interact with the service. Depending on the version of k8s being +// used in the e2e test, it will either switch to using kubectl create token (TokenRequest) +// or use a new custom secret to extract the token required via kubernetes.io/service-account-token +// type. This returns the JWT token and an error message. +func ServiceAccountToken(kbc *utils.TestContext) (out string, err error) { By("Creating the ServiceAccount token") - secretFile, err := createServiceAccountToken(kbc.Kubectl.ServiceAccount, kbc.Dir) - Expect(err).NotTo(HaveOccurred()) - Eventually(func() error { - _, err = kbc.Kubectl.Apply(true, "-f", secretFile) - return err - }, time.Minute, time.Second).Should(Succeed()) -} - -var saSecretTemplate = `--- -apiVersion: v1 -kind: Secret -type: kubernetes.io/service-account-token -metadata: - name: %s - annotations: - kubernetes.io/service-account.name: "%s" -` - -// createServiceAccountToken writes a service account token secret to a file. -// It returns a string to the file or an error if it fails to write the file -func createServiceAccountToken(name string, dir string) (string, error) { - secretName := name + "-secret" - fileName := dir + "/" + secretName + ".yaml" - err := os.WriteFile(fileName, []byte(fmt.Sprintf(saSecretTemplate, secretName, name)), 0777) + secretName := fmt.Sprintf("%s-token-request", kbc.Kubectl.ServiceAccount) + tokenRequestFile := filepath.Join(kbc.Dir, secretName) + err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0755)) if err != nil { - return "", err + return out, err } + var rawJson string + Eventually(func() error { + // Output of this is already a valid JWT token. No need to covert this from base64 to string format + rawJson, err = kbc.Kubectl.Command( + "create", + "--raw", fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s/token", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount), + "-f", tokenRequestFile, + ) + if err != nil { + return err + } + var token tokenRequest + err = json.Unmarshal([]byte(rawJson), &token) + if err != nil { + return err + } + out = token.Status.Token + return nil + }, time.Minute, time.Second).Should(Succeed()) - return fileName, nil + return out, err }