Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🌱 GIT-2731: enable TokenRequest for ServiceAccount token generation #2733

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions test/e2e/kind-config-v1.18.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
metadata:
name: config
apiServer:
extraArgs:
"service-account-signing-key-file": /etc/kubernetes/pki/sa.key
"service-account-key-file": /etc/kubernetes/pki/sa.pub
"service-account-issuer": api
"service-account-api-audiences": api,vault,factors
1 change: 1 addition & 0 deletions test/e2e/kind-config-v1.19.yaml
8 changes: 7 additions & 1 deletion test/e2e/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,16 @@ install_kind
# export KIND_CLUSTER=<kind cluster name>
# create_cluster <k8s version>
function create_cluster {
KIND_VERSION=$1
: ${KIND_CLUSTER:?"KIND_CLUSTER must be set"}
: ${1:?"k8s version must be set as arg 1"}
if ! kind get clusters | grep -q $KIND_CLUSTER ; then
kind create cluster -v 4 --name $KIND_CLUSTER --retain --wait=1m --config $(dirname "$0")/kind-config.yaml --image=kindest/node:$1
version_prefix="${KIND_VERSION%.*}"
kind_config=$(dirname "$0")/kind-config.yaml
if test -f $(dirname "$0")/kind-config-${version_prefix}.yaml; then
kind_config=$(dirname "$0")/kind-config-${version_prefix}.yaml
fi
kind create cluster -v 4 --name $KIND_CLUSTER --retain --wait=1m --config ${kind_config} --image=kindest/node:$1
fi
}

Expand Down
127 changes: 54 additions & 73 deletions test/e2e/v3/plugin_cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ limitations under the License.
package v3
harshanarayana marked this conversation as resolved.
Show resolved Hide resolved

import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
"path/filepath"
Expand All @@ -38,11 +38,22 @@ import (
"sigs.k8s.io/kubebuilder/v3/test/e2e/utils"
)

const (
tokenRequestRawString = `{"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest"}`
)

// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type
// that we want to use for extracting the token.
type tokenRequest struct {
Status struct {
Token string `json:"token"`
} `json:"status"`
}

var _ = Describe("kubebuilder", func() {
Context("project version 3", func() {
var (
kbc *utils.TestContext
sat bool
)

BeforeEach(func() {
Expand All @@ -53,8 +64,6 @@ var _ = Describe("kubebuilder", func() {

By("installing the Prometheus operator")
Expect(kbc.InstallPrometheusOperManager()).To(Succeed())

sat = false
})

AfterEach(func() {
Expand Down Expand Up @@ -94,7 +103,7 @@ var _ = Describe("kubebuilder", func() {
kbc.Kubectl.ServiceAccount = "default"
defer func() { kbc.Kubectl.ServiceAccount = tmp }()
GenerateV2(kbc)
Run(kbc, sat)
Run(kbc)
})
})

Expand All @@ -119,15 +128,7 @@ var _ = Describe("kubebuilder", func() {
}

GenerateV3(kbc, "v1")

// only if running on Kubernetes >= 1.24 do we need to generate the ServiceAccount token Secret
// TODO: Remove this once a better implementation using something like the TokenRequest API
// is used in the e2e tests
if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() == 1 && srvVer.GetMinorInt() >= 24 {
sat = true
}

Run(kbc, sat)
Run(kbc)
})
It("should generate a runnable project with the golang base plugin v3 and kustomize v4-alpha", func() {
// Skip if cluster version < 1.16, when v1 CRDs and webhooks did not exist.
Expand All @@ -139,15 +140,7 @@ var _ = Describe("kubebuilder", func() {
}

GenerateV3WithKustomizeV2(kbc, "v1")

// only if running on Kubernetes >= 1.24 do we need to generate the ServiceAccount token Secret
// TODO: Remove this once a better implementation using something like the TokenRequest API
// is used in the e2e tests
if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() == 1 && srvVer.GetMinorInt() >= 24 {
sat = true
}

Run(kbc, sat)
Run(kbc)
})
It("should generate a runnable project with v1beta1 CRDs and Webhooks", func() {
// Skip if cluster version < 1.15, when `.spec.preserveUnknownFields` was not a v1beta1 CRD field.
Expand All @@ -161,14 +154,14 @@ var _ = Describe("kubebuilder", func() {
}

GenerateV3(kbc, "v1beta1")
Run(kbc, sat)
Run(kbc)
})
})
})
})

// Run runs a set of e2e tests for a scaffolded project defined by a TestContext.
func Run(kbc *utils.TestContext, sat bool) {
func Run(kbc *utils.TestContext) {
var controllerPodName string
var err error

Expand Down Expand Up @@ -233,10 +226,6 @@ func Run(kbc *utils.TestContext, sat bool) {
fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount))
ExpectWithOffset(1, err).NotTo(HaveOccurred())

if sat {
ServiceAccountToken(kbc)
}

_ = curlMetrics(kbc)

By("validating that cert-manager has provisioned the certificate Secret")
Expand Down Expand Up @@ -347,21 +336,14 @@ func Run(kbc *utils.TestContext, sat bool) {
func curlMetrics(kbc *utils.TestContext) string {
By("reading the metrics token")
// Filter token query by service account in case more than one exists in a namespace.
// TODO: Parsing a token this way is not ideal or best practice and should not be used.
// Instead, a TokenRequest should be created to get a token to use for this step.
query := fmt.Sprintf(`{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="%s")].data.token}`,
kbc.Kubectl.ServiceAccount,
)
b64Token, err := kbc.Kubectl.Get(true, "secrets", "-o=jsonpath="+query)
ExpectWithOffset(2, err).NotTo(HaveOccurred())
token, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Token))
token, err := ServiceAccountToken(kbc)
ExpectWithOffset(2, err).NotTo(HaveOccurred())
ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0))

By("creating a curl pod")
cmdOpts := []string{
"run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--",
"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token),
"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, strings.TrimSpace(token)),
fmt.Sprintf("https://e2e-%s-controller-manager-metrics-service.%s.svc:8443/metrics",
kbc.TestSuffix, kbc.Kubectl.Namespace),
}
Expand Down Expand Up @@ -398,43 +380,42 @@ func curlMetrics(kbc *utils.TestContext) string {
return metricsOutput
}

// TODO: Remove this when and other related functions when
// tests have been changed to use a better implementation.
// ServiceAccountToken creates the ServiceAccount token Secret.
// This is to be used when Kubernetes cluster version is >= 1.24.
// In k8s 1.24+ a ServiceAccount token Secret is no longer
// automatically generated
func ServiceAccountToken(kbc *utils.TestContext) {
// As of Kubernetes 1.24 a ServiceAccount no longer has a ServiceAccount token
// secret autogenerated. We have to create it manually here
// ServiceAccountToken provides a helper function that can provide you with a service account
// token that you can use to interact with the service. This function leverages the k8s'
// TokenRequest API in raw format in order to make it generic for all version of the k8s that
// is currently being supported in kubebuilder test infra.
// TokenRequest API returns the token in raw JWT format itself. There is no conversion required.
func ServiceAccountToken(kbc *utils.TestContext) (out string, err error) {
By("Creating the ServiceAccount token")
secretFile, err := createServiceAccountToken(kbc.Kubectl.ServiceAccount, kbc.Dir)
Expect(err).NotTo(HaveOccurred())
Eventually(func() error {
_, err = kbc.Kubectl.Apply(true, "-f", secretFile)
return err
}, time.Minute, time.Second).Should(Succeed())
}

var saSecretTemplate = `---
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
name: %s
annotations:
kubernetes.io/service-account.name: "%s"
`

// createServiceAccountToken writes a service account token secret to a file.
// It returns a string to the file or an error if it fails to write the file
func createServiceAccountToken(name string, dir string) (string, error) {
secretName := name + "-secret"
fileName := dir + "/" + secretName + ".yaml"
err := os.WriteFile(fileName, []byte(fmt.Sprintf(saSecretTemplate, secretName, name)), 0777)
secretName := fmt.Sprintf("%s-token-request", kbc.Kubectl.ServiceAccount)
tokenRequestFile := filepath.Join(kbc.Dir, secretName)
err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0755))
if err != nil {
return "", err
return out, err
}
var rawJson string
Eventually(func() error {
// Output of this is already a valid JWT token. No need to covert this from base64 to string format
rawJson, err = kbc.Kubectl.Command(
"create",
"--raw", fmt.Sprintf(
"/api/v1/namespaces/%s/serviceaccounts/%s/token",
kbc.Kubectl.Namespace,
kbc.Kubectl.ServiceAccount,
),
"-f", tokenRequestFile,
)
if err != nil {
return err
}
var token tokenRequest
err = json.Unmarshal([]byte(rawJson), &token)
if err != nil {
return err
}
out = token.Status.Token
return nil
}, time.Minute, time.Second).Should(Succeed())

return fileName, nil
return out, err
}