Skip to content

Commit

Permalink
GIT-2731: enable TokenRequest for ServiceAccount token generation
Browse files Browse the repository at this point in the history 
GIT-2731: enable usage of raw TokenRequest workflow
  • Loading branch information
@harshanarayana
harshanarayana committed Jun 16, 2022
1 parent 0dbbf57 commit 6965ce4
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 74 deletions.
30 changes: 30 additions & 0 deletions test/e2e/kind-config-v1.18.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
kubeadmConfigPatches:
- |
kind: ClusterConfiguration
metadata:
name: config
apiServer:
extraArgs:
"service-account-signing-key-file": /etc/kubernetes/pki/sa.key
"service-account-key-file": /etc/kubernetes/pki/sa.pub
"service-account-issuer": api
"service-account-api-audiences": api,vault,factors
8 changes: 7 additions & 1 deletion test/e2e/setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,16 @@ install_kind
# export KIND_CLUSTER=<kind cluster name>
# create_cluster <k8s version>
function create_cluster {
KIND_VERSION=$1
: ${KIND_CLUSTER:?"KIND_CLUSTER must be set"}
: ${1:?"k8s version must be set as arg 1"}
if ! kind get clusters | grep -q $KIND_CLUSTER ; then
kind create cluster -v 4 --name $KIND_CLUSTER --retain --wait=1m --config $(dirname "$0")/kind-config.yaml --image=kindest/node:$1
version_prefix="${KIND_VERSION%.*}"
kind_config=$(dirname "$0")/kind-config.yaml
if test -f $(dirname "$0")/kind-config-${version_prefix}.yaml; then
kind_config=$(dirname "$0")/kind-config-${version_prefix}.yaml
fi
kind create cluster -v 4 --name $KIND_CLUSTER --retain --wait=1m --config ${kind_config} --image=kindest/node:$1
fi
}

Expand Down
123 changes: 50 additions & 73 deletions test/e2e/v3/plugin_cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ limitations under the License.
package v3

import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
"path/filepath"
Expand All @@ -38,11 +38,22 @@ import (
"sigs.k8s.io/kubebuilder/v3/test/e2e/utils"
)

const (
tokenRequestRawString = `{"apiVersion": "authentication.k8s.io/v1", "kind": "TokenRequest"}`
)

// tokenRequest is a trimmed down version of the authentication.k8s.io/v1/TokenRequest Type
// that we want to use for extracting the token.
type tokenRequest struct {
Status struct {
Token string `json:"token"`
} `json:"status"`
}

var _ = Describe("kubebuilder", func() {
Context("project version 3", func() {
var (
kbc *utils.TestContext
sat bool
)

BeforeEach(func() {
Expand All @@ -53,8 +64,6 @@ var _ = Describe("kubebuilder", func() {

By("installing the Prometheus operator")
Expect(kbc.InstallPrometheusOperManager()).To(Succeed())

sat = false
})

AfterEach(func() {
Expand Down Expand Up @@ -94,7 +103,7 @@ var _ = Describe("kubebuilder", func() {
kbc.Kubectl.ServiceAccount = "default"
defer func() { kbc.Kubectl.ServiceAccount = tmp }()
GenerateV2(kbc)
Run(kbc, sat)
Run(kbc)
})
})

Expand All @@ -119,15 +128,7 @@ var _ = Describe("kubebuilder", func() {
}

GenerateV3(kbc, "v1")

// only if running on Kubernetes >= 1.24 do we need to generate the ServiceAccount token Secret
// TODO: Remove this once a better implementation using something like the TokenRequest API
// is used in the e2e tests
if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() == 1 && srvVer.GetMinorInt() >= 24 {
sat = true
}

Run(kbc, sat)
Run(kbc)
})
It("should generate a runnable project with the golang base plugin v3 and kustomize v4-alpha", func() {
// Skip if cluster version < 1.16, when v1 CRDs and webhooks did not exist.
Expand All @@ -139,15 +140,7 @@ var _ = Describe("kubebuilder", func() {
}

GenerateV3WithKustomizeV2(kbc, "v1")

// only if running on Kubernetes >= 1.24 do we need to generate the ServiceAccount token Secret
// TODO: Remove this once a better implementation using something like the TokenRequest API
// is used in the e2e tests
if srvVer := kbc.K8sVersion.ServerVersion; srvVer.GetMajorInt() == 1 && srvVer.GetMinorInt() >= 24 {
sat = true
}

Run(kbc, sat)
Run(kbc)
})
It("should generate a runnable project with v1beta1 CRDs and Webhooks", func() {
// Skip if cluster version < 1.15, when `.spec.preserveUnknownFields` was not a v1beta1 CRD field.
Expand All @@ -161,14 +154,14 @@ var _ = Describe("kubebuilder", func() {
}

GenerateV3(kbc, "v1beta1")
Run(kbc, sat)
Run(kbc)
})
})
})
})

// Run runs a set of e2e tests for a scaffolded project defined by a TestContext.
func Run(kbc *utils.TestContext, sat bool) {
func Run(kbc *utils.TestContext) {
var controllerPodName string
var err error

Expand Down Expand Up @@ -233,10 +226,6 @@ func Run(kbc *utils.TestContext, sat bool) {
fmt.Sprintf("--serviceaccount=%s:%s", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount))
ExpectWithOffset(1, err).NotTo(HaveOccurred())

if sat {
ServiceAccountToken(kbc)
}

_ = curlMetrics(kbc)

By("validating that cert-manager has provisioned the certificate Secret")
Expand Down Expand Up @@ -347,21 +336,14 @@ func Run(kbc *utils.TestContext, sat bool) {
func curlMetrics(kbc *utils.TestContext) string {
By("reading the metrics token")
// Filter token query by service account in case more than one exists in a namespace.
// TODO: Parsing a token this way is not ideal or best practice and should not be used.
// Instead, a TokenRequest should be created to get a token to use for this step.
query := fmt.Sprintf(`{.items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="%s")].data.token}`,
kbc.Kubectl.ServiceAccount,
)
b64Token, err := kbc.Kubectl.Get(true, "secrets", "-o=jsonpath="+query)
ExpectWithOffset(2, err).NotTo(HaveOccurred())
token, err := base64.StdEncoding.DecodeString(strings.TrimSpace(b64Token))
token, err := ServiceAccountToken(kbc)
ExpectWithOffset(2, err).NotTo(HaveOccurred())
ExpectWithOffset(2, len(token)).To(BeNumerically(">", 0))

By("creating a curl pod")
cmdOpts := []string{
"run", "curl", "--image=curlimages/curl:7.68.0", "--restart=OnFailure", "--",
"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, token),
"curl", "-v", "-k", "-H", fmt.Sprintf(`Authorization: Bearer %s`, strings.TrimSpace(token)),
fmt.Sprintf("https://e2e-%s-controller-manager-metrics-service.%s.svc:8443/metrics",
kbc.TestSuffix, kbc.Kubectl.Namespace),
}
Expand Down Expand Up @@ -398,43 +380,38 @@ func curlMetrics(kbc *utils.TestContext) string {
return metricsOutput
}

// TODO: Remove this when and other related functions when
// tests have been changed to use a better implementation.
// ServiceAccountToken creates the ServiceAccount token Secret.
// This is to be used when Kubernetes cluster version is >= 1.24.
// In k8s 1.24+ a ServiceAccount token Secret is no longer
// automatically generated
func ServiceAccountToken(kbc *utils.TestContext) {
// As of Kubernetes 1.24 a ServiceAccount no longer has a ServiceAccount token
// secret autogenerated. We have to create it manually here
// ServiceAccountToken provides a helper function that can provide you with a service account
// token that you can use to interact with the service. Depending on the version of k8s being
// used in the e2e test, it will either switch to using kubectl create token (TokenRequest)
// or use a new custom secret to extract the token required via kubernetes.io/service-account-token
// type. This returns the JWT token and an error message.
func ServiceAccountToken(kbc *utils.TestContext) (out string, err error) {
By("Creating the ServiceAccount token")
secretFile, err := createServiceAccountToken(kbc.Kubectl.ServiceAccount, kbc.Dir)
Expect(err).NotTo(HaveOccurred())
Eventually(func() error {
_, err = kbc.Kubectl.Apply(true, "-f", secretFile)
return err
}, time.Minute, time.Second).Should(Succeed())
}

var saSecretTemplate = `---
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
name: %s
annotations:
kubernetes.io/service-account.name: "%s"
`

// createServiceAccountToken writes a service account token secret to a file.
// It returns a string to the file or an error if it fails to write the file
func createServiceAccountToken(name string, dir string) (string, error) {
secretName := name + "-secret"
fileName := dir + "/" + secretName + ".yaml"
err := os.WriteFile(fileName, []byte(fmt.Sprintf(saSecretTemplate, secretName, name)), 0777)
secretName := fmt.Sprintf("%s-token-request", kbc.Kubectl.ServiceAccount)
tokenRequestFile := filepath.Join(kbc.Dir, secretName)
err = os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0755))
if err != nil {
return "", err
return out, err
}
var rawJson string
Eventually(func() error {
// Output of this is already a valid JWT token. No need to covert this from base64 to string format
rawJson, err = kbc.Kubectl.Command(
"create",
"--raw", fmt.Sprintf("/api/v1/namespaces/%s/serviceaccounts/%s/token", kbc.Kubectl.Namespace, kbc.Kubectl.ServiceAccount),
"-f", tokenRequestFile,
)
if err != nil {
return err
}
var token tokenRequest
err = json.Unmarshal([]byte(rawJson), &token)
if err != nil {
return err
}
out = token.Status.Token
return nil
}, time.Minute, time.Second).Should(Succeed())

return fileName, nil
return out, err
}

0 comments on commit 6965ce4

Please sign in to comment.