Skip to content

Commit

Permalink
add e2e test for v1 kubebuilder
Browse files Browse the repository at this point in the history
  • Loading branch information
Mengqi Yu committed Jul 19, 2018
1 parent 6bb9d63 commit 88726be
Show file tree
Hide file tree
Showing 6 changed files with 314 additions and 39 deletions.
2 changes: 1 addition & 1 deletion common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ function setup_envs {
header_text "setting up env vars"

# Setup env vars
export PATH=/tmp/kubebuilder/bin/:$PATH
export PATH=/tmp/kubebuilder/bin:$PATH
export TEST_ASSET_KUBECTL=/tmp/kubebuilder/bin/kubectl
export TEST_ASSET_KUBE_APISERVER=/tmp/kubebuilder/bin/kube-apiserver
export TEST_ASSET_ETCD=/tmp/kubebuilder/bin/etcd
Expand Down
72 changes: 72 additions & 0 deletions test/e2e/common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"log"
"os"
"path/filepath"

"github.com/kubernetes-sigs/kubebuilder/test/e2e/framework"
e2einternal "github.com/kubernetes-sigs/kubebuilder/test/internal/e2e"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

func prepare(workDir string) {
By("create a path under given project dir, as the test work dir")
err := os.MkdirAll(workDir, 0755)
Expect(err).NotTo(HaveOccurred())
}

func cleanupv0(builderTest *e2einternal.KubebuilderTest, workDir string, imageName string) {
By("clean up created API objects during test process")
inputFile := filepath.Join(workDir, "hack", "install.yaml")
deleteOptions := []string{"delete", "-f", inputFile}
builderTest.RunKubectlCommand(framework.GetKubectlArgs(deleteOptions))

By("remove container image created during test")
builderTest.CleanupImage([]string{imageName})

By("remove test work dir")
os.RemoveAll(workDir)
}

func cleanupv1(builderTest *e2einternal.KubebuilderTest, workDir string, imageName string) {
By("clean up created API objects during test process")

kustomizeOptions := []string{"build", filepath.Join("config", "default")}
resources, err := builderTest.RunKustomizeCommand(kustomizeOptions)
if err != nil {
log.Printf("error when runing kustomize build during cleaning up: %v", err)
}

deleteOptions := []string{"delete", "--recursive", "-f", "-"}
_, err = builderTest.RunKubectlCommandWithInput(framework.GetKubectlArgs(deleteOptions), resources)
if err != nil {
log.Printf("error when runing kubectl delete during cleaning up: %v", err)
}

deleteOptions = []string{"delete", "--recursive", "-f", filepath.Join("config", "crds")}
_, err = builderTest.RunKubectlCommand(framework.GetKubectlArgs(deleteOptions))

By("remove container image created during test")
builderTest.CleanupImage([]string{imageName})

By("remove test work dir")
os.RemoveAll(workDir)
}
54 changes: 54 additions & 0 deletions test/e2e/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,18 @@ limitations under the License.
package e2e

import (
"log"
"os"
"os/exec"
"path"
"strings"
"testing"

"github.com/golang/glog"
"github.com/kubernetes-sigs/kubebuilder/test/e2e/framework"
"github.com/kubernetes-sigs/kubebuilder/test/e2e/framework/ginkgowrapper"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/spf13/pflag"
)

Expand All @@ -28,6 +37,51 @@ func init() {
pflag.Parse()
}

var kubebuilderBinDir string

/*
2 steps to do before running the e2e test suite
1) go build a new kubebuilder binary
2) install all required binaries, such as etcd and kube-apiserver
If you have run testv0.sh or testv1.sh before running this e2e suite,
it will build the new kubebuilder.
*/
// TODO: ensure the required binaries are installed when integrate with Prow.
var _ = BeforeSuite(func(done Done) {
// Uncomment the following line to set the image name before runing the e2e test
//os.Setenv("IMG", "gcr.io/<my-project-name>/<iamge-name:tag>")
// If you want to run the test against a GKE cluster, run the following command first
// $ kubectl create clusterrolebinding myname-cluster-admin-binding --clusterrole=cluster-admin --user=myname@mycompany.com
framework.TestContext.BinariesDir = "/tmp/kubebuilder/bin/"
// build a kubebuilder
targets := []string{"kubebuilder", "kubebuilder-gen"}
for _, target := range targets {
buildOptions := []string{
"build", "-o", path.Join(framework.TestContext.BinariesDir, target), path.Join("github.com/kubernetes-sigs/kubebuilder/cmd", target)}
cmd := exec.Command("go", buildOptions...)
cmd.Env = os.Environ()
command := strings.Join(cmd.Args, " ")
log.Printf("running %v", command)
output, err := cmd.CombinedOutput()
log.Printf("output when running:\n%s", output)
Expect(err).NotTo(HaveOccurred())
}

close(done)
}, 60)

var _ = AfterSuite(func() {
os.RemoveAll(kubebuilderBinDir)
})

// RunE2ETests checks configuration parameters (specified through flags) and then runs
// E2E tests using the Ginkgo runner.
func RunE2ETests(t *testing.T) {
RegisterFailHandler(ginkgowrapper.Fail)
glog.Infof("Starting kubebuilder suite")
RunSpecs(t, "Kubebuilder e2e suite")
}

func TestE2E(t *testing.T) {
RunE2ETests(t)
}
43 changes: 6 additions & 37 deletions test/e2e/e2e.go → test/e2e/e2e_v0.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
Expand All @@ -18,42 +18,30 @@ package e2e

import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"

"github.com/golang/glog"
"github.com/kubernetes-sigs/kubebuilder/cmd/kubebuilder/util"
"github.com/kubernetes-sigs/kubebuilder/test/e2e/framework"
"github.com/kubernetes-sigs/kubebuilder/test/e2e/framework/ginkgowrapper"
e2einternal "github.com/kubernetes-sigs/kubebuilder/test/internal/e2e"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

// RunE2ETests checks configuration parameters (specified through flags) and then runs
// E2E tests using the Ginkgo runner.
func RunE2ETests(t *testing.T) {
RegisterFailHandler(ginkgowrapper.Fail)
glog.Infof("Starting kubebuilder suite")
RunSpecs(t, "Kubebuilder e2e suite")
}

var _ = Describe("main workflow", func() {
var _ = Describe("v0 main workflow", func() {
It("should perform main kubebuilder workflow successfully", func() {
testSuffix := framework.RandomSuffix()
c := initConfig(testSuffix)
kubebuilderTest := e2einternal.NewKubebuilderTest(c.workDir, framework.TestContext.BinariesDir)

prepare(c.workDir)
defer cleanup(kubebuilderTest, c.workDir, c.controllerImageName)
defer cleanupv0(kubebuilderTest, c.workDir, c.controllerImageName)

var controllerPodName string

By("init project")
initOptions := []string{"--domain", c.domain}
initOptions := []string{"--domain", c.domain, "--project-version", "v0"}
err := kubebuilderTest.Init(initOptions)
Expect(err).NotTo(HaveOccurred())

Expand Down Expand Up @@ -86,7 +74,7 @@ var _ = Describe("main workflow", func() {
Expect(err).NotTo(HaveOccurred())

By("installing controller-manager in cluster")
inputFile := filepath.Join(kubebuilderTest.Dir, "hack", "install.yaml")
inputFile := filepath.Join("hack", "install.yaml")
installOptions := []string{"apply", "-f", inputFile}
_, err = kubebuilderTest.RunKubectlCommand(framework.GetKubectlArgs(installOptions))
Expect(err).NotTo(HaveOccurred())
Expand All @@ -104,7 +92,7 @@ var _ = Describe("main workflow", func() {
return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
}
controllerPodName = podNames[0]
Expect(controllerPodName).Should(HavePrefix(c.installName+"-controller-manager"))
Expect(controllerPodName).Should(HavePrefix(c.installName + "-controller-manager"))

// Validate pod status
getOptions = []string{"get", "pods", controllerPodName, "-n", c.namespace, "-o", "jsonpath={.status.phase}"}
Expand Down Expand Up @@ -150,22 +138,3 @@ var _ = Describe("main workflow", func() {
Eventually(controllerContainerLogs, 1*time.Minute, 500*time.Millisecond).Should(ContainSubstring("to reconcile deployment-example"))
})
})

func prepare(workDir string) {
By("create a path under given project dir, as the test work dir")
err := os.MkdirAll(workDir, 0755)
Expect(err).NotTo(HaveOccurred())
}

func cleanup(builderTest *e2einternal.KubebuilderTest, workDir string, imageName string) {
By("clean up created API objects during test process")
inputFile := filepath.Join(workDir, "hack", "install.yaml")
createOptions := []string{"delete", "-f", inputFile}
builderTest.RunKubectlCommand(framework.GetKubectlArgs(createOptions))

By("remove container image created during test")
builderTest.CleanupImage([]string{imageName})

By("remove test work dir")
os.RemoveAll(workDir)
}
142 changes: 142 additions & 0 deletions test/e2e/e2e_v1.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"fmt"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/kubernetes-sigs/kubebuilder/test/e2e/framework"
e2einternal "github.com/kubernetes-sigs/kubebuilder/test/internal/e2e"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

var _ = Describe("v1 main workflow", func() {
It("should perform main kubebuilder workflow successfully", func() {
testSuffix := framework.RandomSuffix()
c := initConfig(testSuffix)
kubebuilderTest := e2einternal.NewKubebuilderTest(c.workDir, framework.TestContext.BinariesDir)

prepare(c.workDir)
defer cleanupv1(kubebuilderTest, c.workDir, c.controllerImageName)

var controllerPodName string

output, errrr := exec.Command("which", "kubebuilder").Output()
fmt.Printf("output of which: %s", output)
Expect(errrr).NotTo(HaveOccurred())

By("init v1 project")
initOptions := []string{
"--project-version", "v1",
"--domain", c.domain,
"--dep", "true",
}
err := kubebuilderTest.Init(initOptions)
Expect(err).NotTo(HaveOccurred())

By("creating api definition")
crdOptions := []string{
"--group", c.group,
"--version", c.version,
"--kind", c.kind,
"--namespaced", "false",
"--resource", "true",
"--controller", "true",
}
err = kubebuilderTest.CreateAPI(crdOptions)
Expect(err).NotTo(HaveOccurred())

// TODO: enable this test after we support gen rbac for core type controller
By("creating core-type resource controller")
coreControllerOptions := []string{
"--group", "apps",
"--version", "v1",
"--kind", "Deployment",
"--namespaced", "true",
"--resource", "false",
"--controller", "true",
}
err = kubebuilderTest.CreateAPI(coreControllerOptions)
Expect(err).NotTo(HaveOccurred())

By("building image")
makeDockerBuildOptions := []string{"docker-build"}
err = kubebuilderTest.Make(makeDockerBuildOptions)
Expect(err).NotTo(HaveOccurred())

By("pushing image")
makeDockerPushOptions := []string{"docker-push"}
err = kubebuilderTest.Make(makeDockerPushOptions)
Expect(err).NotTo(HaveOccurred())

// NOTE: If you want to run the test against a GKE cluster, you will need to grant yourself permission.
// Otherwise, you may see "... is forbidden: attempt to grant extra privileges"
// $ kubectl create clusterrolebinding myname-cluster-admin-binding --clusterrole=cluster-admin --user=myname@mycompany.com
// https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control
By("deploying controller manager")
makeDeployOptions := []string{"deploy"}
err = kubebuilderTest.Make(makeDeployOptions)

By("validate the controller-manager pod running as expected")
verifyContollerUp := func() error {
// Get pod name
getOptions := []string{"get", "pods", "-l", "control-plane=controller-manager", "-n", fmt.Sprintf("e2e-%s-system", testSuffix), "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}{{ \"\\n\" }}{{ end }}{{ end }}"}
podOutput, err := kubebuilderTest.RunKubectlCommand(framework.GetKubectlArgs(getOptions))
Expect(err).NotTo(HaveOccurred())
podNames := framework.ParseCmdOutput(podOutput)
if len(podNames) != 1 {
return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
}
controllerPodName = podNames[0]
Expect(controllerPodName).Should(ContainSubstring("controller-manager"))

// Validate pod status
getOptions = []string{"get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", fmt.Sprintf("e2e-%s-system", testSuffix)}
status, err := kubebuilderTest.RunKubectlCommand(framework.GetKubectlArgs(getOptions))
Expect(err).NotTo(HaveOccurred())
if status != "Running" {
return fmt.Errorf("controller pod in %s status", status)
}

return nil
}
Eventually(verifyContollerUp, 5*time.Minute, time.Second).Should(BeNil())

By("creating an instance of CR")
inputFile := filepath.Join("config", "samples", fmt.Sprintf("%s_%s_%s.yaml", c.group, c.version, strings.ToLower(c.kind)))
createOptions := []string{"apply", "-f", inputFile}
_, err = kubebuilderTest.RunKubectlCommand(framework.GetKubectlArgs(createOptions))
Expect(err).NotTo(HaveOccurred())

By("validate the created resource object gets reconciled in controller")
controllerContainerLogs := func() string {
// Check container log to validate that the created resource object gets reconciled in controller
logOptions := []string{"logs", controllerPodName, "-n", fmt.Sprintf("e2e-%s-system", testSuffix)}
logOutput, err := kubebuilderTest.RunKubectlCommand(framework.GetKubectlArgs(logOptions))
Expect(err).NotTo(HaveOccurred())

return logOutput
}
Eventually(controllerContainerLogs, 3*time.Minute, time.Second).Should(ContainSubstring("Updating"))
Eventually(controllerContainerLogs, 3*time.Minute, time.Second).Should(ContainSubstring("Updating"))
})
})
Loading

0 comments on commit 88726be

Please sign in to comment.