Skip to content

Commit

Permalink
Add conformance tests
Browse files Browse the repository at this point in the history
  • Loading branch information
cpanato committed Nov 27, 2020
1 parent 0b30d06 commit 32f7748
Show file tree
Hide file tree
Showing 9 changed files with 232 additions and 7 deletions.
11 changes: 9 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -234,9 +234,16 @@ e2e-image:
.PHONY: e2e
e2e: e2e-image
# This is the name used inside the component.yaml for the container that runs the manager
# The image gets loaded inside kind from ./test/e2e/config/packet-dev.yaml
# The image gets loaded inside kind from ./test/e2e/config/packet-ci.yaml
$(E2E_FLAGS) $(MAKE) -C $(TEST_E2E_DIR) run

# Run conformance tests
.PHONY: conformance
conformance: e2e-image
# This is the name used inside the component.yaml for the container that runs the manager
# The image gets loaded inside kind from ./test/e2e/config/packet-ci.yaml
$(E2E_FLAGS) $(MAKE) -C $(TEST_E2E_DIR) run-conformance

# Build manager binary
manager: $(MANAGER)
$(MANAGER): generate fmt vet
Expand Down Expand Up @@ -407,4 +414,4 @@ cluster-init-manual: core managerless release
.PHONY: modules
modules: ## Runs go mod to ensure modules are up to date.
go mod tidy
cd $(TOOLS_DIR); go mod tidy
cd $(TOOLS_DIR); go mod tidy
File renamed without changes.
78 changes: 78 additions & 0 deletions scripts/ci-conformance.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/bin/bash

# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

###############################################################################

# This script is executed by presubmit `pull-cluster-api-provider-azure-e2e`
# To run locally, set PACKET_API_KEY, PROJECT_ID, ``

set -o errexit
set -o nounset
set -o pipefail

REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
cd "${REPO_ROOT}" || exit 1

# shellcheck source=../hack/ensure-go.sh
source "${REPO_ROOT}/hack/ensure-go.sh"
# shellcheck source=../hack/ensure-kind.sh
source "${REPO_ROOT}/hack/ensure-kind.sh"
# shellcheck source=../hack/ensure-kubectl.sh
source "${REPO_ROOT}/hack/ensure-kubectl.sh"
# shellcheck source=../hack/ensure-kustomize.sh
source "${REPO_ROOT}/hack/ensure-kustomize.sh"
# shellcheck source=../hack/ensure-packet-cli.sh
source "${REPO_ROOT}/hack/ensure-packet-cli.sh"

# Verify the required Environment Variables are present.
: "${PACKET_API_KEY:?Environment variable empty or not defined.}"
: "${PROJECT_ID:?Environment variable empty or not defined.}"

get_random_facility() {
# local FACILITIES=("sjc1" "lax1" "ams1" "dfw2" "ewr1" "ny5")
local FACILITIES=("ewr1")
echo "${FACILITIES[${RANDOM} % ${#FACILITIES[@]}]}"
}

export GINKGO_NODES=3
export FACILITY="${FACILITY:-$(get_random_facility)}"
export PACKET_TOKEN=${PACKET_API_KEY}

export SSH_KEY_NAME=capp-e2e-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 12 ; echo '')
export SSH_KEY_PATH=/tmp/${SSH_KEY_NAME}
export SSH_KEY_UUID=""
create_ssh_key() {
echo "generating new ssh key"
ssh-keygen -t rsa -f ${SSH_KEY_PATH} -N '' 2>/dev/null <<< y >/dev/null
echo "importing ssh key "
SSH_KEY_STRING=$(cat ${SSH_KEY_PATH}.pub)
SSH_KEY_UUID=$(packet ssh-key create --key "${SSH_KEY_STRING}" --label "${SSH_KEY_NAME}" --json | jq -r '.id')
}

cleanup() {
echo "removing ssh key"
packet ssh-key delete --id ${SSH_KEY_UUID} --force || true
rm -f ${SSH_KEY_PATH} || true

${REPO_ROOT}/hack/log/redact.sh || true
}

create_ssh_key
trap cleanup EXIT

export SSH_KEY=${SSH_KEY_NAME}
make conformance
test_status="${?}"
14 changes: 13 additions & 1 deletion test/e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,22 @@ ARTIFACTS ?= ${REPO_ROOT}/_artifacts
SKIP_RESOURCE_CLEANUP ?= false
USE_EXISTING_CLUSTER ?= false
GINKGO_NOCOLOR ?= false
E2E_DATA_DIR ?= $(TEST_E2E_DIR)/data
KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)

.PHONY: run
run: ginkgo ## Run the end-to-end tests
cd $(TEST_E2E_DIR); $(GINKGO) -v -trace -tags=e2e -focus=$(GINKGO_FOCUS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) . -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE)" \
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)

.PHONY: run-conformance
run-conformance: ginkgo ## Run the conformance tests
cd $(TEST_E2E_DIR); $(GINKGO) -v -trace -stream -progress -tags=e2e -focus='Conformance Tests' -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) . -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE)" \
-kubetest.config-file=$(KUBETEST_CONF_PATH) \
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)
4 changes: 1 addition & 3 deletions test/e2e/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,8 @@ func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterPr
Deleter: bootstrapClusterProxy.GetClient(),
Name: namespace.Name,
})

// Will call the clean resources just to make sure we clean everything
By(fmt.Sprintf("Making sure there is no leftover running for %s", cluster.Name))
}

cancelWatches()
redactLogs()
}
Expand Down
4 changes: 3 additions & 1 deletion test/e2e/config/packet-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ providers:
files:
- sourcePath: "../../../metadata.yaml"
targetName: "metadata.yaml"
- sourcePath: "../../../templates/cluster-template.yaml"
- sourcePath: "../../../templates/cluster-template-ci.yaml"
targetName: "cluster-template.yaml"

variables:
Expand All @@ -69,6 +69,8 @@ variables:
WORKER_NODE_TYPE: "t1.small"
POD_CIDR: "192.168.0.0/16"
SERVICE_CIDR: "172.26.0.0/16"
CONFORMANCE_WORKER_MACHINE_COUNT: "3"
CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1"
CNI: "../../templates/addons/calico.yaml"
REDACT_LOG_SCRIPT: "../../../hack/log/redact.sh"

Expand Down
117 changes: 117 additions & 0 deletions test/e2e/conformance_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// +build e2e

/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
"sigs.k8s.io/cluster-api/test/framework/kubetest"
"sigs.k8s.io/cluster-api/util"
)

var _ = Describe("Conformance Tests", func() {
var (
ctx = context.TODO()
specName = "conformance-tests"
namespace *corev1.Namespace
cancelWatches context.CancelFunc
cluster *clusterv1.Cluster
clusterName string
clusterctlLogFolder string
)

BeforeEach(func() {
Expect(e2eConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName)
Expect(clusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. clusterctlConfigPath must be an existing file when calling %s spec", specName)
Expect(bootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. bootstrapClusterProxy can't be nil when calling %s spec", specName)
Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName)
Expect(kubetestConfigFilePath).ToNot(BeNil(), "Invalid argument. kubetestConfigFilePath can't be nil")

Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.KubernetesVersion))
Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath))

clusterName = fmt.Sprintf("capp-conf-%s", util.RandomString(6))

// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)

// We need to override clusterctl apply log folder to avoid getting our credentials exposed.
clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName())
})

Measure(specName, func(b Benchmarker) {
var err error

workerMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_WORKER_MACHINE_COUNT"), 10, 64)
Expect(err).NotTo(HaveOccurred())
controlPlaneMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT"), 10, 64)
Expect(err).NotTo(HaveOccurred())

runtime := b.Time("cluster creation", func() {
result := clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
ClusterProxy: bootstrapClusterProxy,
ConfigCluster: clusterctl.ConfigClusterInput{
LogFolder: clusterctlLogFolder,
ClusterctlConfigPath: clusterctlConfigPath,
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: clusterName,
KubernetesVersion: e2eConfig.GetVariable(capi_e2e.KubernetesVersion),
ControlPlaneMachineCount: pointer.Int64Ptr(controlPlaneMachineCount),
WorkerMachineCount: pointer.Int64Ptr(workerMachineCount),
},
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
})

cluster = result.Cluster
})

b.RecordValue("cluster creation", runtime.Seconds())
workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterName)
runtime = b.Time("conformance suite", func() {
kubetest.Run(
kubetest.RunInput{
ClusterProxy: workloadProxy,
NumberOfNodes: int(workerMachineCount),
ConfigFilePath: kubetestConfigFilePath,
},
)
})
b.RecordValue("conformance suite run time", runtime.Seconds())
}, 1)

AfterEach(func() {
dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, cluster, e2eConfig.GetIntervals, clusterName, clusterctlLogFolder, skipCleanup)
})
})
7 changes: 7 additions & 0 deletions test/e2e/data/kubetest/conformance.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
ginkgo.focus: \[Conformance\]
disable-log-dump: true
ginkgo.progress: true
ginkgo.slowSpecThreshold: 120.0
ginkgo.flakeAttempts: 3
ginkgo.trace: true
ginkgo.v: true
4 changes: 4 additions & 0 deletions test/e2e/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,17 @@ var (

// bootstrapClusterProxy allows to interact with the bootstrap cluster to be used for the e2e tests.
bootstrapClusterProxy framework.ClusterProxy

// kubetestConfigFilePath is the path to the kubetest configuration file
kubetestConfigFilePath string
)

func init() {
flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file")
flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored")
flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped")
flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
flag.StringVar(&kubetestConfigFilePath, "kubetest.config-file", "", "path to the kubetest configuration file")
}

func TestE2E(t *testing.T) {
Expand Down

0 comments on commit 32f7748

Please sign in to comment.