diff --git a/Makefile b/Makefile index 915c597780..7c876f253a 100644 --- a/Makefile +++ b/Makefile @@ -56,9 +56,13 @@ test-integration: chmod +x ${AWS_K8S_TESTER_PATH} ${AWS_K8S_TESTER_PATH} csi test integration --terminate-on-exit=true --timeout=20m ${PR_NUM_FLAG} ${VPC_ID_FLAG} -.PHONY: test-e2e -test-e2e: - ./hack/run-e2e-test +.PHONY: test-e2e-single-az +test-e2e-single-az: + AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" ./hack/run-e2e-test + +.PHONY: test-e2e-multi-az +test-e2e-multi-az: + AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" ./hack/run-e2e-test .PHONY: image-release image-release: diff --git a/docs/README.md b/docs/README.md index 15508e1c44..eecba351ef 100644 --- a/docs/README.md +++ b/docs/README.md @@ -122,7 +122,7 @@ In order to make sure that the driver complies with the CSI specification, run: make test-sanity ``` -To execute integration tests, run: +See [Ingetration Testing](../tests/integration/README.md) for more details. To execute integration tests, run: ``` make test-integration @@ -130,26 +130,13 @@ make test-integration **Note**: EC2 instance is required to run integration test, since it is exercising the actual flow of creating EBS volume, attaching it and read/write on the disk. -To execute e2e tests: +See [E2E Testing](../tests/e2e/README.md) for more details. To execute e2e tests, run: -Some tests marked with `[env]` require specific environmental variables to be set, if not set these tests will be skipped. - -``` -export AWS_AVAILABILITY_ZONES="us-west-2a,us-west-2b" -``` - -Replacing `us-west-2a,us-west-2b` with the AZ(s) where your Kubernetes worker nodes are located. - -These tests also rely on having proper [AWS credentials](https://docs.aws.amazon.com/amazonswf/latest/awsrbflowguide/set-up-creds.html) set either via environmental variables or a credentials file. - -Finally run: ``` -export KUBECONFIG=~/.kube/config -make test-e2e +make test-e2e-single-az // executes single az test suite +make test-e2e-multi-az // executes multi az test suite ``` -**Note**: By default `make test-e2e` will run 32 tests concurrently, set `GINKGO_NODES` to change the parallelism. - ### Build and Publish Container Image Build image and push it with latest tag: diff --git a/hack/additional-policies.yaml b/hack/additional-policies.yaml new file mode 100644 index 0000000000..b183a57776 --- /dev/null +++ b/hack/additional-policies.yaml @@ -0,0 +1,22 @@ + additionalPolicies: + node: | + [ + { + "Effect": "Allow", + "Action": [ + "ec2:AttachVolume", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DeleteSnapshot", + "ec2:DeleteTags", + "ec2:DeleteVolume", + "ec2:DescribeInstances", + "ec2:DescribeSnapshots", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DetachVolume" + ], + "Resource": "*" + } + ] diff --git a/hack/feature-gates.yaml b/hack/feature-gates.yaml new file mode 100644 index 0000000000..19f0e33a97 --- /dev/null +++ b/hack/feature-gates.yaml @@ -0,0 +1,8 @@ + kubeAPIServer: + featureGates: + CSIDriverRegistry: "true" + CSINodeInfo: "true" + kubelet: + featureGates: + CSIDriverRegistry: "true" + CSINodeInfo: "true" diff --git a/hack/run-e2e-test b/hack/run-e2e-test index 6155f526e3..e4c63db96b 100755 --- a/hack/run-e2e-test +++ b/hack/run-e2e-test @@ -14,8 +14,93 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -euo pipefail +set -uo pipefail +OS_ARCH=$(go env GOOS)-amd64 +TEST_ID=$RANDOM +CLUSTER_NAME=test-cluster-$TEST_ID +TEST_DIR=/tmp/ebs-e2e-test +BASE_DIR=$(dirname $0) +REGION=${AWS_REGION-us-east-1} +ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c} +FOCUS=${GINKGO_FOCUS-"[ebs-csi-e2e]"} NODES=${GINKGO_NODES:-32} -ginkgo -p -nodes=$NODES -v --focus ebs-csi-e2e tests/e2e +echo "Testing in region: $REGION and zones: $ZONES" + +KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/1.11.0/kops-$OS_ARCH +KOPS_PATH=$TEST_DIR/kops +KOPS_STATE_FILE=s3://k8s-kops-csi-e2e + +# Download kops if not yet +if [[ ! -e $KOPS_PATH ]]; then + mkdir -p $TEST_DIR + echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH" + curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH +fi + +chmod +x $KOPS_PATH + +echo "Creating cluster $CLUSTER_NAME" +CLUSTER_YAML_PATH=$TEST_DIR/$CLUSTER_NAME.yaml +$KOPS_PATH create cluster --state $KOPS_STATE_FILE \ + --zones $ZONES \ + --node-count=3 \ + --kubernetes-version=1.13.0 \ + $CLUSTER_NAME.k8s.local +$KOPS_PATH get cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local -o yaml > $CLUSTER_YAML_PATH +cat $BASE_DIR/feature-gates.yaml >> $CLUSTER_YAML_PATH +cat $BASE_DIR/additional-policies.yaml >> $CLUSTER_YAML_PATH +$KOPS_PATH replace --state $KOPS_STATE_FILE -f $CLUSTER_YAML_PATH +$KOPS_PATH update cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local --yes + +# Wait for cluster creation +while [[ 1 ]]; do + $KOPS_PATH validate cluster --state $KOPS_STATE_FILE + ret=$? + if [[ $ret -eq 0 ]]; then + break + else + echo "Waiting cluster to be created" + sleep 30 + fi +done; + +# Push test driver image +eval $(aws ecr get-login --region $REGION --no-include-email) +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +IMAGE_TAG=$TEST_ID +IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/aws-ebs-csi-driver:$IMAGE_TAG +docker build -t $IMAGE_NAME . +docker push $IMAGE_NAME + +# Update manifest files +cp deploy/kubernetes/*.yaml $TEST_DIR +sed -i '' "s/image: amazon\/aws-ebs-csi-driver:latest/image: $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com\/aws-ebs-csi-driver:$IMAGE_TAG/g" $TEST_DIR/controller.yaml +sed -i '' "s/image: amazon\/aws-ebs-csi-driver:latest/image: $AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com\/aws-ebs-csi-driver:$IMAGE_TAG/g" $TEST_DIR/node.yaml + +echo "Deploying driver" +kubectl create -f https://raw.githubusercontent.com/kubernetes/csi-api/release-1.13/pkg/crd/manifests/csinodeinfo.yaml --validate=false +kubectl apply -f $TEST_DIR/controller.yaml +kubectl apply -f $TEST_DIR/node.yaml +kubectl apply -f $TEST_DIR/secret.yaml + +# Run the test +go install -v github.com/onsi/ginkgo/ginkgo +export KUBECONFIG=$HOME/.kube/config +ginkgo -p -nodes=$NODES -v --focus="$FOCUS" tests/e2e +TEST_PASS=$? + +echo "Removing driver" +kubectl delete -f $TEST_DIR/controller.yaml +kubectl delete -f $TEST_DIR/node.yaml +kubectl delete -f $TEST_DIR/secret.yaml + +echo "Deleting cluster $CLUSTER_NAME" +$KOPS_PATH delete cluster --name $CLUSTER_NAME.k8s.local --state $KOPS_STATE_FILE --yes + +rm -rf $TEST_DIR + +if [[ $TEST_PASS -ne 0 ]]; then + exit 1 +fi diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000000..c61904d618 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,24 @@ +## E2E Testing +E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc. + +### Requirements +1. AWS credential is [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) +1. AWS CLI v1.16+ +1. Kubectl v1.13+ +1. Docker CLI v18.09+ +1. curl +1. sed +1. Golang 1.11+ + +### Notes +Some tests marked with `[env]` require specific environmental variables to be set, if not set these tests will be skipped. + +``` +export AWS_AVAILABILITY_ZONES="us-west-2a,us-west-2b" +``` + +Replacing `us-west-2a,us-west-2b` with the AZ(s) where your Kubernetes worker nodes are located. + +By default `make test-e2e-` targets will run 32 tests concurrently, set `GINKGO_NODES` to change the parallelism. + +