Merge pull request #55 from wongma7/e2e-test

Run upstream e2e tests

Makefile

@@ -43,6 +43,10 @@ test:
 test-e2e:
 	echo "e2e test"
 
+.PHONY: test-e2e
+test-e2e:
+	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c ./hack/run-e2e-test
+
 .PHONY: image
 image:
 	docker build -t $(IMAGE):latest .

hack/run-e2e-test

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+OS_ARCH=$(go env GOOS)-amd64
+TEST_ID=$RANDOM
+CLUSTER_NAME=test-cluster-$TEST_ID
+TEST_DIR=/tmp/efs-e2e-test
+BASE_DIR=$(dirname $0)
+REGION=${AWS_REGION-us-east-1}
+ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c}
+K8S_VERSION=${K8S_VERSION-1.14.1}
+
+echo "Testing in region: $REGION and zones: $ZONES"
+
+KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/1.14.0-alpha.3/kops-$OS_ARCH
+KOPS_PATH=$TEST_DIR/kops
+KOPS_STATE_FILE=s3://k8s-kops-csi-e2e
+
+# Download kops if not yet
+if [[ ! -e $KOPS_PATH ]]; then
+  mkdir -p $TEST_DIR 
+  echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH"
+  curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH
+  chmod +x $KOPS_PATH
+fi
+
+# Push test driver image
+eval $(aws ecr get-login --region $REGION --no-include-email)
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+IMAGE_TAG=$TEST_ID
+IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/aws-efs-csi-driver
+docker build -t $IMAGE_NAME:$IMAGE_TAG .
+docker push $IMAGE_NAME:$IMAGE_TAG
+
+set +e
+echo "Creating cluster $CLUSTER_NAME"
+CLUSTER_YAML_PATH=$TEST_DIR/$CLUSTER_NAME.yaml
+SSH_KEY_PATH=$TEST_DIR/id_rsa
+ssh-keygen -P csi-e2e -f $SSH_KEY_PATH
+
+$KOPS_PATH create cluster --state $KOPS_STATE_FILE \
+  --zones $ZONES \
+  --node-count=3 \
+  --kubernetes-version=$K8S_VERSION \
+  --ssh-public-key=$SSH_KEY_PATH.pub \
+  $CLUSTER_NAME.k8s.local
+$KOPS_PATH get cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local -o yaml > $CLUSTER_YAML_PATH
+$KOPS_PATH replace --state $KOPS_STATE_FILE -f $CLUSTER_YAML_PATH
+$KOPS_PATH update cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local --yes
+
+# Wait for cluster creation
+while [[ 1 ]]; do
+  $KOPS_PATH validate cluster --state $KOPS_STATE_FILE
+  ret=$?
+  if [[ $ret -eq 0 ]]; then
+    break
+  else
+    echo "Waiting cluster to be created"
+    sleep 30
+  fi
+done;
+
+
+echo "Deploying driver"
+cat deploy/kubernetes/manifest.yaml | sed s,amazon/aws-efs-csi-driver:latest,$IMAGE_NAME:$IMAGE_TAG, > $TEST_DIR/manifest.yaml
+kubectl apply -f $TEST_DIR/manifest.yaml
+
+echo "Creating EFS file system"
+aws efs create-file-system --creation-token $TEST_ID --tags Key=KubernetesCluster,Value=$CLUSTER_NAME.k8s.local --region $REGION
+FILE_SYSTEM_ID=$(aws efs describe-file-systems --creation-token $TEST_ID --region $REGION | jq -r '.FileSystems[0].FileSystemId')
+
+GROUP_ID=$(aws ec2 describe-security-groups --filters Name=group-name,Values=nodes.$CLUSTER_NAME.k8s.local --region $REGION | jq -r '.SecurityGroups[0].GroupId')
+aws ec2 authorize-security-group-ingress --group-id $GROUP_ID --protocol tcp --port 2049 --source-group $GROUP_ID --region $REGION
+
+for zone in ${ZONES//,/ }; do
+  SUBNET_NAME=${zone}.$CLUSTER_NAME.k8s.local
+  SUBNET_ID=$(aws ec2 describe-subnets --filters Name=tag:Name,Values=$SUBNET_NAME --region $REGION | jq -r '.Subnets[0].SubnetId')
+  aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_ID --security-groups $GROUP_ID --region $REGION
+done
+
+# Run the test
+export KUBECONFIG=$HOME/.kube/config
+
+pushd ./test/e2e
+go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]" \
+  -file-system-id=$FILE_SYSTEM_ID
+TEST_PASS=$?
+popd
+
+echo "Deleting EFS file system $FILE_SYSTEM_ID"
+MOUNT_TARGETS=$(aws efs describe-mount-targets --file-system-id $FILE_SYSTEM_ID --region $REGION)
+i=0
+for zone in ${ZONES//,/ }; do
+  echo "Deleting EFS mount target $(echo $MOUNT_TARGETS | jq -r '.MountTargets['$i'].MountTargetId')"
+  aws efs delete-mount-target --mount-target-id $(echo $MOUNT_TARGETS | jq -r '.MountTargets['$i'].MountTargetId') --region $REGION
+  ((i++))
+done
+i=0
+until aws efs delete-file-system --file-system-id $FILE_SYSTEM_ID --region $REGION || [ $i == 10 ]; do
+  echo "Deleting EFS file system $FILE_SYSTEM_ID"
+  sleep 5
+  ((i++))
+done;
+
+echo "Deleting cluster $CLUSTER_NAME"
+$KOPS_PATH delete cluster --name $CLUSTER_NAME.k8s.local --state $KOPS_STATE_FILE --yes
+
+rm -rf $TEST_DIR
+
+if [[ $TEST_PASS -ne 0 ]]; then
+  exit 1
+fi

test/e2e/README.md

@@ -0,0 +1,15 @@
+# Prerequisites
+- Amazon EFS file system
+- kubernetes 1.14+ cluster whose workers (preferably 2 or more) can mount the Amazon EFS file system
+
+# Run
+```sh
+go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]" \
+  -file-system-id=fs-c2a43e69
+```
+
+# Update dependencies
+```sh
+go mod edit -require=k8s.io/kubernetes@v1.15.3
+./hack/update-gomod.sh v1.15.3
+```

test/e2e/e2e_test.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/ginkgo/reporters"
+	"github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+)
+
+func init() {
+	framework.HandleFlags()
+	framework.AfterReadingAllFlags(&framework.TestContext)
+	// PWD is test/e2e inside the git repo
+	testfiles.AddFileSource(testfiles.RootFileSource{Root: "../.."})
+}
+
+var fileSystemID = flag.String("file-system-id", "", "required. file system ID of the EFS file system to use for the test")
+
+func TestEFSCSI(t *testing.T) {
+	if *fileSystemID == "" {
+		log.Fatalf("file-system-id required. file system ID of the EFS file system to use for the test.")
+	}
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
+	var r []ginkgo.Reporter
+	if framework.TestContext.ReportDir != "" {
+		if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil {
+			log.Fatalf("Failed creating report directory: %v", err)
+		} else {
+			r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode))))
+		}
+	}
+	log.Printf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
+
+	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "EFS CSI Suite", r)
+}
+
+type efsDriver struct {
+	driverInfo testsuites.DriverInfo
+}
+
+var _ testsuites.TestDriver = &efsDriver{}
+
+// TODO implement Inline (unless it's redundant) and DynamicPV
+// var _ testsuites.InlineVolumeTestDriver = &efsDriver{}
+var _ testsuites.PreprovisionedPVTestDriver = &efsDriver{}
+
+func InitEFSCSIDriver() testsuites.TestDriver {
+	return &efsDriver{
+		driverInfo: testsuites.DriverInfo{
+			Name:                 "efs.csi.aws.com",
+			SupportedFsType:      sets.NewString(""),
+			SupportedMountOption: sets.NewString("tls"),
+			Capabilities: map[testsuites.Capability]bool{
+				testsuites.CapPersistence: true,
+				testsuites.CapExec:        true,
+				testsuites.CapMultiPODs:   true,
+				testsuites.CapRWX:         true,
+			},
+		},
+	}
+}
+
+func (e *efsDriver) GetDriverInfo() *testsuites.DriverInfo {
+	return &e.driverInfo
+}
+
+func (e *efsDriver) SkipUnsupportedTest(testpatterns.TestPattern) {}
+
+func (e *efsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+	ginkgo.By("Deploying EFS CSI driver")
+	cancelPodLogs := testsuites.StartPodLogs(f)
+
+	return &testsuites.PerTestConfig{
+			Driver:    e,
+			Prefix:    "efs",
+			Framework: f,
+		}, func() {
+			ginkgo.By("Cleaning up EFS CSI driver")
+			cancelPodLogs()
+		}
+}
+
+func (e *efsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+	return nil
+}
+
+func (e *efsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, volume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+	pvSource := v1.PersistentVolumeSource{
+		CSI: &v1.CSIPersistentVolumeSource{
+			Driver:       e.driverInfo.Name,
+			VolumeHandle: *fileSystemID,
+		},
+	}
+	return &pvSource, nil
+}
+
+// List of testSuites to be executed in below loop
+var csiTestSuites = []func() testsuites.TestSuite{
+	testsuites.InitVolumesTestSuite,
+	testsuites.InitVolumeIOTestSuite,
+	testsuites.InitVolumeModeTestSuite,
+	testsuites.InitSubPathTestSuite,
+	testsuites.InitProvisioningTestSuite,
+	//testsuites.InitSnapshottableTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
+}
+
+var _ = ginkgo.Describe("[efs-csi] EFS CSI", func() {
+	driver := InitEFSCSIDriver()
+	ginkgo.Context(testsuites.GetDriverNameWithFeatureTags(driver), func() {
+		testsuites.DefineTestSuite(driver, csiTestSuites)
+	})
+})

test/e2e/go.mod

@@ -0,0 +1,59 @@
+module github.com/kubernetes-sigs/aws-efs-csi-driver/test/e2e
+
+go 1.12
+
+replace k8s.io/api => k8s.io/api v0.0.0-20190805141119-fdd30b57c827
+
+replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190805143126-cdb999c96590
+
+replace k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719
+
+replace k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190805142138-368b2058237c
+
+replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20190805143448-a07e59fb081d
+
+replace k8s.io/client-go => k8s.io/client-go v0.0.0-20190805141520-2fe0317bcee0
+
+replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20190805144409-8484242760e7
+
+replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.0.0-20190805144246-c01ee70854a1
+
+replace k8s.io/code-generator => k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b
+
+replace k8s.io/component-base => k8s.io/component-base v0.0.0-20190805141645-3a5e5ac800ae
+
+replace k8s.io/cri-api => k8s.io/cri-api v0.0.0-20190531030430-6117653b35f1
+
+replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.0.0-20190805144531-3985229e1802
+
+replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.0.0-20190805142416-fd821fbbb94e
+
+replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.0.0-20190805144128-269742da31dd
+
+replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20190805143734-7f1675b90353
+
+replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20190805144012-2a1ed1f3d8a4
+
+replace k8s.io/kubectl => k8s.io/kubectl v0.0.0-20190602132728-7075c07e78bf
+
+replace k8s.io/kubelet => k8s.io/kubelet v0.0.0-20190805143852-517ff267f8d1
+
+replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20190805144654-3d5bf3a310c1
+
+replace k8s.io/metrics => k8s.io/metrics v0.0.0-20190805143318-16b07057415d
+
+replace k8s.io/node-api => k8s.io/node-api v0.0.0-20190805144819-9dd62e4d5327
+
+replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20190805142637-3b65bc4bb24f
+
+replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.0.0-20190805143616-1485e5142db3
+
+replace k8s.io/sample-controller => k8s.io/sample-controller v0.0.0-20190805142825-b16fad786282
+
+require (
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v1.5.0
+	k8s.io/api v0.0.0
+	k8s.io/apimachinery v0.0.0
+	k8s.io/kubernetes v1.15.2
+)