Run upstream e2e tests

Makefile

@@ -19,6 +19,10 @@ VERSION=0.1.0
 GIT_COMMIT?=$(shell git rev-parse HEAD)
 BUILD_DATE?=$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
 LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE}"
+GO111MODULE=on
+GOPROXY=direct
+
+.EXPORT_ALL_VARIABLES:
 
 .PHONY: aws-efs-csi-driver
 aws-efs-csi-driver:
@@ -33,6 +37,10 @@ verify:
 test:
 	go test -v -race ./pkg/...
 
+.PHONY: test-e2e
+test-e2e:
+	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c ./hack/run-e2e-test
+
 .PHONY: image
 image:
 	docker build -t $(IMAGE):latest .

hack/run-e2e-test

@@ -0,0 +1,125 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+OS_ARCH=$(go env GOOS)-amd64
+TEST_ID=$RANDOM
+CLUSTER_NAME=test-cluster-$TEST_ID
+TEST_DIR=/tmp/efs-e2e-test
+BASE_DIR=$(dirname $0)
+REGION=${AWS_REGION-us-east-1}
+ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c}
+K8S_VERSION=${K8S_VERSION-1.14.1}
+
+echo "Testing in region: $REGION and zones: $ZONES"
+
+KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/1.14.0-alpha.3/kops-$OS_ARCH
+KOPS_PATH=$TEST_DIR/kops
+KOPS_STATE_FILE=s3://k8s-kops-csi-e2e
+
+# Download kops if not yet
+if [[ ! -e $KOPS_PATH ]]; then
+  mkdir -p $TEST_DIR 
+  echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH"
+  curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH
+  chmod +x $KOPS_PATH
+fi
+
+# Push test driver image
+eval $(aws ecr get-login --region $REGION --no-include-email)
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+IMAGE_TAG=$TEST_ID
+IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/aws-efs-csi-driver
+docker build -t $IMAGE_NAME:$IMAGE_TAG .
+docker push $IMAGE_NAME:$IMAGE_TAG
+
+set +e
+echo "Creating cluster $CLUSTER_NAME"
+CLUSTER_YAML_PATH=$TEST_DIR/$CLUSTER_NAME.yaml
+SSH_KEY_PATH=$TEST_DIR/id_rsa
+ssh-keygen -P csi-e2e -f $SSH_KEY_PATH
+
+$KOPS_PATH create cluster --state $KOPS_STATE_FILE \
+  --zones $ZONES \
+  --node-count=3 \
+  --kubernetes-version=$K8S_VERSION \
+  --ssh-public-key=$SSH_KEY_PATH.pub \
+  $CLUSTER_NAME.k8s.local
+$KOPS_PATH get cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local -o yaml > $CLUSTER_YAML_PATH
+$KOPS_PATH replace --state $KOPS_STATE_FILE -f $CLUSTER_YAML_PATH
+$KOPS_PATH update cluster --state $KOPS_STATE_FILE $CLUSTER_NAME.k8s.local --yes
+
+# Wait for cluster creation
+while [[ 1 ]]; do
+  $KOPS_PATH validate cluster --state $KOPS_STATE_FILE
+  ret=$?
+  if [[ $ret -eq 0 ]]; then
+    break
+  else
+    echo "Waiting cluster to be created"
+    sleep 30
+  fi
+done;
+
+
+echo "Deploying driver"
+cat deploy/kubernetes/manifest.yaml | sed s,amazon/aws-efs-csi-driver:latest,$IMAGE_NAME:$IMAGE_TAG, > $TEST_DIR/manifest.yaml
+kubectl apply -f $TEST_DIR/manifest.yaml
+
+echo "Creating EFS file system"
+aws efs create-file-system --creation-token $TEST_ID --tags Key=KubernetesCluster,Value=$CLUSTER_NAME.k8s.local --region $REGION
+FILE_SYSTEM_ID=$(aws efs describe-file-systems --creation-token $TEST_ID --region $REGION | jq -r '.FileSystems[0].FileSystemId')
+
+GROUP_ID=$(aws ec2 describe-security-groups --filters Name=group-name,Values=nodes.$CLUSTER_NAME.k8s.local --region $REGION | jq -r '.SecurityGroups[0].GroupId')
+aws ec2 authorize-security-group-ingress --group-id $GROUP_ID --protocol tcp --port 2049 --source-group $GROUP_ID --region $REGION
+
+for zone in ${ZONES//,/ }; do
+  SUBNET_NAME=${zone}.$CLUSTER_NAME.k8s.local
+  SUBNET_ID=$(aws ec2 describe-subnets --filters Name=tag:Name,Values=$SUBNET_NAME --region $REGION | jq -r '.Subnets[0].SubnetId')
+  aws efs create-mount-target --file-system-id $FILE_SYSTEM_ID --subnet-id $SUBNET_ID --security-groups $GROUP_ID --region $REGION
+done
+
+# Run the test
+export KUBECONFIG=$HOME/.kube/config
+
+go test -v -timeout 0 ./test/e2e/... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]" \
+  -file-system-id=$FILE_SYSTEM_ID
+TEST_PASS=$?
+
+echo "Deleting EFS file system $FILE_SYSTEM_ID"
+MOUNT_TARGETS=$(aws efs describe-mount-targets --file-system-id $FILE_SYSTEM_ID --region $REGION)
+i=0
+for zone in ${ZONES//,/ }; do
+  echo "Deleting EFS mount target $(echo $MOUNT_TARGETS | jq -r '.MountTargets['$i'].MountTargetId')"
+  aws efs delete-mount-target --mount-target-id $(echo $MOUNT_TARGETS | jq -r '.MountTargets['$i'].MountTargetId') --region $REGION
+  ((i++))
+done
+i=0
+until aws efs delete-file-system --file-system-id $FILE_SYSTEM_ID --region $REGION || [ $i == 10 ]; do
+  echo "Deleting EFS file system $FILE_SYSTEM_ID"
+  sleep 5
+  ((i++))
+done;
+
+echo "Deleting cluster $CLUSTER_NAME"
+$KOPS_PATH delete cluster --name $CLUSTER_NAME.k8s.local --state $KOPS_STATE_FILE --yes
+
+rm -rf $TEST_DIR
+
+if [[ $TEST_PASS -ne 0 ]]; then
+  exit 1
+fi

test/e2e/README.md

@@ -0,0 +1,9 @@
+# Prerequisites
+- Amazon EFS file system
+- kubernetes 1.14+ cluster whose workers (preferably 2 or more) can mount the Amazon EFS file system
+
+# Run
+```sh
+go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[efs-csi\]" -ginkgo.skip="\[Disruptive\]" \
+  -file-system-id=fs-c2a43e69
+```

test/e2e/e2e_test.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/ginkgo/reporters"
+	"github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+)
+
+func init() {
+	framework.HandleFlags()
+	framework.AfterReadingAllFlags(&framework.TestContext)
+	// PWD is test/e2e inside the git repo
+	testfiles.AddFileSource(testfiles.RootFileSource{Root: "../.."})
+}
+
+var fileSystemID = flag.String("file-system-id", "", "required. file system ID of the EFS file system to use for the test")
+
+func TestEFSCSI(t *testing.T) {
+	if *fileSystemID == "" {
+		log.Fatalf("file-system-id required. file system ID of the EFS file system to use for the test.")
+	}
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
+	var r []ginkgo.Reporter
+	if framework.TestContext.ReportDir != "" {
+		if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil {
+			log.Fatalf("Failed creating report directory: %v", err)
+		} else {
+			r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode))))
+		}
+	}
+	log.Printf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
+
+	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "EFS CSI Suite", r)
+}
+
+type efsDriver struct {
+	driverInfo testsuites.DriverInfo
+}
+
+var _ testsuites.TestDriver = &efsDriver{}
+
+// TODO implement Inline (unless it's redundant) and DynamicPV
+// var _ testsuites.InlineVolumeTestDriver = &efsDriver{}
+var _ testsuites.PreprovisionedPVTestDriver = &efsDriver{}
+
+func InitEFSCSIDriver() testsuites.TestDriver {
+	return &efsDriver{
+		driverInfo: testsuites.DriverInfo{
+			Name:                 "efs.csi.aws.com",
+			SupportedFsType:      sets.NewString(""),
+			SupportedMountOption: sets.NewString("tls"),
+			Capabilities: map[testsuites.Capability]bool{
+				testsuites.CapPersistence: true,
+				testsuites.CapExec:        true,
+				testsuites.CapMultiPODs:   true,
+				testsuites.CapRWX:         true,
+			},
+		},
+	}
+}
+
+func (e *efsDriver) GetDriverInfo() *testsuites.DriverInfo {
+	return &e.driverInfo
+}
+
+func (e *efsDriver) SkipUnsupportedTest(testpatterns.TestPattern) {}
+
+func (e *efsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+	ginkgo.By("Deploying EFS CSI driver")
+	cancelPodLogs := testsuites.StartPodLogs(f)
+
+	return &testsuites.PerTestConfig{
+			Driver:    e,
+			Prefix:    "efs",
+			Framework: f,
+		}, func() {
+			ginkgo.By("Cleaning up EFS CSI driver")
+			cancelPodLogs()
+		}
+}
+
+func (e *efsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+	return nil
+}
+
+func (e *efsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, volume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+	pvSource := v1.PersistentVolumeSource{
+		CSI: &v1.CSIPersistentVolumeSource{
+			Driver:       e.driverInfo.Name,
+			VolumeHandle: *fileSystemID,
+		},
+	}
+	return &pvSource, nil
+}
+
+// List of testSuites to be executed in below loop
+var csiTestSuites = []func() testsuites.TestSuite{
+	testsuites.InitVolumesTestSuite,
+	testsuites.InitVolumeIOTestSuite,
+	testsuites.InitVolumeModeTestSuite,
+	testsuites.InitSubPathTestSuite,
+	testsuites.InitProvisioningTestSuite,
+	//testsuites.InitSnapshottableTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
+}
+
+var _ = ginkgo.Describe("[efs-csi] EFS CSI", func() {
+	driver := InitEFSCSIDriver()
+	ginkgo.Context(testsuites.GetDriverNameWithFeatureTags(driver), func() {
+		testsuites.DefineTestSuite(driver, csiTestSuites)
+	})
+})