fix(e2e): set up variables for non-bare e2e runs

Makefile

@@ -69,21 +69,20 @@ run-local-shift:
 	. ./scripts/install_local.sh local build/resources
 	rm -rf build
 
+# useful if running e2e directly with `go test -tags=bare`
 setup-bare:
 	. ./scripts/build_bare.sh
 	. ./scripts/package-release.sh 1.0.0-e2e test/e2e/resources test/e2e/e2e-bare-values.yaml
-	. ./scripts/install_bare.sh e2e test/e2e/resources
+	. ./scripts/install_bare.sh $(shell cat ./e2e.namespace) test/e2e/resources
 
 e2e:
-	export NAMESPACE=default
-	go test ./test/e2e/...
+	go test -v -timeout 20m ./test/e2e/... -namespace=default -kubeconfig=${KUBECONFIG} -olmNamespace=openshift-operator-lifecycle-manager
 
 e2e-local:
 	. ./scripts/build_local.sh
 	. ./scripts/run_e2e_local.sh $(TEST)
 
-e2e-bare:
-	. ./scripts/build_bare.sh
+e2e-bare: setup-bare
 	. ./scripts/run_e2e_bare.sh $(TEST)
 
 e2e-local-shift:

manifests/0000_30_11-catalog-operator.deployment.yaml

@@ -26,6 +26,7 @@ spec:
           - /bin/catalog
           - '-namespace'
           - openshift-operator-lifecycle-manager
+          - -debug
           image: quay.io/coreos/olm@sha256:1639d570809c5827810a1870763016e8c046283632d47e0b47183c82f8e515f2
           imagePullPolicy: IfNotPresent
           ports:

scripts/build_bare.sh

@@ -12,10 +12,9 @@ if [ -z "$NO_MINIKUBE" ]; then
   umask 0077 && kubectl config view --minify --flatten --context=minikube > minikube.kubeconfig
 fi
 
-kubectl delete crds --all
-kubectl delete namespace e2e || true
-kubectl wait --for=delete namespace/e2e || true
-kubectl create namespace e2e
+timestamp=$(date +%s)
+namespace="e2e-tests-${timestamp}-$RANDOM"
+printf  "${namespace}" > e2e.namespace
 
-# only used for package server, other operators run locally
-docker build -t quay.io/coreos/olm:local .
+kubectl delete crds --all
+kubectl create namespace ${namespace}

scripts/install_bare.sh

@@ -28,6 +28,3 @@ do
         kubectl apply -f ${f};
     fi
 done
-
-# wait for package server to be ready
-#kubectl rollout status -w deployment.apps/package-server --namespace=${namespace}

scripts/run_e2e_bare.sh

@@ -1,38 +1,10 @@
 #!/usr/bin/env bash
 
 # Note: run from root
-# Individual tests can be run by calling ./test/e2e/run_e2e_local.sh TestName
+# Individual tests can be run by calling ./test/e2e/run_e2e_bare.sh TestName
 
 set -e
 
-timestamp=$(date +%s)
-namespace="e2e-tests-${timestamp}-$RANDOM"
-
-function cleanup {
- 	kubectl delete namespace ${namespace}
- 	rm -rf test/e2e/resources
-}
-
-function cleanupAndExit {
-	exitCode=$?
-	if [ "$exitCode" -ne "0" ]; then
-		echo "error running tests. logs written to package.log";
-		kubectl -n ${namespace} logs -l app=package-server > package.log
-	else
-		cleanup
-	fi
-
-    exit $exitCode
-}
-
-trap cleanupAndExit SIGINT SIGTERM EXIT
-
-
-kubectl create namespace ${namespace}
-
-./scripts/package-release.sh 1.0.0-e2e test/e2e/resources test/e2e/e2e-bare-values.yaml
-./scripts/install_bare.sh ${namespace} test/e2e/resources
-
 # run tests
 if [ -z "$1" ]; then
   test_flags="";
@@ -41,4 +13,5 @@ else
 fi
 
 echo "${test_flags}"
-go test -tags=bare -mod=vendor -covermode=count -coverpkg ./pkg/controller/...  -test.v -test.timeout 20m ${test_flags} ./test/e2e/... -kubeconfig=${KUBECONFIG:-~/.kube/config} -namespace=${namespace}
+go test -c -tags=bare -mod=vendor -v -o e2e-bare github.com/operator-framework/operator-lifecycle-manager/test/e2e
+./e2e-bare  -test.v -test.timeout 20m ${test_flags} -kubeconfig=minikube.kubeconfig -namespace=$(cat e2e.namespace)

scripts/run_e2e_local.sh

@@ -41,5 +41,11 @@ trap cleanupAndExit SIGINT SIGTERM EXIT
 ./scripts/install_local.sh ${namespace} test/e2e/resources
 
 # run tests
-e2e_kubeconfig=${KUBECONFIG:-~/.kube/config}
-KUBECONFIG=${e2e_kubeconfig} NAMESPACE=${namespace} go test -v -timeout 20m ./test/e2e/... ${1/[[:alnum:]-]*/-run ${1}}
+if [ -z "$1" ]; then
+  test_flags="";
+else
+  test_flags="-test.run ${1}"
+fi
+
+echo "${test_flags}"
+go test -mod=vendor -tags=local -covermode=count -coverpkg ./pkg/controller/...  -test.v -test.timeout 20m ${test_flags} ./test/e2e/... -kubeconfig=${KUBECONFIG:-~/.kube/config} -namespace=${namespace} -olmNamespace=${namespace}

test/e2e/catalog_e2e_test.go

@@ -4,9 +4,9 @@ package e2e
 
 import (
 	"fmt"
-	"github.com/coreos/go-semver/semver"
 	"testing"
 
+	"github.com/coreos/go-semver/semver"
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
 	extv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
@@ -45,14 +45,13 @@ func TestCatalogLoadingBetweenRestarts(t *testing.T) {
 	crc := newCRClient(t)
 
 	catalogSourceName := genName("mock-ocs")
-	_, cleanupCatalogSource, err := createInternalCatalogSource(t, c, crc, catalogSourceName, testNamespace, manifests, []extv1beta1.CustomResourceDefinition{crd}, []v1alpha1.ClusterServiceVersion{csv})
+	_, cleanupCatalogSource, err := createInternalCatalogSource(t, c, crc, catalogSourceName, operatorNamespace, manifests, []extv1beta1.CustomResourceDefinition{crd}, []v1alpha1.ClusterServiceVersion{csv})
 	require.NoError(t, err)
 	defer cleanupCatalogSource()
 
 	// ensure the mock catalog exists and has been synced by the catalog operator
-	catalogSource, err := fetchCatalogSource(t, crc, catalogSourceName, testNamespace, catalogSourceSynced)
+	catalogSource, err := fetchCatalogSource(t, crc, catalogSourceName, operatorNamespace, catalogSourceSynced)
 	require.NoError(t, err)
-	t.Logf("catalogSource: %+v", catalogSource)
 
 	// get catalog operator deployment
 	deployment, err := getOperatorDeployment(c, labels.Set{"app": "catalog-operator"})
@@ -67,7 +66,7 @@ func TestCatalogLoadingBetweenRestarts(t *testing.T) {
 
 	// check for last synced update to catalogsource
 	t.Log("Checking for catalogsource lastSync updates")
-	_, err = fetchCatalogSource(t, crc, catalogSourceName, testNamespace, func(cs *v1alpha1.CatalogSource) bool {
+	_, err = fetchCatalogSource(t, crc, catalogSourceName, operatorNamespace, func(cs *v1alpha1.CatalogSource) bool {
 		if cs.Status.LastSync.After(catalogSource.Status.LastSync.Time) {
 			t.Logf("lastSync updated: %s -> %s", catalogSource.Status.LastSync, cs.Status.LastSync)
 			return true
@@ -79,7 +78,7 @@ func TestCatalogLoadingBetweenRestarts(t *testing.T) {
 }
 
 func getOperatorDeployment(c operatorclient.ClientInterface, operatorLabels labels.Set) (*appsv1.Deployment, error) {
-	deployments, err := c.ListDeploymentsWithLabels(testNamespace, operatorLabels)
+	deployments, err := c.ListDeploymentsWithLabels(operatorNamespace, operatorLabels)
 	if err != nil || deployments == nil || len(deployments.Items) != 1 {
 		return nil, fmt.Errorf("Error getting single operator deployment for label: %v", operatorLabels)
 	}
@@ -96,7 +95,7 @@ func rescaleDeployment(c operatorclient.ClientInterface, deployment *appsv1.Depl
 	}
 
 	waitForScaleup := func() (bool, error) {
-		fetchedDeployment, err := c.GetDeployment(testNamespace, deployment.GetName())
+		fetchedDeployment, err := c.GetDeployment(operatorNamespace, deployment.GetName())
 		if err != nil {
 			return true, err
 		}

test/e2e/csv_e2e_test.go

@@ -95,7 +95,7 @@ func newNginxDeployment(name string) appsv1.DeploymentSpec {
 				Containers: []v1.Container{
 					{
 						Name:  genName("nginx"),
-						Image: "nginx:1.7.9",
+						Image: "bitnami/nginx:latest",
 						Ports: []v1.ContainerPort{
 							{
 								ContainerPort: 80,
@@ -652,13 +652,30 @@ func TestCreateCSVRequirementsMetCRD(t *testing.T) {
 	require.NoError(t, err)
 	defer cleanupCSV()
 
-	fetchedCSV, err := fetchCSV(t, crc, csv.Name, csvSucceededChecker)
+	fmt.Println("checking for deployment")
+	// Poll for deployment to be ready
+	err = wait.Poll(pollInterval, pollDuration, func() (bool, error) {
+		dep, err := c.GetDeployment(testNamespace, depName)
+		if k8serrors.IsNotFound(err) {
+			fmt.Printf("deployment %s not found", depName)
+			return false, nil
+		} else if err != nil {
+			fmt.Printf("unexpected error fetching deployment %s", depName)
+			return false, err
+		}
+		if dep.Status.UpdatedReplicas == *(dep.Spec.Replicas) &&
+			dep.Status.Replicas == *(dep.Spec.Replicas) &&
+			dep.Status.AvailableReplicas == *(dep.Spec.Replicas) {
+			fmt.Printf("deployment ready")
+			return true, nil
+		}
+		fmt.Printf("deployment not ready")
+		return false, nil
+	})
 	require.NoError(t, err)
 
-	// Should create deployment
-	dep, err := c.GetDeployment(testNamespace, depName)
+	fetchedCSV, err := fetchCSV(t, crc, csv.Name, csvSucceededChecker)
 	require.NoError(t, err)
-	require.Equal(t, depName, dep.Name)
 
 	// Fetch cluster service version again to check for unnecessary control loops
 	sameCSV, err := fetchCSV(t, crc, csv.Name, csvSucceededChecker)
@@ -817,13 +834,32 @@ func TestCreateCSVRequirementsMetAPIService(t *testing.T) {
 	require.NoError(t, err)
 	defer cleanupCSV()
 
-	fetchedCSV, err := fetchCSV(t, crc, csv.Name, csvSucceededChecker)
+	fmt.Println("checking for deployment")
+	// Poll for deployment to be ready
+	err = wait.Poll(pollInterval, pollDuration, func() (bool, error) {
+		dep, err := c.GetDeployment(testNamespace, depName)
+		if k8serrors.IsNotFound(err) {
+			fmt.Printf("deployment %s not found", depName)
+			return false, nil
+		} else if err != nil {
+			fmt.Printf("unexpected error fetching deployment %s", depName)
+			return false, err
+		}
+		fmt.Printf("waiting for deployment to be available: %#v", dep)
+
+		if dep.Status.UpdatedReplicas == *(dep.Spec.Replicas) &&
+			dep.Status.Replicas == *(dep.Spec.Replicas) &&
+			dep.Status.AvailableReplicas == *(dep.Spec.Replicas) {
+			fmt.Printf("deployment ready")
+			return true, nil
+		}
+		fmt.Printf("deployment not ready")
+		return false, nil
+	})
 	require.NoError(t, err)
 
-	// Should create deployment
-	dep, err := c.GetDeployment(testNamespace, depName)
+	fetchedCSV, err := fetchCSV(t, crc, csv.Name, csvSucceededChecker)
 	require.NoError(t, err)
-	require.Equal(t, depName, dep.Name)
 
 	// Fetch cluster service version again to check for unnecessary control loops
 	sameCSV, err := fetchCSV(t, crc, csv.Name, csvSucceededChecker)

test/e2e/installplan_e2e_test.go

@@ -103,7 +103,7 @@ func newNginxInstallStrategy(name string, permissions []install.StrategyDeployme
 						Spec: corev1.PodSpec{Containers: []corev1.Container{
 							{
 								Name:  genName("nginx"),
-								Image: "nginx:1.7.9",
+								Image: "bitnami/nginx:latest",
 								Ports: []corev1.ContainerPort{{ContainerPort: 80}},
 							},
 						}},
@@ -222,7 +222,7 @@ func TestCreateInstallPlanManualApproval(t *testing.T) {
 	}
 
 	// Attempt to get the catalog source before creating install plan
-	_, err = fetchCatalogSource(t, crc, ocsConfigMap, testNamespace, catalogSourceSynced)
+	_, err = fetchCatalogSource(t, crc, ocsConfigMap, operatorNamespace, catalogSourceSynced)
 	require.NoError(t, err)
 
 	// Create a new InstallPlan for Vault with manual approval
@@ -322,7 +322,7 @@ func TestCreateInstallPlanFromInvalidClusterServiceVersionName(t *testing.T) {
 	}
 
 	// Attempt to get the catalog source before creating install plan
-	_, err := fetchCatalogSource(t, crc, ocsConfigMap, testNamespace, catalogSourceSynced)
+	_, err := fetchCatalogSource(t, crc, ocsConfigMap, operatorNamespace, catalogSourceSynced)
 	require.NoError(t, err)
 
 	cleanup, err := decorateCommonAndCreateInstallPlan(crc, testNamespace, installPlan)
@@ -396,14 +396,14 @@ func TestCreateInstallPlanWithCSVsAcrossMultipleCatalogSources(t *testing.T) {
 	require.NoError(t, err)
 	defer cleanupDependentCatalogSource()
 	// Attempt to get the catalog source before creating install plan
-	_, err = fetchCatalogSource(t, crc, dependentCatalogName, testNamespace, catalogSourceSynced)
+	_, err = fetchCatalogSource(t, crc, dependentCatalogName, operatorNamespace, catalogSourceSynced)
 	require.NoError(t, err)
 
 	_, cleanupMainCatalogSource, err := createInternalCatalogSource(t, c, crc, mainCatalogName, testNamespace, mainManifests, nil, []v1alpha1.ClusterServiceVersion{mainCSV})
 	require.NoError(t, err)
 	defer cleanupMainCatalogSource()
 	// Attempt to get the catalog source before creating install plan
-	_, err = fetchCatalogSource(t, crc, mainCatalogName, testNamespace, catalogSourceSynced)
+	_, err = fetchCatalogSource(t, crc, mainCatalogName, operatorNamespace, catalogSourceSynced)
 	require.NoError(t, err)
 
 	// Create expected install plan step sources
@@ -546,7 +546,7 @@ func TestCreateInstallPlanWithPreExistingCRDOwners(t *testing.T) {
 		require.NoError(t, err)
 		defer cleanupCatalogSource()
 		// Attempt to get the catalog source before creating install plan(s)
-		_, err = fetchCatalogSource(t, crc, catalogSourceName, testNamespace, catalogSourceSynced)
+		_, err = fetchCatalogSource(t, crc, catalogSourceName, operatorNamespace, catalogSourceSynced)
 		require.NoError(t, err)
 
 		expectedSteps := map[registry.ResourceKey]struct{}{
@@ -677,7 +677,7 @@ func TestCreateInstallPlanWithPreExistingCRDOwners(t *testing.T) {
 		require.NoError(t, err)
 		defer cleanupCatalogSource()
 		// Attempt to get the catalog source before creating install plan(s)
-		_, err = fetchCatalogSource(t, crc, catalogSourceName, testNamespace, catalogSourceSynced)
+		_, err = fetchCatalogSource(t, crc, catalogSourceName, operatorNamespace, catalogSourceSynced)
 		require.NoError(t, err)
 
 		secondOwnerCSV := v1alpha1.ClusterServiceVersion{
@@ -796,7 +796,7 @@ func TestCreateInstallPlanWithPreExistingCRDOwners(t *testing.T) {
 		require.NoError(t, err)
 		defer cleanupCatalogSource()
 		// Attempt to get the catalog source before creating install plan(s)
-		_, err = fetchCatalogSource(t, crc, catalogSourceName, testNamespace, catalogSourceSynced)
+		_, err = fetchCatalogSource(t, crc, catalogSourceName, operatorNamespace, catalogSourceSynced)
 		require.NoError(t, err)
 
 		// Create default test installplan
@@ -978,7 +978,7 @@ func TestCreateInstallPlanWithPreExistingCRDOwners(t *testing.T) {
 		require.NoError(t, err)
 		defer cleanupCatalogSource()
 		// Attempt to get the catalog source before creating install plan(s)
-		_, err = fetchCatalogSource(t, crc, catalogSourceName, testNamespace, catalogSourceSynced)
+		_, err = fetchCatalogSource(t, crc, catalogSourceName, operatorNamespace, catalogSourceSynced)
 		require.NoError(t, err)
 
 		// Create a dummy installplan with a non-existent csv
@@ -1137,7 +1137,7 @@ func TestCreateInstallPlanWithPermissions(t *testing.T) {
 	defer cleanupCatalogSource()
 
 	// Attempt to get CatalogSource
-	_, err = fetchCatalogSource(t, crc, catalogSourceName, testNamespace, catalogSourceSynced)
+	_, err = fetchCatalogSource(t, crc, catalogSourceName, operatorNamespace, catalogSourceSynced)
 	require.NoError(t, err)
 
 	// Create InstallPlan

test/e2e/metrics_e2e_test.go

@@ -16,7 +16,7 @@ func TestMetricsEndpoint(t *testing.T) {
 	c := newKubeClient(t)
 
 	listOptions := metav1.ListOptions{LabelSelector: "app=olm-operator"}
-	podList, err := c.KubernetesInterface().CoreV1().Pods(testNamespace).List(listOptions)
+	podList, err := c.KubernetesInterface().CoreV1().Pods(operatorNamespace).List(listOptions)
 	if err != nil {
 		log.Infof("Error %v\n", err)
 		t.Fatalf("Listing pods failed: %v\n", err)
@@ -27,7 +27,7 @@ func TestMetricsEndpoint(t *testing.T) {
 
 	podName := podList.Items[0].GetName()
 
-	rawOutput, err := getMetricsFromPod(t, c, podName, testNamespace, 8080)
+	rawOutput, err := getMetricsFromPod(t, c, podName, operatorNamespace, 8080)
 	if err != nil {
 		t.Fatalf("Metrics test failed: %v\n", err)
 	}

test/e2e/ocs_e2e_test.go

@@ -26,10 +26,10 @@ func TestInstallEtcdOCS(t *testing.T) {
 	c := newKubeClient(t)
 	crc := newCRClient(t)
 
-	catalogSource, err := fetchCatalogSource(t, crc, ocsConfigMap, testNamespace, catalogSourceSynced)
+	catalogSource, err := fetchCatalogSource(t, crc, ocsConfigMap, operatorNamespace, catalogSourceSynced)
 	require.NoError(t, err)
 	require.NotNil(t, catalogSource)
-	inMem, err := registry.NewInMemoryFromConfigMap(c, testNamespace, catalogSource.Spec.ConfigMap)
+	inMem, err := registry.NewInMemoryFromConfigMap(c, operatorNamespace, catalogSource.Spec.ConfigMap)
 	require.NoError(t, err)
 	require.NotNil(t, inMem)
 	latestEtcdCSV, err := inMem.FindCSVForPackageNameUnderChannel("etcd", "alpha")
@@ -166,10 +166,10 @@ func TestInstallPrometheusOCS(t *testing.T) {
 	c := newKubeClient(t)
 	crc := newCRClient(t)
 
-	catalogSource, err := fetchCatalogSource(t, crc, ocsConfigMap, testNamespace, catalogSourceSynced)
+	catalogSource, err := fetchCatalogSource(t, crc, ocsConfigMap, operatorNamespace, catalogSourceSynced)
 	require.NoError(t, err)
 	require.NotNil(t, catalogSource)
-	inMem, err := registry.NewInMemoryFromConfigMap(c, testNamespace, catalogSource.Spec.ConfigMap)
+	inMem, err := registry.NewInMemoryFromConfigMap(c, operatorNamespace, catalogSource.Spec.ConfigMap)
 	require.NoError(t, err)
 	require.NotNil(t, inMem)
 	latestPrometheusCSV, err := inMem.FindCSVForPackageNameUnderChannel("prometheus", "preview")