From 51b3345e36cf4c3ab0fed3bebe309b50f50d7020 Mon Sep 17 00:00:00 2001 From: Evan Cordell Date: Mon, 19 Aug 2019 18:26:47 -0400 Subject: [PATCH] test(e2e): add test verifying permissions are updated during an operator upgrade --- test/e2e/installplan_e2e_test.go | 378 +++++++++++++++++++++++++++++- test/e2e/subscription_e2e_test.go | 36 +-- 2 files changed, 398 insertions(+), 16 deletions(-) diff --git a/test/e2e/installplan_e2e_test.go b/test/e2e/installplan_e2e_test.go index 54930b04d9..e37827c1d7 100644 --- a/test/e2e/installplan_e2e_test.go +++ b/test/e2e/installplan_e2e_test.go @@ -10,6 +10,7 @@ import ( "github.com/blang/semver" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" @@ -961,7 +962,7 @@ func TestInstallPlanWithCRDSchemaChange(t *testing.T) { } } -func TestUpdateInstallPlan(t *testing.T) { +func TestUpdateCatalogForSubscription(t *testing.T) { defer cleaner.NotifyTestComplete(t, true) // crdVersionKey uniquely identifies a version within a CRD. @@ -971,6 +972,381 @@ func TestUpdateInstallPlan(t *testing.T) { storage bool } + t.Run("AmplifyPermissions", func(t *testing.T) { + defer cleaner.NotifyTestComplete(t, true) + + c := newKubeClient(t) + crc := newCRClient(t) + defer func() { + require.NoError(t, crc.OperatorsV1alpha1().Subscriptions(testNamespace).DeleteCollection(&metav1.DeleteOptions{}, metav1.ListOptions{})) + }() + + // Build initial catalog + mainPackageName := genName("nginx-amplify-") + mainPackageStable := fmt.Sprintf("%s-stable", mainPackageName) + stableChannel := "stable" + crdPlural := genName("ins-amplify-") + crdName := crdPlural + ".cluster.com" + mainCRD := apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: crdName, + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "cluster.com", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "v1alpha1", + Served: true, + Storage: true, + }, + }, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: crdPlural, + Singular: crdPlural, + Kind: crdPlural, + ListKind: "list" + crdPlural, + }, + Scope: "Namespaced", + }, + } + + // Generate permissions + serviceAccountName := genName("nginx-sa") + permissions := []install.StrategyDeploymentPermissions{ + { + ServiceAccountName: serviceAccountName, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"cluster.com"}, + Resources: []string{crdPlural}, + }, + }, + }, + } + // Generate permissions + clusterPermissions := []install.StrategyDeploymentPermissions{ + { + ServiceAccountName: serviceAccountName, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"cluster.com"}, + Resources: []string{crdPlural}, + }, + }, + }, + } + + // Create the catalog sources + mainNamedStrategy := newNginxInstallStrategy(genName("dep-"), permissions, clusterPermissions) + mainCSV := newCSV(mainPackageStable, testNamespace, "", semver.MustParse("0.1.0"), nil, nil, mainNamedStrategy) + mainCatalogName := genName("mock-ocs-amplify-") + mainManifests := []registry.PackageManifest{ + { + PackageName: mainPackageName, + Channels: []registry.PackageChannel{ + {Name: stableChannel, CurrentCSVName: mainCSV.GetName()}, + }, + DefaultChannelName: stableChannel, + }, + } + _, cleanupMainCatalogSource := createInternalCatalogSource(t, c, crc, mainCatalogName, testNamespace, mainManifests, []apiextensions.CustomResourceDefinition{mainCRD}, []v1alpha1.ClusterServiceVersion{mainCSV}) + defer cleanupMainCatalogSource() + // Attempt to get the catalog source before creating install plan + _, err := fetchCatalogSource(t, crc, mainCatalogName, testNamespace, catalogSourceRegistryPodSynced) + require.NoError(t, err) + + subscriptionName := genName("sub-nginx-update-perms1") + subscriptionCleanup := createSubscriptionForCatalog(t, crc, testNamespace, subscriptionName, mainCatalogName, mainPackageName, stableChannel, "", v1alpha1.ApprovalAutomatic) + defer subscriptionCleanup() + + subscription, err := fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionHasInstallPlanChecker) + require.NoError(t, err) + require.NotNil(t, subscription) + require.NotNil(t, subscription.Status.InstallPlanRef) + require.Equal(t, mainCSV.GetName(), subscription.Status.CurrentCSV) + + installPlanName := subscription.Status.InstallPlanRef.Name + + // Wait for InstallPlan to be status: Complete before checking resource presence + fetchedInstallPlan, err := fetchInstallPlan(t, crc, installPlanName, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete)) + require.NoError(t, err) + + require.Equal(t, v1alpha1.InstallPlanPhaseComplete, fetchedInstallPlan.Status.Phase) + + // Verify CSV is created + _, err = awaitCSV(t, crc, testNamespace, mainCSV.GetName(), csvSucceededChecker) + require.NoError(t, err) + + // Update CatalogSource with a new CSV with more permissions + updatedPermissions := []install.StrategyDeploymentPermissions{ + { + ServiceAccountName: serviceAccountName, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"cluster.com"}, + Resources: []string{crdPlural}, + }, + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"local.cluster.com"}, + Resources: []string{"locals"}, + }, + }, + }, + } + updatedClusterPermissions := []install.StrategyDeploymentPermissions{ + { + ServiceAccountName: serviceAccountName, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"cluster.com"}, + Resources: []string{crdPlural}, + }, + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"two.cluster.com"}, + Resources: []string{"twos"}, + }, + }, + }, + } + + // Create the catalog sources + updatedNamedStrategy := newNginxInstallStrategy(genName("dep-"), updatedPermissions, updatedClusterPermissions) + updatedCSV := newCSV(mainPackageStable+"-next", testNamespace, mainCSV.GetName(), semver.MustParse("0.2.0"), []apiextensions.CustomResourceDefinition{mainCRD}, nil, updatedNamedStrategy) + updatedManifests := []registry.PackageManifest{ + { + PackageName: mainPackageName, + Channels: []registry.PackageChannel{ + {Name: stableChannel, CurrentCSVName: updatedCSV.GetName()}, + }, + DefaultChannelName: stableChannel, + }, + } + // Update catalog with updated CSV with more permissions + updateInternalCatalog(t, c, crc, mainCatalogName, testNamespace, []apiextensions.CustomResourceDefinition{mainCRD}, []v1alpha1.ClusterServiceVersion{mainCSV, updatedCSV}, updatedManifests) + + _, err = fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionHasInstallPlanDifferentChecker(fetchedInstallPlan.GetName())) + require.NoError(t, err) + + updatedInstallPlanName := subscription.Status.InstallPlanRef.Name + + // Wait for InstallPlan to be status: Complete before checking resource presence + fetchedUpdatedInstallPlan, err := fetchInstallPlan(t, crc, updatedInstallPlanName, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete)) + require.NoError(t, err) + require.Equal(t, v1alpha1.InstallPlanPhaseComplete, fetchedUpdatedInstallPlan.Status.Phase) + + // Wait for csv to update + _, err = awaitCSV(t, crc, testNamespace, updatedCSV.GetName(), csvSucceededChecker) + require.NoError(t, err) + + // If the CSV is succeeded, we successfully rolled out the RBAC changes + }) + + t.Run("AttenuatePermissions", func(t *testing.T) { + defer cleaner.NotifyTestComplete(t, true) + + c := newKubeClient(t) + crc := newCRClient(t) + defer func() { + require.NoError(t, crc.OperatorsV1alpha1().Subscriptions(testNamespace).DeleteCollection(&metav1.DeleteOptions{}, metav1.ListOptions{})) + }() + + // Build initial catalog + mainPackageName := genName("nginx-attenuate-") + mainPackageStable := fmt.Sprintf("%s-stable", mainPackageName) + stableChannel := "stable" + crdPlural := genName("ins-attenuate-") + crdName := crdPlural + ".cluster.com" + mainCRD := apiextensions.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: crdName, + }, + Spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "cluster.com", + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "v1alpha1", + Served: true, + Storage: true, + }, + }, + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: crdPlural, + Singular: crdPlural, + Kind: crdPlural, + ListKind: "list" + crdPlural, + }, + Scope: "Namespaced", + }, + } + + // Generate permissions + serviceAccountName := genName("nginx-sa") + permissions := []install.StrategyDeploymentPermissions{ + { + ServiceAccountName: serviceAccountName, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"cluster.com"}, + Resources: []string{crdPlural}, + }, + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"local.cluster.com"}, + Resources: []string{"locals"}, + }, + }, + }, + } + // Generate permissions + clusterPermissions := []install.StrategyDeploymentPermissions{ + { + ServiceAccountName: serviceAccountName, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"cluster.com"}, + Resources: []string{crdPlural}, + }, + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"two.cluster.com"}, + Resources: []string{"twos"}, + }, + }, + }, + } + + // Create the catalog sources + mainNamedStrategy := newNginxInstallStrategy(genName("dep-"), permissions, clusterPermissions) + mainCSV := newCSV(mainPackageStable, testNamespace, "", semver.MustParse("0.1.0"), nil, nil, mainNamedStrategy) + mainCatalogName := genName("mock-ocs-main-update-perms1-") + mainManifests := []registry.PackageManifest{ + { + PackageName: mainPackageName, + Channels: []registry.PackageChannel{ + {Name: stableChannel, CurrentCSVName: mainCSV.GetName()}, + }, + DefaultChannelName: stableChannel, + }, + } + _, cleanupMainCatalogSource := createInternalCatalogSource(t, c, crc, mainCatalogName, testNamespace, mainManifests, []apiextensions.CustomResourceDefinition{mainCRD}, []v1alpha1.ClusterServiceVersion{mainCSV}) + defer cleanupMainCatalogSource() + // Attempt to get the catalog source before creating install plan + _, err := fetchCatalogSource(t, crc, mainCatalogName, testNamespace, catalogSourceRegistryPodSynced) + require.NoError(t, err) + + subscriptionName := genName("sub-nginx-update-perms1") + subscriptionCleanup := createSubscriptionForCatalog(t, crc, testNamespace, subscriptionName, mainCatalogName, mainPackageName, stableChannel, "", v1alpha1.ApprovalAutomatic) + defer subscriptionCleanup() + + subscription, err := fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionHasInstallPlanChecker) + require.NoError(t, err) + require.NotNil(t, subscription) + require.NotNil(t, subscription.Status.InstallPlanRef) + require.Equal(t, mainCSV.GetName(), subscription.Status.CurrentCSV) + + installPlanName := subscription.Status.InstallPlanRef.Name + + // Wait for InstallPlan to be status: Complete before checking resource presence + fetchedInstallPlan, err := fetchInstallPlan(t, crc, installPlanName, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete)) + require.NoError(t, err) + + require.Equal(t, v1alpha1.InstallPlanPhaseComplete, fetchedInstallPlan.Status.Phase) + + // Verify CSV is created + _, err = awaitCSV(t, crc, testNamespace, mainCSV.GetName(), csvSucceededChecker) + require.NoError(t, err) + + // Update CatalogSource with a new CSV with more permissions + updatedPermissions := []install.StrategyDeploymentPermissions{ + { + ServiceAccountName: serviceAccountName, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"local.cluster.com"}, + Resources: []string{"locals"}, + }, + }, + }, + } + updatedClusterPermissions := []install.StrategyDeploymentPermissions{ + { + ServiceAccountName: serviceAccountName, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{rbac.VerbAll}, + APIGroups: []string{"two.cluster.com"}, + Resources: []string{"twos"}, + }, + }, + }, + } + + // Create the catalog sources + updatedNamedStrategy := newNginxInstallStrategy(genName("dep-"), updatedPermissions, updatedClusterPermissions) + updatedCSV := newCSV(mainPackageStable+"-next", testNamespace, mainCSV.GetName(), semver.MustParse("0.2.0"), []apiextensions.CustomResourceDefinition{mainCRD}, nil, updatedNamedStrategy) + updatedManifests := []registry.PackageManifest{ + { + PackageName: mainPackageName, + Channels: []registry.PackageChannel{ + {Name: stableChannel, CurrentCSVName: updatedCSV.GetName()}, + }, + DefaultChannelName: stableChannel, + }, + } + // Update catalog with updated CSV with more permissions + updateInternalCatalog(t, c, crc, mainCatalogName, testNamespace, []apiextensions.CustomResourceDefinition{mainCRD}, []v1alpha1.ClusterServiceVersion{mainCSV, updatedCSV}, updatedManifests) + + // Wait for subscription to update its status + _, err = fetchSubscription(t, crc, testNamespace, subscriptionName, subscriptionHasInstallPlanDifferentChecker(fetchedInstallPlan.GetName())) + require.NoError(t, err) + + updatedInstallPlanName := subscription.Status.InstallPlanRef.Name + + // Wait for InstallPlan to be status: Complete before checking resource presence + fetchedUpdatedInstallPlan, err := fetchInstallPlan(t, crc, updatedInstallPlanName, buildInstallPlanPhaseCheckFunc(v1alpha1.InstallPlanPhaseComplete)) + require.NoError(t, err) + require.Equal(t, v1alpha1.InstallPlanPhaseComplete, fetchedUpdatedInstallPlan.Status.Phase) + + // Wait for csv to update + _, err = awaitCSV(t, crc, testNamespace, updatedCSV.GetName(), csvSucceededChecker) + require.NoError(t, err) + + // Wait for ServiceAccount to not have access anymore + err = wait.Poll(pollInterval, pollDuration, func() (bool, error) { + res, err := c.KubernetesInterface().AuthorizationV1().SubjectAccessReviews().Create(&authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + User: "system:serviceaccount:" + testNamespace + ":" + serviceAccountName, + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Group: "cluster.com", + Version: "v1alpha1", + Resource: crdPlural, + Verb: rbac.VerbAll, + }, + }, + }) + if err != nil { + return false, err + } + if res == nil { + return false, nil + } + t.Log("checking serviceaccount for permission") + + // should not be allowed + return !res.Status.Allowed, nil + }) + + }) + t.Run("UpdateSingleExistingCRDOwner", func(t *testing.T) { defer cleaner.NotifyTestComplete(t, true) diff --git a/test/e2e/subscription_e2e_test.go b/test/e2e/subscription_e2e_test.go index 6e92eee5e7..77e94b5ed9 100644 --- a/test/e2e/subscription_e2e_test.go +++ b/test/e2e/subscription_e2e_test.go @@ -10,15 +10,7 @@ import ( "github.com/blang/semver" "github.com/ghodss/yaml" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/tools/clientcmd" - configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" + configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1" "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned" "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install" @@ -27,6 +19,14 @@ import ( "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/comparison" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/version" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/clientcmd" ) var doubleInstance = int32(2) @@ -308,7 +308,13 @@ func subscriptionStateAtLatestChecker(subscription *v1alpha1.Subscription) bool } func subscriptionHasInstallPlanChecker(subscription *v1alpha1.Subscription) bool { - return subscription.Status.Install != nil + return subscription.Status.InstallPlanRef != nil +} + +func subscriptionHasInstallPlanDifferentChecker(currentInstallPlanName string) subscriptionStateChecker { + return func(subscription *v1alpha1.Subscription) bool { + return subscriptionHasInstallPlanChecker(subscription) && subscription.Status.InstallPlanRef.Name != currentInstallPlanName + } } func subscriptionStateNoneChecker(subscription *v1alpha1.Subscription) bool { @@ -1113,7 +1119,7 @@ func TestCreateNewSubscriptionWithPodConfig(t *testing.T) { newConfigClient := func(t *testing.T) configv1client.ConfigV1Interface { config, err := clientcmd.BuildConfigFromFlags("", *kubeConfigPath) - require.NoError(t, err,) + require.NoError(t, err) client, err := configv1client.NewForConfig(config) require.NoError(t, err) @@ -1134,21 +1140,21 @@ func TestCreateNewSubscriptionWithPodConfig(t *testing.T) { require.NotNil(t, proxy) proxyEnv := []corev1.EnvVar{} - if proxy.Status.HTTPProxy !="" { + if proxy.Status.HTTPProxy != "" { proxyEnv = append(proxyEnv, corev1.EnvVar{ Name: "HTTP_PROXY", Value: proxy.Status.HTTPProxy, }) } - if proxy.Status.HTTPSProxy !="" { + if proxy.Status.HTTPSProxy != "" { proxyEnv = append(proxyEnv, corev1.EnvVar{ Name: "HTTPS_PROXY", Value: proxy.Status.HTTPSProxy, }) } - if proxy.Status.NoProxy !="" { + if proxy.Status.NoProxy != "" { proxyEnv = append(proxyEnv, corev1.EnvVar{ Name: "NO_PROXY", Value: proxy.Status.NoProxy, @@ -1186,7 +1192,7 @@ func TestCreateNewSubscriptionWithPodConfig(t *testing.T) { subscriptionName := genName("podconfig-sub-") subSpec.Config = podConfig - cleanupSubscription := createSubscriptionForCatalogWithSpec(t, crClient, testNamespace, subscriptionName, subSpec) + cleanupSubscription := createSubscriptionForCatalogWithSpec(t, crClient, testNamespace, subscriptionName, subSpec) defer cleanupSubscription() subscription, err := fetchSubscription(t, crClient, testNamespace, subscriptionName, subscriptionStateAtLatestChecker)