Skip to content

Commit

Permalink
feat(olm): generate clusterroles for provided apis in csv loop
Browse files Browse the repository at this point in the history
and aggregate them to operatorgroup clusterroles

this lets us speed up the tests for operatorgroups and in the future can
simplify installplan generation (can remove the clusterrole gen from
installplans)
  • Loading branch information
ecordell committed Dec 11, 2018
1 parent 85a6c8a commit bbc68a9
Show file tree
Hide file tree
Showing 6 changed files with 154 additions and 124 deletions.
2 changes: 0 additions & 2 deletions pkg/controller/operators/catalog/operator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import (
"testing"
"time"

"github.com/sirupsen/logrus"

"github.com/ghodss/yaml"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
Expand Down
6 changes: 6 additions & 0 deletions pkg/controller/operators/olm/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -460,10 +460,16 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error)
logger.WithError(err).Info("couldn't copy CSV to target namespaces")
}

// Ensure operator has access to targetnamespaces
if err := a.ensureRBACInTargetNamespace(updatedCSV, operatorGroup); err != nil {
logger.WithError(err).Info("couldn't ensure RBAC in target namespaces")
}

// Ensure cluster roles exist for using provided apis
if err := a.ensureClusterRolesForCSV(updatedCSV, operatorGroup); err != nil {
logger.WithError(err).Info("couldn't ensure clusterroles for provided api types")
}

return
}

Expand Down
192 changes: 117 additions & 75 deletions pkg/controller/operators/olm/operatorgroup.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,22 @@ const (
operatorGroupAnnotationKey = "olm.operatorGroup"
operatorGroupNamespaceAnnotationKey = "olm.operatorNamespace"
operatorGroupTargetsAnnotationKey = "olm.targetNamespaces"
operatorGroupAggregrationKeyPrefix = "olm.opgroup.permissions/aggregate-to-"
kubeRBACAggregationKeyPrefix = "rbac.authorization.k8s.io/aggregate-to-"
AdminSuffix = "admin"
EditSuffix = "edit"
ViewSuffix = "view"
)

var (
AdminVerbs = []string{"*"}
EditVerbs = []string{"create", "update", "patch", "delete"}
ViewVerbs = []string{"get", "list", "watch"}
VerbsForSuffix = map[string][]string{
AdminSuffix: AdminVerbs,
EditSuffix: EditVerbs,
ViewSuffix: ViewVerbs,
}
)

func (a *Operator) syncOperatorGroups(obj interface{}) error {
Expand All @@ -37,8 +53,8 @@ func (a *Operator) syncOperatorGroups(obj interface{}) error {
return err
}

if err := a.ensureClusterRoles(op); err != nil {
a.Log.Errorf("ensureClusterRoles error: %v", err)
if err := a.ensureOpGroupClusterRoles(op); err != nil {
a.Log.Errorf("ensureOpGroupClusterRoles error: %v", err)
return err
}
a.Log.Debug("Cluster roles completed")
Expand All @@ -59,6 +75,72 @@ func (a *Operator) syncOperatorGroups(obj interface{}) error {
return nil
}

// ensureProvidedAPIClusterRole ensures that a clusterrole exists (admin, edit, or view) for a single provided API Type
func (a *Operator) ensureProvidedAPIClusterRole(operatorGroup *v1alpha2.OperatorGroup, csv *v1alpha1.ClusterServiceVersion, namePrefix, suffix, group, resource string) error {
clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: namePrefix + suffix,
Labels: map[string]string{
kubeRBACAggregationKeyPrefix + suffix: "true",
operatorGroupAggregrationKeyPrefix + suffix: operatorGroup.GetName(),
},
},
Rules: []rbacv1.PolicyRule{{Verbs: VerbsForSuffix[suffix], APIGroups: []string{group}, Resources: []string{resource}}},
}
ownerutil.AddNonBlockingOwner(clusterRole, csv)
existingCR, err := a.OpClient.KubernetesInterface().RbacV1().ClusterRoles().Create(clusterRole)
if k8serrors.IsAlreadyExists(err) {
if existingCR != nil && reflect.DeepEqual(existingCR.Labels, clusterRole.Labels) && reflect.DeepEqual(existingCR.Rules, clusterRole.Rules) {
return nil
}
if _, err = a.OpClient.UpdateClusterRole(clusterRole); err != nil {
a.Log.WithError(err).Errorf("Update existing cluster role failed: %v", clusterRole)
return err
}
} else if err != nil {
a.Log.WithError(err).Errorf("Create cluster role failed: %v", clusterRole)
return err
}
return nil
}

// ensureClusterRolesForCSV ensures that ClusterRoles for writing and reading provided APIs exist for each operator
func (a *Operator) ensureClusterRolesForCSV(csv *v1alpha1.ClusterServiceVersion, operatorGroup *v1alpha2.OperatorGroup) error {
for _, owned := range csv.Spec.CustomResourceDefinitions.Owned {
nameGroupPair := strings.SplitN(owned.Name, ".", 2) // -> etcdclusters etcd.database.coreos.com
if len(nameGroupPair) != 2 {
return fmt.Errorf("Invalid parsing of name '%v', got %v", owned.Name, nameGroupPair)
}
plural := nameGroupPair[0]
group := nameGroupPair[1]
namePrefix := fmt.Sprintf("%s-%s-", owned.Name, owned.Version)

if err := a.ensureProvidedAPIClusterRole(operatorGroup, csv, namePrefix, AdminSuffix, group, plural); err != nil {
return err
}
if err := a.ensureProvidedAPIClusterRole(operatorGroup, csv, namePrefix, EditSuffix, group, plural); err != nil {
return err
}
if err := a.ensureProvidedAPIClusterRole(operatorGroup, csv, namePrefix, ViewSuffix, group, plural); err != nil {
return err
}
}
for _, owned := range csv.Spec.APIServiceDefinitions.Owned {
namePrefix := fmt.Sprintf("%s-%s-", owned.Name, owned.Version)

if err := a.ensureProvidedAPIClusterRole(operatorGroup, csv, namePrefix, AdminSuffix, owned.Group, owned.Name); err != nil {
return err
}
if err := a.ensureProvidedAPIClusterRole(operatorGroup, csv, namePrefix, EditSuffix, owned.Group, owned.Name); err != nil {
return err
}
if err := a.ensureProvidedAPIClusterRole(operatorGroup, csv, namePrefix, ViewSuffix, owned.Group, owned.Name); err != nil {
return err
}
}
return nil
}

func (a *Operator) ensureRBACInTargetNamespace(csv *v1alpha1.ClusterServiceVersion, operatorGroup *v1alpha2.OperatorGroup) error {
opPerms, err := resolver.RBACForClusterServiceVersion(csv)
if err != nil {
Expand Down Expand Up @@ -356,81 +438,41 @@ func (a *Operator) updateNamespaceList(op *v1alpha2.OperatorGroup) ([]string, er
return namespaceList, nil
}

func (a *Operator) ensureClusterRoles(op *v1alpha2.OperatorGroup) error {
currentNamespace := op.GetNamespace()
csvsInNamespace := a.csvSet(currentNamespace, v1alpha1.CSVPhaseSucceeded)
for _, csv := range csvsInNamespace {
managerPolicyRules := []rbacv1.PolicyRule{}
apiEditPolicyRules := []rbacv1.PolicyRule{}
apiViewPolicyRules := []rbacv1.PolicyRule{}
for _, owned := range csv.Spec.CustomResourceDefinitions.Owned {
nameGroupPair := strings.SplitN(owned.Name, ".", 2) // -> etcdclusters etcd.database.coreos.com
if len(nameGroupPair) != 2 {
return fmt.Errorf("Invalid parsing of name '%v', got %v", owned.Name, nameGroupPair)
}
managerPolicyRules = append(managerPolicyRules, rbacv1.PolicyRule{Verbs: []string{"*"}, APIGroups: []string{nameGroupPair[1]}, Resources: []string{nameGroupPair[0]}})
apiEditPolicyRules = append(apiEditPolicyRules, rbacv1.PolicyRule{Verbs: []string{"create", "update", "patch", "delete"}, APIGroups: []string{nameGroupPair[1]}, Resources: []string{nameGroupPair[0]}})
apiViewPolicyRules = append(apiViewPolicyRules, rbacv1.PolicyRule{Verbs: []string{"get", "list", "watch"}, APIGroups: []string{nameGroupPair[1]}, Resources: []string{nameGroupPair[0]}})
}
for _, owned := range csv.Spec.APIServiceDefinitions.Owned {
managerPolicyRules = append(managerPolicyRules, rbacv1.PolicyRule{Verbs: []string{"*"}, APIGroups: []string{owned.Group}, Resources: []string{owned.Name}})
apiEditPolicyRules = append(apiEditPolicyRules, rbacv1.PolicyRule{Verbs: []string{"create", "update", "patch", "delete"}, APIGroups: []string{owned.Group}, Resources: []string{owned.Name}})
apiViewPolicyRules = append(apiViewPolicyRules, rbacv1.PolicyRule{Verbs: []string{"get", "list", "watch"}, APIGroups: []string{owned.Group}, Resources: []string{owned.Name}})
}

clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("owned-crd-manager-%s", csv.GetName()),
func (a *Operator) ensureOpGroupClusterRole(op *v1alpha2.OperatorGroup, suffix string) error {
clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: strings.Join([]string{op.GetName(), suffix}, "-"),
},
AggregationRule: &rbacv1.AggregationRule{
ClusterRoleSelectors: []metav1.LabelSelector{
{
MatchLabels: map[string]string{
operatorGroupAggregrationKeyPrefix + suffix: op.GetName(),
},
},
},
Rules: managerPolicyRules,
}
ownerutil.AddNonBlockingOwner(clusterRole, csv)
_, err := a.OpClient.KubernetesInterface().RbacV1().ClusterRoles().Create(clusterRole)
if k8serrors.IsAlreadyExists(err) {
if _, err = a.OpClient.UpdateClusterRole(clusterRole); err != nil {
a.Log.Errorf("Update CRD existing cluster role failed: %v", err)
return err
}
} else if err != nil {
a.Log.Errorf("Update CRD cluster role failed: %v", err)
return err
}
},
}
ownerutil.AddNonBlockingOwner(clusterRole, op)
_, err := a.OpClient.KubernetesInterface().RbacV1().ClusterRoles().Create(clusterRole)
if k8serrors.IsAlreadyExists(err) {
return nil
} else if err != nil {
a.Log.WithError(err).Errorf("Create cluster role failed: %v", clusterRole)
return err
}
return nil
}

// operator group specific roles
operatorGroupEditClusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-edit", op.Name),
},
Rules: apiEditPolicyRules,
}
//ownerutil.AddNonBlockingOwner(operatorGroupEditClusterRole, csv)
_, err = a.OpClient.KubernetesInterface().RbacV1().ClusterRoles().Create(operatorGroupEditClusterRole)
if k8serrors.IsAlreadyExists(err) {
if _, err = a.OpClient.UpdateClusterRole(operatorGroupEditClusterRole); err != nil {
a.Log.Errorf("Update existing edit cluster role failed: %v", err)
return err
}
} else if err != nil {
a.Log.Errorf("Update edit cluster role failed: %v", err)
return err
}
operatorGroupViewClusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-view", op.GetName()),
},
Rules: apiViewPolicyRules,
}
//ownerutil.AddNonBlockingOwner(operatorGroupViewClusterRole, csv)
_, err = a.OpClient.KubernetesInterface().RbacV1().ClusterRoles().Create(operatorGroupViewClusterRole)
if k8serrors.IsAlreadyExists(err) {
if _, err = a.OpClient.UpdateClusterRole(operatorGroupViewClusterRole); err != nil {
a.Log.Errorf("Update existing view cluster role failed: %v", err)
return err
}
} else if err != nil {
a.Log.Errorf("Update view cluster role failed: %v", err)
return err
}
func (a *Operator) ensureOpGroupClusterRoles(op *v1alpha2.OperatorGroup) error {
if err := a.ensureOpGroupClusterRole(op, AdminSuffix); err != nil {
return err
}
if err := a.ensureOpGroupClusterRole(op, EditSuffix); err != nil {
return err
}
if err := a.ensureOpGroupClusterRole(op, ViewSuffix); err != nil {
return err
}
return nil
}
8 changes: 8 additions & 0 deletions pkg/lib/ownerutil/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"fmt"

"github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
"github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha2"

log "github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
Expand Down Expand Up @@ -253,6 +255,12 @@ func InferGroupVersionKind(obj runtime.Object) error {
Version: v1alpha1.GroupVersion,
Kind: v1alpha1.CatalogSourceKind,
})
case *v1alpha2.OperatorGroup:
objectKind.SetGroupVersionKind(schema.GroupVersionKind{
Group: v1alpha2.GroupName,
Version: v1alpha2.GroupVersion,
Kind: "OperatorGroup",
})
default:
return fmt.Errorf("could not infer GVK for object: %#v, %#v", obj, objectKind)
}
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/e2e-values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ package:
pullPolicy: IfNotPresent
service:
internalPort: 5443
commandArgs: -debug
commandArgs: --debug

e2e:
image:
Expand Down
68 changes: 22 additions & 46 deletions test/e2e/operator_groups_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -254,52 +254,6 @@ func TestOperatorGroup(t *testing.T) {
})
require.NoError(t, err)

t.Log("Waiting for expected operator group test APIGroup from CSV")
err = wait.Poll(pollInterval, pollDuration, func() (bool, error) {
// (view role is the last role created, so the rest should exist as well by this point)
viewRole, fetchErr := c.KubernetesInterface().RbacV1().ClusterRoles().Get(operatorGroup.Name+"-view", metav1.GetOptions{})
if fetchErr != nil {
if errors.IsNotFound(fetchErr) {
return false, nil
}
t.Logf("Unable to fetch view role: %v", fetchErr.Error())
return false, fetchErr
}
for _, rule := range viewRole.Rules {
for _, group := range rule.APIGroups {
if group == apiGroup {
return true, nil
}
}
}
return false, nil
})
require.NoError(t, err)

t.Log("Checking for proper generated operator group RBAC roles")
editRole, err := c.KubernetesInterface().RbacV1().ClusterRoles().Get(operatorGroup.Name+"-edit", metav1.GetOptions{})
require.NoError(t, err)
editPolicyRules := []rbacv1.PolicyRule{
{Verbs: []string{"create", "update", "patch", "delete"}, APIGroups: []string{apiGroup}, Resources: []string{mainCRDPlural}},
}
require.Equal(t, editPolicyRules, editRole.Rules)

viewRole, err := c.KubernetesInterface().RbacV1().ClusterRoles().Get(operatorGroup.Name+"-view", metav1.GetOptions{})
require.NoError(t, err)
viewPolicyRules := []rbacv1.PolicyRule{
{Verbs: []string{"get", "list", "watch"}, APIGroups: []string{apiGroup}, Resources: []string{mainCRDPlural}},
}
t.Log(viewRole)
require.Equal(t, viewPolicyRules, viewRole.Rules)

managerRole, err := c.KubernetesInterface().RbacV1().ClusterRoles().Get("owned-crd-manager-"+csvName, metav1.GetOptions{})
require.NoError(t, err)
managerPolicyRules := []rbacv1.PolicyRule{
{Verbs: []string{"*"}, APIGroups: []string{apiGroup}, Resources: []string{mainCRDPlural}},
}
t.Log(managerRole)
require.Equal(t, managerPolicyRules, managerRole.Rules)

t.Log("Waiting for operator namespace csv to have annotations")
err = wait.Poll(pollInterval, pollDuration, func() (bool, error) {
fetchedCSV, fetchErr := crc.OperatorsV1alpha1().ClusterServiceVersions(opGroupNamespace).Get(csvName, metav1.GetOptions{})
Expand Down Expand Up @@ -409,6 +363,28 @@ func TestOperatorGroup(t *testing.T) {
return true, nil
})

// validate provided API clusterroles for the operatorgroup
adminRole, err := c.KubernetesInterface().RbacV1().ClusterRoles().Get(operatorGroup.Name+"-admin", metav1.GetOptions{})
require.NoError(t, err)
adminPolicyRules := []rbacv1.PolicyRule{
{Verbs: []string{"*"}, APIGroups: []string{apiGroup}, Resources: []string{mainCRDPlural}},
}
require.Equal(t, adminPolicyRules, adminRole.Rules)

editRole, err := c.KubernetesInterface().RbacV1().ClusterRoles().Get(operatorGroup.Name+"-edit", metav1.GetOptions{})
require.NoError(t, err)
editPolicyRules := []rbacv1.PolicyRule{
{Verbs: []string{"create", "update", "patch", "delete"}, APIGroups: []string{apiGroup}, Resources: []string{mainCRDPlural}},
}
require.Equal(t, editPolicyRules, editRole.Rules)

viewRole, err := c.KubernetesInterface().RbacV1().ClusterRoles().Get(operatorGroup.Name+"-view", metav1.GetOptions{})
require.NoError(t, err)
viewPolicyRules := []rbacv1.PolicyRule{
{Verbs: []string{"get", "list", "watch"}, APIGroups: []string{apiGroup}, Resources: []string{mainCRDPlural}},
}
require.Equal(t, viewPolicyRules, viewRole.Rules)

// ensure deletion cleans up copied CSV
t.Log("Deleting CSV")
err = crc.OperatorsV1alpha1().ClusterServiceVersions(opGroupNamespace).Delete(csvName, &metav1.DeleteOptions{})
Expand Down

0 comments on commit bbc68a9

Please sign in to comment.