From 597dbf7274e17c79224dec923aec594aa219d45d Mon Sep 17 00:00:00 2001 From: EaseWay Date: Fri, 12 Apr 2019 14:49:31 -0700 Subject: [PATCH] tenant-controller: add namespace/rbac sync on update of tenant (#12) --- poc/README.md | 8 + .../data/manifests/rbac.yaml | 2 +- .../pkg/controllers/tenants/controller.go | 197 ++++++++++++++---- 3 files changed, 160 insertions(+), 47 deletions(-) diff --git a/poc/README.md b/poc/README.md index e3c327d17..e6551519c 100644 --- a/poc/README.md +++ b/poc/README.md @@ -54,3 +54,11 @@ devtk build # build all binaries from all projects devtk build tenants-ctl # build the specified binary devtk build tenant-controller/tenants-ctl # build the specified binary within project explicitly specified ``` + +#### Pack Container Image + +``` +devtk pack # pack all container images from all projects +devtk pack tenants # pack the specified container image +devtk pack tenant-controller/tenants # pack the specified container image within project explicitly specified +``` diff --git a/poc/tenant-controller/data/manifests/rbac.yaml b/poc/tenant-controller/data/manifests/rbac.yaml index 1b1dc59f1..42d45ff8f 100644 --- a/poc/tenant-controller/data/manifests/rbac.yaml +++ b/poc/tenant-controller/data/manifests/rbac.yaml @@ -11,7 +11,7 @@ metadata: rules: - apiGroups: [""] resources: ["namespaces"] - verbs: ["get", "list", "watch", "create", "update", "patch"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"] verbs: ["create"] diff --git a/poc/tenant-controller/pkg/controllers/tenants/controller.go b/poc/tenant-controller/pkg/controllers/tenants/controller.go index ce862325c..a9f716fbd 100644 --- a/poc/tenant-controller/pkg/controllers/tenants/controller.go +++ b/poc/tenant-controller/pkg/controllers/tenants/controller.go @@ -14,8 +14,11 @@ package tenants import ( "context" "fmt" + "sort" + "strings" "github.com/golang/glog" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8srt "k8s.io/apimachinery/pkg/runtime" @@ -25,7 +28,6 @@ import ( tenantsapi "sigs.k8s.io/multi-tenancy/poc/tenant-controller/pkg/apis/tenants/v1alpha1" tenantsclient "sigs.k8s.io/multi-tenancy/poc/tenant-controller/pkg/clients/tenants/clientset/v1alpha1" tenantsinformers "sigs.k8s.io/multi-tenancy/poc/tenant-controller/pkg/clients/tenants/informers/externalversions" - corev1 "k8s.io/api/core/v1" ) // Controller is k8s controller managing Tenant CRDs. @@ -35,6 +37,11 @@ type Controller struct { k8sclient k8sclient.Interface } +const ( + defaultAdminRoleBindingName = "admins" + defaultAdminClusterRole = "admin" +) + // NewController creates the controller. func NewController(k8sclient k8sclient.Interface, tenantsclient tenantsclient.Interface, informerFactory tenantsinformers.SharedInformerFactory) *Controller { c := &Controller{ @@ -67,48 +74,158 @@ func (c *Controller) Run(ctx context.Context) error { } func (c *Controller) createTenant(obj *tenantsapi.Tenant) { - glog.Info("createTenant: %#v", obj) - syncRBACForTenantCRD(obj) + glog.V(2).Infof("createTenant: %#v", obj) + c.syncRBACForTenant(obj) + for _, nsReq := range obj.Spec.Namespaces { + c.createNamespaceForTenant(obj, &nsReq) + } + glog.V(2).Infof("createTenant completed: %#v", obj) +} - // TODO Add later ... sanity checks to ensure namespaces being requested are valid and not already assigned to another tenant +func (c *Controller) updateTenant(old, obj *tenantsapi.Tenant) { + glog.V(2).Infof("updateTenant: %#v", obj) + c.syncRBACForTenant(obj) + // sort namespaces in old and new tenants to find out which ones + // to be created and which ones to be deleted. + oldNsList := make([]string, len(old.Spec.Namespaces)) + for i, ns := range old.Spec.Namespaces { + oldNsList[i] = ns.Name + } + sort.Strings(oldNsList) + nsList := make([]*tenantsapi.TenantNamespace, len(obj.Spec.Namespaces)) + for i := range obj.Spec.Namespaces { + nsList[i] = &obj.Spec.Namespaces[i] + } + sort.Slice(nsList, func(i, j int) bool { + return strings.Compare(nsList[i].Name, nsList[j].Name) < 0 + }) + var i, j int + for i < len(oldNsList) && j < len(nsList) { + if res := strings.Compare(oldNsList[i], nsList[j].Name); res == 0 { + c.syncNamespaceForTenant(obj, nsList[j]) + i++ + j++ + } else if res < 0 { + c.deleteNamespaceForTenant(obj, oldNsList[i]) + i++ + } else { + c.createNamespaceForTenant(obj, nsList[j]) + j++ + } + } - namespace := corev1.Namespace{} + for ; j < len(nsList); j++ { + c.createNamespaceForTenant(obj, nsList[j]) + } + for ; i < len(oldNsList); i++ { + c.deleteNamespaceForTenant(obj, oldNsList[i]) + } +} + +func (c *Controller) deleteTenant(obj *tenantsapi.Tenant) { + glog.V(2).Infof("deleteTenant: %#v", obj) - for n := range obj.Spec.Namespaces { - namespace.ObjectMeta.Name = obj.Spec.Namespaces[n].Name - //Create namespace - c.k8sclient.CoreV1().Namespaces().Create(&namespace) + // TODO with OwnerReferences, no extra work is needed in deletion, + // remove the following code later. - glog.Info("Created namespace: %s", obj.Spec.Namespaces[n].Name) + c.deleteRBACForTenant(obj) + for _, nsReq := range obj.Spec.Namespaces { + c.deleteNamespaceForTenant(obj, nsReq.Name) } - - glog.Info("createTenant completed: %#v", obj) - // TODO create RBAC inside namespace + glog.Infof("deleteTenant completed: %#v", obj) } -func (c *Controller) updateTenant(old, obj *tenantsapi.Tenant) { - glog.Info("updateTenant: %#v", obj) - syncRBACForTenantCRD(obj) - // TODO sync namespace - // TODO sync RBAC inside namespace +func (c *Controller) createNamespaceForTenant(tenant *tenantsapi.Tenant, nsReq *tenantsapi.TenantNamespace) error { + if err := c.ensureNamespaceExists(tenant, nsReq.Name); err != nil { + return err + } + if err := c.syncNamespaceForTenant(tenant, nsReq); err != nil { + return err + } + glog.Infof("Tenant %q namespace %q created", tenant.Name, nsReq.Name) + return nil } -func (c *Controller) deleteTenant(obj *tenantsapi.Tenant) { - glog.Info("deleteTenant: %#v", obj) - deleteRBACForTenantCRD(obj) +func (c *Controller) deleteNamespaceForTenant(tenant *tenantsapi.Tenant, nsName string) error { // TODO add a full set of sanity checks in future before deleting + if err := c.k8sclient.CoreV1().Namespaces().Delete(nsName, nil); err != nil { + glog.Errorf("Tenant %q delete namespace %q error: %v", tenant.Name, nsName, err) + return err + } + glog.Infof("Tenant %q namespace %q deleted", tenant.Name, nsName) + return nil +} + +func (c *Controller) ensureNamespaceExists(tenant *tenantsapi.Tenant, nsName string) error { + // TODO Add later ... sanity checks to ensure namespaces being requested are valid and not already assigned to another tenant + if err := newKubeCtl().addObjects(&corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Namespace", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + OwnerReferences: ownerRefsForTenant(tenant), + }, + }).apply(); err != nil { + // TODO add error condition in Tenant object and emit event. + glog.Errorf("Tenant %q create namespace %q error: %v", tenant.Name, nsName, err) + return err + } + return nil +} - for n := range obj.Spec.Namespaces { - glog.Info("Deleting namespace: %s", obj.Spec.Namespaces[n].Name) - // Delete namespace - c.k8sclient.CoreV1().Namespaces().Delete(obj.Spec.Namespaces[n].Name, nil) +func (c *Controller) syncNamespaceForTenant(tenant *tenantsapi.Tenant, nsReq *tenantsapi.TenantNamespace) error { + if err := newKubeCtl().withNamespace(nsReq.Name).addObjects(&rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: defaultAdminRoleBindingName, + }, + Subjects: tenant.Spec.Admins, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: defaultAdminClusterRole, + }, + }).apply(); err != nil { + glog.Errorf("Tenant %q namespace %q sync error: %v", tenant.Name, nsReq.Name, err) + return err } + return nil +} - glog.Info("deleteTenant completed: %#v", obj) +func (c *Controller) syncRBACForTenant(tenant *tenantsapi.Tenant) error { + if err := newKubeCtl().addObjects(rbacForTenant(tenant)...).apply(); err != nil { + glog.Errorf("Tenant %q syncRBAC error: %v", tenant.Name, err) + return err + } + return nil } -func rbacForTenantCRD(obj *tenantsapi.Tenant) []k8srt.Object { - name := fmt.Sprintf("tenant-admins-%s", obj.Name) +func (c *Controller) deleteRBACForTenant(tenant *tenantsapi.Tenant) error { + if err := newKubeCtl().addObjects(rbacForTenant(tenant)...).delete(); err != nil { + glog.Errorf("Tenant %q deleteRBAC error: %v", tenant.Name, err) + return err + } + return nil +} + +func ownerRefsForTenant(tenant *tenantsapi.Tenant) []metav1.OwnerReference { + return []metav1.OwnerReference{ + { + APIVersion: tenantsapi.SchemeGroupVersion.String(), + Kind: "Tenant", + Name: tenant.Name, + UID: tenant.UID, + }, + } +} + +func rbacForTenant(tenant *tenantsapi.Tenant) []k8srt.Object { + name := fmt.Sprintf("tenant-admins-%s", tenant.Name) return []k8srt.Object{ &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{ @@ -116,14 +233,15 @@ func rbacForTenantCRD(obj *tenantsapi.Tenant) []k8srt.Object { Kind: "ClusterRole", }, ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + OwnerReferences: ownerRefsForTenant(tenant), }, Rules: []rbacv1.PolicyRule{ { Verbs: []string{"get", "update", "patch", "delete"}, APIGroups: []string{tenantsapi.SchemeGroupVersion.Group}, Resources: []string{"tenants"}, - ResourceNames: []string{obj.Name}, + ResourceNames: []string{tenant.Name}, }, }, }, &rbacv1.ClusterRoleBinding{ @@ -132,9 +250,10 @@ func rbacForTenantCRD(obj *tenantsapi.Tenant) []k8srt.Object { Kind: "ClusterRoleBinding", }, ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + OwnerReferences: ownerRefsForTenant(tenant), }, - Subjects: obj.Spec.Admins, + Subjects: tenant.Spec.Admins, RoleRef: rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, Kind: "ClusterRole", @@ -143,17 +262,3 @@ func rbacForTenantCRD(obj *tenantsapi.Tenant) []k8srt.Object { }, } } - -func syncRBACForTenantCRD(obj *tenantsapi.Tenant) { - if err := newKubeCtl().addObjects(rbacForTenantCRD(obj)...).apply(); err != nil { - glog.Errorf("syncRBACForTenantCRD error: %v", err) - // TODO retry logic. - } -} - -func deleteRBACForTenantCRD(obj *tenantsapi.Tenant) { - if err := newKubeCtl().addObjects(rbacForTenantCRD(obj)...).delete(); err != nil { - glog.Errorf("deleteRBACForTenantCRD error: %v", err) - // TODO retry logic. - } -}