diff --git a/incubator/virtualcluster/pkg/syncer/cluster/cluster.go b/incubator/virtualcluster/pkg/syncer/cluster/cluster.go index a2f413bdb..ca314b720 100644 --- a/incubator/virtualcluster/pkg/syncer/cluster/cluster.go +++ b/incubator/virtualcluster/pkg/syncer/cluster/cluster.go @@ -29,6 +29,7 @@ import ( restclient "k8s.io/client-go/rest" clientgocache "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -291,6 +292,7 @@ func (c *Cluster) Start() error { func (c *Cluster) WaitForCacheSync() bool { ca, err := c.getCache() if err != nil { + klog.Errorf("Fail to get cache: %v", err) return false } return ca.WaitForCacheSync(c.stopCh) diff --git a/incubator/virtualcluster/pkg/syncer/handler/enqueue_object.go b/incubator/virtualcluster/pkg/syncer/handler/enqueue_object.go index 5e08be9c0..60d3c430c 100644 --- a/incubator/virtualcluster/pkg/syncer/handler/enqueue_object.go +++ b/incubator/virtualcluster/pkg/syncer/handler/enqueue_object.go @@ -27,27 +27,29 @@ type EnqueueRequestForObject struct { Queue Queue } -func (e *EnqueueRequestForObject) enqueue(obj interface{}, event reconciler.EventType) { +func (e *EnqueueRequestForObject) enqueue(obj interface{}) { o, err := meta.Accessor(obj) if err != nil { return } - r := reconciler.Request{Cluster: e.Cluster, Event: event, Obj: obj} + r := reconciler.Request{} + r.ClusterName = e.Cluster.Name r.Namespace = o.GetNamespace() r.Name = o.GetName() + r.UID = string(o.GetUID()) e.Queue.Add(r) } func (e *EnqueueRequestForObject) OnAdd(obj interface{}) { - e.enqueue(obj, reconciler.AddEvent) + e.enqueue(obj) } func (e *EnqueueRequestForObject) OnUpdate(oldObj, newObj interface{}) { - e.enqueue(newObj, reconciler.UpdateEvent) + e.enqueue(newObj) } func (e *EnqueueRequestForObject) OnDelete(obj interface{}) { - e.enqueue(obj, reconciler.DeleteEvent) + e.enqueue(obj) } diff --git a/incubator/virtualcluster/pkg/syncer/mccontroller/mccontroller.go b/incubator/virtualcluster/pkg/syncer/mccontroller/mccontroller.go index e1d80d8bd..20e69549b 100644 --- a/incubator/virtualcluster/pkg/syncer/mccontroller/mccontroller.go +++ b/incubator/virtualcluster/pkg/syncer/mccontroller/mccontroller.go @@ -321,9 +321,12 @@ func (c *MultiClusterController) RequeueObject(clusterName string, obj interface if cluster == nil { return fmt.Errorf("could not find cluster %s", clusterName) } - r := reconciler.Request{Cluster: cluster.GetClientInfo(), Event: event, Obj: obj} + //FIXME: we dont need event here. + r := reconciler.Request{} + r.ClusterName = clusterName r.Namespace = o.GetNamespace() r.Name = o.GetName() + r.UID = string(o.GetUID()) c.Queue.Add(r) return nil @@ -369,9 +372,9 @@ func (c *MultiClusterController) processNextWorkItem() bool { // Return true, don't take a break return true } - if c.getCluster(req.Cluster.Name) == nil { + if c.getCluster(req.ClusterName) == nil { // The virtual cluster has been removed, do not reconcile for its dws requests. - klog.Warningf("The cluster %s has been removed, drop the dws request %v", req.Cluster.Name, req) + klog.Warningf("The cluster %s has been removed, drop the dws request %v", req.ClusterName, req) c.Queue.Forget(obj) return true } diff --git a/incubator/virtualcluster/pkg/syncer/reconciler/reconciler.go b/incubator/virtualcluster/pkg/syncer/reconciler/reconciler.go index b9f003498..7660ae339 100644 --- a/incubator/virtualcluster/pkg/syncer/reconciler/reconciler.go +++ b/incubator/virtualcluster/pkg/syncer/reconciler/reconciler.go @@ -45,17 +45,13 @@ const ( ) // Request contains the information needed by a multicluster Reconciler to Reconcile: -// a context, namespace, and name. +// It ONLY contains the meta that can uniquely identify an object without any state information which can lead to parallel reconcile. type Request struct { - Cluster *ClusterInfo + ClusterName string types.NamespacedName - Event EventType - Obj interface{} + UID string } -// Result is the return type of a Reconciler's Reconcile method. -// By default, the Request is forgotten after it's been processed, -// but you can also requeue it immediately, or after some time. type Result reconcile.Result // Reconciler is the interface used by a Controller to reconcile. diff --git a/incubator/virtualcluster/pkg/syncer/resources/configmap/dws.go b/incubator/virtualcluster/pkg/syncer/resources/configmap/dws.go index 523f556bc..3dac785d8 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/configmap/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/configmap/dws.go @@ -39,62 +39,68 @@ func (c *controller) StartDWS(stopCh <-chan struct{}) error { // The reconcile logic for tenant master configMap informer func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) { - klog.V(4).Infof("reconcile configmap %s/%s %s event for cluster %s", request.Namespace, request.Name, request.Event, request.Cluster.Name) + klog.V(4).Infof("reconcile configmap %s/%s event for cluster %s", request.Namespace, request.Name, request.ClusterName) - switch request.Event { - case reconciler.AddEvent: - err := c.reconcileConfigMapCreate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ConfigMap)) + targetNamespace := conversion.ToSuperMasterNamespace(request.ClusterName, request.Namespace) + pConfigMap, err := c.configMapLister.ConfigMaps(targetNamespace).Get(request.Name) + pExists := true + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + pExists = false + } + vExists := true + vConfigMapObj, err := c.multiClusterConfigMapController.Get(request.ClusterName, request.Namespace, request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false + } + + if vExists && !pExists { + vConfigMap := vConfigMapObj.(*v1.ConfigMap) + err := c.reconcileConfigMapCreate(request.ClusterName, targetNamespace, vConfigMap) if err != nil { - klog.Errorf("failed reconcile configmap %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) - return reconciler.Result{Requeue: true}, nil + klog.Errorf("failed reconcile configmap %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) + return reconciler.Result{Requeue: true}, err } - case reconciler.UpdateEvent: - err := c.reconcileConfigMapUpdate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ConfigMap)) + } else if !vExists && pExists { + err := c.reconcileConfigMapRemove(request.ClusterName, targetNamespace, request.Name) if err != nil { - klog.Errorf("failed reconcile configmap %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile configmap %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.DeleteEvent: - err := c.reconcileConfigMapRemove(request.Cluster.Name, request.Namespace, request.Name) + } else if vExists && pExists { + vConfigMap := vConfigMapObj.(*v1.ConfigMap) + err := c.reconcileConfigMapUpdate(request.ClusterName, targetNamespace, pConfigMap, vConfigMap) if err != nil { - klog.Errorf("failed reconcile configmap %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile configmap %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } + } else { + // object is gone. } return reconciler.Result{}, nil } -func (c *controller) reconcileConfigMapCreate(cluster, namespace, name string, configMap *v1.ConfigMap) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - _, err := c.configMapLister.ConfigMaps(targetNamespace).Get(name) - if err == nil { - return c.reconcileConfigMapUpdate(cluster, namespace, name, configMap) - } - - newObj, err := conversion.BuildMetadata(cluster, targetNamespace, configMap) +func (c *controller) reconcileConfigMapCreate(clusterName, targetNamespace string, configMap *v1.ConfigMap) error { + newObj, err := conversion.BuildMetadata(clusterName, targetNamespace, configMap) if err != nil { return err } _, err = c.configMapClient.ConfigMaps(targetNamespace).Create(newObj.(*v1.ConfigMap)) if errors.IsAlreadyExists(err) { - klog.Infof("configmap %s/%s of cluster %s already exist in super master", namespace, name, cluster) + klog.Infof("configmap %s/%s of cluster %s already exist in super master", targetNamespace, configMap.Name, clusterName) return nil } return err } -func (c *controller) reconcileConfigMapUpdate(cluster, namespace, name string, vConfigMap *v1.ConfigMap) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - pConfigMap, err := c.configMapLister.ConfigMaps(targetNamespace).Get(name) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - return err - } - - spec, err := c.multiClusterConfigMapController.GetSpec(cluster) +func (c *controller) reconcileConfigMapUpdate(clusterName, targetNamespace string, pConfigMap, vConfigMap *v1.ConfigMap) error { + spec, err := c.multiClusterConfigMapController.GetSpec(clusterName) if err != nil { return err } @@ -105,18 +111,16 @@ func (c *controller) reconcileConfigMapUpdate(cluster, namespace, name string, v return err } } - return nil } -func (c *controller) reconcileConfigMapRemove(cluster, namespace, name string) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcileConfigMapRemove(clusterName, targetNamespace, name string) error { opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, } err := c.configMapClient.ConfigMaps(targetNamespace).Delete(name, opts) if errors.IsNotFound(err) { - klog.Warningf("configmap %s/%s of cluster %s not found in super master", namespace, name, cluster) + klog.Warningf("configmap %s/%s of cluster %s not found in super master", targetNamespace, name, clusterName) return nil } return err diff --git a/incubator/virtualcluster/pkg/syncer/resources/endpoints/dws.go b/incubator/virtualcluster/pkg/syncer/resources/endpoints/dws.go index 413a7e31f..b70b2b460 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/endpoints/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/endpoints/dws.go @@ -43,39 +43,53 @@ func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, e // For now, we bypass all ep events beside the default kubernetes ep. The tenant/master ep controllers handle ep lifecycle independently. return reconciler.Result{}, nil } - klog.Infof("reconcile endpoints %s/%s %s event for cluster %s", request.Namespace, request.Name, request.Event, request.Cluster.Name) + klog.Infof("reconcile endpoints %s/%s for cluster %s", request.Namespace, request.Name, request.ClusterName) + targetNamespace := conversion.ToSuperMasterNamespace(request.ClusterName, request.Namespace) + pEndpoints, err := c.endpointsLister.Endpoints(targetNamespace).Get(request.Name) + pExists := true + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + pExists = false + } + vExists := true + vEndpointsObj, err := c.multiClusterEndpointsController.Get(request.ClusterName, request.Namespace, request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false + } - switch request.Event { - case reconciler.AddEvent: - err := c.reconcileEndpointsCreate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Endpoints)) + if vExists && !pExists { + vEndpoints := vEndpointsObj.(*v1.Endpoints) + err := c.reconcileEndpointsCreate(request.ClusterName, targetNamespace, vEndpoints) if err != nil { - klog.Errorf("failed reconcile endpoints %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile endpoints %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.UpdateEvent: - err := c.reconcileEndpointsUpdate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Endpoints)) + } else if !vExists && pExists { + err := c.reconcileEndpointsRemove(request.ClusterName, targetNamespace, request.Name) if err != nil { - klog.Errorf("failed reconcile endpoints %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile endpoints %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.DeleteEvent: - err := c.reconcileEndpointsRemove(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Endpoints)) + } else if vExists && pExists { + vEndpoints := vEndpointsObj.(*v1.Endpoints) + err := c.reconcileEndpointsUpdate(request.ClusterName, targetNamespace, pEndpoints, vEndpoints) if err != nil { - klog.Errorf("failed reconcile endpoints %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile endpoints %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } + } else { + // object is gone. } return reconciler.Result{}, nil } -func (c *controller) reconcileEndpointsCreate(cluster, namespace, name string, ep *v1.Endpoints) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - _, err := c.endpointsLister.Endpoints(targetNamespace).Get(name) - if err == nil { - return c.reconcileEndpointsUpdate(cluster, namespace, name, ep) - } - - newObj, err := conversion.BuildMetadata(cluster, targetNamespace, ep) +func (c *controller) reconcileEndpointsCreate(clusterName, targetNamespace string, ep *v1.Endpoints) error { + newObj, err := conversion.BuildMetadata(clusterName, targetNamespace, ep) if err != nil { return err } @@ -84,23 +98,14 @@ func (c *controller) reconcileEndpointsCreate(cluster, namespace, name string, e _, err = c.endpointClient.Endpoints(targetNamespace).Create(pEndpoints) if errors.IsAlreadyExists(err) { - klog.Infof("endpoints %s/%s of cluster %s already exist in super master", namespace, name, cluster) + klog.Infof("endpoints %s/%s of cluster %s already exist in super master", targetNamespace, pEndpoints.Name, clusterName) return nil } return err } -func (c *controller) reconcileEndpointsUpdate(cluster, namespace, name string, vEP *v1.Endpoints) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - pEP, err := c.endpointsLister.Endpoints(targetNamespace).Get(name) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - return err - } - - spec, err := c.multiClusterEndpointsController.GetSpec(cluster) +func (c *controller) reconcileEndpointsUpdate(clusterName, targetNamespace string, pEP, vEP *v1.Endpoints) error { + spec, err := c.multiClusterEndpointsController.GetSpec(clusterName) if err != nil { return err } @@ -111,18 +116,16 @@ func (c *controller) reconcileEndpointsUpdate(cluster, namespace, name string, v return err } } - return nil } -func (c *controller) reconcileEndpointsRemove(cluster, namespace, name string, ep *v1.Endpoints) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcileEndpointsRemove(clusterName, targetNamespace, name string) error { opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, } err := c.endpointClient.Endpoints(targetNamespace).Delete(name, opts) if errors.IsNotFound(err) { - klog.Warningf("endpoints %s/%s of cluster not found in super master", namespace, name) + klog.Warningf("endpoints %s/%s of %s cluster not found in super master", targetNamespace, name, clusterName) return nil } return err diff --git a/incubator/virtualcluster/pkg/syncer/resources/namespace/checker.go b/incubator/virtualcluster/pkg/syncer/resources/namespace/checker.go index f7ce14dcc..2d02d918b 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/namespace/checker.go +++ b/incubator/virtualcluster/pkg/syncer/resources/namespace/checker.go @@ -82,22 +82,29 @@ func (c *controller) checkNamespaces() { if len(clusterName) == 0 || len(vNamespace) == 0 { continue } - - _, err := c.multiClusterNamespaceController.Get(clusterName, "", vNamespace) - if err != nil { - if errors.IsNotFound(err) { - // vNamespace not found and pNamespace still exist, we need to delete pNamespace manually - opts := &metav1.DeleteOptions{ - PropagationPolicy: &constants.DefaultDeletionPolicy, - } - if err := c.namespaceClient.Namespaces().Delete(pNamespace.Name, opts); err != nil { - klog.Errorf("error deleting pNamespace %s in super master: %v", pNamespace.Name, err) - } else { - metrics.CheckerRemedyStats.WithLabelValues("numDeletedOrphanSuperMasterNamespaces").Inc() - } - continue + shouldDelete := false + vNamespaceObj, err := c.multiClusterNamespaceController.Get(clusterName, "", vNamespace) + if errors.IsNotFound(err) { + shouldDelete = true + } + if err == nil { + vNs := vNamespaceObj.(*v1.Namespace) + if pNamespace.Annotations[constants.LabelUID] != string(vNs.UID) { + shouldDelete = true + klog.Warningf("Found pNamespace %s delegated UID is different from tenant object.", pNamespace.Name) + } + } + if shouldDelete { + // vNamespace not found and pNamespace still exist, we need to delete pNamespace manually + opts := &metav1.DeleteOptions{ + PropagationPolicy: &constants.DefaultDeletionPolicy, + Preconditions: metav1.NewUIDPreconditions(string(pNamespace.UID)), + } + if err := c.namespaceClient.Namespaces().Delete(pNamespace.Name, opts); err != nil { + klog.Errorf("error deleting pNamespace %s in super master: %v", pNamespace.Name, err) + } else { + metrics.CheckerRemedyStats.WithLabelValues("numDeletedOrphanSuperMasterNamespaces").Inc() } - klog.Errorf("error getting vNamespace %s from cluster %s cache: %v", vNamespace, clusterName, err) } } } @@ -113,7 +120,7 @@ func (c *controller) checkNamespacesOfTenantCluster(clusterName string) { namespaceList := listObj.(*v1.NamespaceList) for i, vNamespace := range namespaceList.Items { targetNamespace := conversion.ToSuperMasterNamespace(clusterName, vNamespace.Name) - _, err := c.nsLister.Get(targetNamespace) + pNamespace, err := c.nsLister.Get(targetNamespace) if errors.IsNotFound(err) { // pNamespace not found and vNamespace still exists, we need to create pNamespace again if err := c.multiClusterNamespaceController.RequeueObject(clusterName, &namespaceList.Items[i], reconciler.AddEvent); err != nil { @@ -127,5 +134,9 @@ func (c *controller) checkNamespacesOfTenantCluster(clusterName string) { if err != nil { klog.Errorf("error getting pNamespace %s from super master cache: %v", targetNamespace, err) } + + if pNamespace.Annotations[constants.LabelUID] != string(vNamespace.UID) { + klog.Errorf("Found pNamespace %s delegated UID is different from tenant object.", targetNamespace) + } } } diff --git a/incubator/virtualcluster/pkg/syncer/resources/namespace/dws.go b/incubator/virtualcluster/pkg/syncer/resources/namespace/dws.go index c5340732d..891401225 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/namespace/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/namespace/dws.go @@ -18,7 +18,6 @@ package namespace import ( "fmt" - "strings" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -40,64 +39,85 @@ func (c *controller) StartDWS(stopCh <-chan struct{}) error { // The reconcile logic for tenant master namespace informer func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) { - klog.V(4).Infof("reconcile namespace %s %s event for cluster %s", request.Name, request.Event, request.Cluster.Name) + klog.V(4).Infof("reconcile namespace %s for cluster %s", request.Name, request.ClusterName) + targetNamespace := conversion.ToSuperMasterNamespace(request.ClusterName, request.Name) + pNamespace, err := c.nsLister.Get(targetNamespace) + pExists := true + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + pExists = false + } + vExists := true + vNamespaceObj, err := c.multiClusterNamespaceController.Get(request.ClusterName, request.Namespace, request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false + } - switch request.Event { - case reconciler.AddEvent: - err := c.reconcileNamespaceCreate(request.Cluster.Name, request.Name, request.Obj.(*v1.Namespace)) + if vExists && !pExists { + vNamespace := vNamespaceObj.(*v1.Namespace) + err := c.reconcileNamespaceCreate(request.ClusterName, targetNamespace, request.UID, vNamespace) if err != nil { - klog.Errorf("failed reconcile namespace %s CREATE of cluster %s %v", request.Name, request.Cluster.Name, err) - return reconciler.Result{Requeue: true}, nil + klog.Errorf("failed reconcile namespace %s CREATE of cluster %s %v", request.Name, request.ClusterName, err) + return reconciler.Result{Requeue: true}, err } - case reconciler.UpdateEvent: - err := c.reconcileNamespaceUpdate(request.Cluster.Name, request.Name, request.Obj.(*v1.Namespace)) + } else if !vExists && pExists { + err := c.reconcileNamespaceRemove(request.ClusterName, targetNamespace, request.UID, pNamespace) if err != nil { - klog.Errorf("failed reconcile namespace %s UPDATE of cluster %s %v", request.Name, request.Cluster.Name, err) - return reconciler.Result{}, err + klog.Errorf("failed reconcile namespace %s DELETE of cluster %s %v", request.Name, request.ClusterName, err) + return reconciler.Result{Requeue: true}, err } - case reconciler.DeleteEvent: - err := c.reconcileNamespaceRemove(request.Cluster.Name, request.Name) + } else if vExists && pExists { + vNamespace := vNamespaceObj.(*v1.Namespace) + err := c.reconcileNamespaceUpdate(request.ClusterName, targetNamespace, request.UID, pNamespace, vNamespace) if err != nil { - klog.Errorf("failed reconcile namespace %s DELETE of cluster %s %v", request.Name, request.Cluster.Name, err) - return reconciler.Result{}, err + klog.Errorf("failed reconcile namespace %s UPDATE of cluster %s %v", request.Name, request.ClusterName, err) + return reconciler.Result{Requeue: true}, err } + } else { + // object is gone. } return reconciler.Result{}, nil } -func (c *controller) reconcileNamespaceCreate(cluster, name string, namespace *v1.Namespace) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, name) - _, err := c.nsLister.Get(targetNamespace) - if err == nil { - // namespace is ready. - return nil - } - - newObj, err := conversion.BuildSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcileNamespaceCreate(clusterName, targetNamespace, requestUID string, vNamespace *v1.Namespace) error { + newObj, err := conversion.BuildSuperMasterNamespace(clusterName, vNamespace) if err != nil { return err } - _, err = c.namespaceClient.Namespaces().Create(newObj.(*v1.Namespace)) + pNamespace, err := c.namespaceClient.Namespaces().Create(newObj.(*v1.Namespace)) if errors.IsAlreadyExists(err) { - klog.Infof("namespace %s of cluster %s already exist in super master", name, cluster) - return nil + if pNamespace.Annotations[constants.LabelUID] == requestUID { + klog.Infof("namespace %s of cluster %s already exist in super master", targetNamespace, clusterName) + return nil + } else { + return fmt.Errorf("pNamespace %s exists but its delegated object UID is different.", targetNamespace) + } } return err } -func (c *controller) reconcileNamespaceUpdate(cluster, name string, namespace *v1.Namespace) error { +func (c *controller) reconcileNamespaceUpdate(clusterName, targetNamespace, requestUID string, pNamespace, vNamespace *v1.Namespace) error { return nil } -func (c *controller) reconcileNamespaceRemove(cluster, name string) error { - targetName := strings.Join([]string{cluster, name}, "-") +func (c *controller) reconcileNamespaceRemove(clusterName, targetNamespace, requestUID string, pNamespace *v1.Namespace) error { + if pNamespace.Annotations[constants.LabelUID] != requestUID { + return fmt.Errorf("To be deleted pNamespace %s delegated UID is different from deleted object.", targetNamespace) + } + opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, + Preconditions: metav1.NewUIDPreconditions(string(pNamespace.UID)), } - err := c.namespaceClient.Namespaces().Delete(targetName, opts) + err := c.namespaceClient.Namespaces().Delete(targetNamespace, opts) if errors.IsNotFound(err) { - klog.Warningf("namespace %s of cluster %s not found in super master", name, cluster) + klog.Warningf("namespace %s of cluster %s not found in super master", targetNamespace, clusterName) return nil } return err diff --git a/incubator/virtualcluster/pkg/syncer/resources/node/dws.go b/incubator/virtualcluster/pkg/syncer/resources/node/dws.go index 34549e5eb..fcf100412 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/node/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/node/dws.go @@ -18,6 +18,7 @@ package node import ( v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog" "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/constants" @@ -31,49 +32,35 @@ func (c *controller) StartDWS(stopCh <-chan struct{}) error { // The reconcile logic for tenant master node informer, the main purpose is to maintain // the nodeNameToCluster mapping func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) { - klog.V(4).Infof("reconcile node %s %s event for cluster %s", request.Name, request.Event, request.Cluster.Name) - vNode := request.Obj.(*v1.Node) - if vNode.Labels[constants.LabelVirtualNode] != "true" { - // We only handle virtual nodes created by syncer - return reconciler.Result{}, nil + klog.V(4).Infof("reconcile node %s for cluster %s", request.Name, request.ClusterName) + vExists := true + vNodeObj, err := c.multiClusterNodeController.Get(request.ClusterName, request.Namespace, request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false } - switch request.Event { - case reconciler.AddEvent: - c.reconcileCreate(request.Cluster.Name, request.Namespace, request.Name, vNode) - case reconciler.UpdateEvent: - c.reconcileUpdate(request.Cluster.Name, request.Namespace, request.Name, vNode) - case reconciler.DeleteEvent: - c.reconcileRemove(request.Cluster.Name, request.Namespace, request.Name, vNode) - } - return reconciler.Result{}, nil -} - -func (c *controller) reconcileCreate(cluster, namespace, name string, node *v1.Node) { - c.Lock() - defer c.Unlock() + if vExists { + vNode := vNodeObj.(*v1.Node) + if vNode.Labels[constants.LabelVirtualNode] != "true" { + // We only handle virtual nodes created by syncer + return reconciler.Result{}, nil + } + c.Lock() + if _, exist := c.nodeNameToCluster[request.Name]; !exist { + c.nodeNameToCluster[request.Name] = make(map[string]struct{}) + } + c.nodeNameToCluster[request.Name][request.ClusterName] = struct{}{} + c.Unlock() + } else { + c.Lock() + if _, exists := c.nodeNameToCluster[request.Name]; exists { + delete(c.nodeNameToCluster[request.Name], request.ClusterName) + } + c.Unlock() - if _, exist := c.nodeNameToCluster[name]; !exist { - c.nodeNameToCluster[name] = make(map[string]struct{}) - } - c.nodeNameToCluster[name][cluster] = struct{}{} -} - -func (c *controller) reconcileUpdate(cluster, namespace, name string, node *v1.Node) { - c.Lock() - defer c.Unlock() - - if _, exist := c.nodeNameToCluster[name]; !exist { - c.nodeNameToCluster[name] = make(map[string]struct{}) - } - c.nodeNameToCluster[name][cluster] = struct{}{} -} - -func (c *controller) reconcileRemove(cluster, namespace, name string, node *v1.Node) { - c.Lock() - defer c.Unlock() - - if _, exists := c.nodeNameToCluster[name]; exists { - delete(c.nodeNameToCluster[name], cluster) } + return reconciler.Result{}, nil } diff --git a/incubator/virtualcluster/pkg/syncer/resources/persistentvolumeclaim/dws.go b/incubator/virtualcluster/pkg/syncer/resources/persistentvolumeclaim/dws.go index 680e2afa6..360892e0d 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/persistentvolumeclaim/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/persistentvolumeclaim/dws.go @@ -39,39 +39,54 @@ func (c *controller) StartDWS(stopCh <-chan struct{}) error { // The reconcile logic for tenant master pvc informer func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) { - klog.V(4).Infof("reconcile pvc %s %s event for cluster %s", request.Name, request.Event, request.Cluster.Name) + klog.V(4).Infof("reconcile pvc %s/%s event for cluster %s", request.Namespace, request.Name, request.ClusterName) - switch request.Event { - case reconciler.AddEvent: - err := c.reconcilePVCCreate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.PersistentVolumeClaim)) + targetNamespace := conversion.ToSuperMasterNamespace(request.ClusterName, request.Namespace) + pPVC, err := c.pvcLister.PersistentVolumeClaims(targetNamespace).Get(request.Name) + pExists := true + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + pExists = false + } + vExists := true + vPVCObj, err := c.multiClusterPersistentVolumeClaimController.Get(request.ClusterName, request.Namespace, request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false + } + + if vExists && !pExists { + vPVC := vPVCObj.(*v1.PersistentVolumeClaim) + err := c.reconcilePVCCreate(request.ClusterName, targetNamespace, vPVC) if err != nil { - klog.Errorf("failed reconcile pvc %s CREATE of cluster %s %v", request.Name, request.Cluster.Name, err) - return reconciler.Result{Requeue: true}, nil + klog.Errorf("failed reconcile pvc %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) + return reconciler.Result{Requeue: true}, err } - case reconciler.UpdateEvent: - err := c.reconcilePVCUpdate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.PersistentVolumeClaim)) + } else if !vExists && pExists { + err := c.reconcilePVCRemove(request.ClusterName, targetNamespace, request.Name) if err != nil { - klog.Errorf("failed reconcile pvc %s UPDATE of cluster %s %v", request.Name, request.Cluster.Name, err) - return reconciler.Result{}, err + klog.Errorf("failed reconcile pvc %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) + return reconciler.Result{Requeue: true}, err } - case reconciler.DeleteEvent: - err := c.reconcilePVCRemove(request.Cluster.Name, request.Namespace, request.Name) + } else if vExists && pExists { + vPVC := vPVCObj.(*v1.PersistentVolumeClaim) + err := c.reconcilePVCUpdate(request.ClusterName, targetNamespace, pPVC, vPVC) if err != nil { - klog.Errorf("failed reconcile pvc %s DELETE of cluster %s %v", request.Name, request.Cluster.Name, err) - return reconciler.Result{}, err + klog.Errorf("failed reconcile pvc %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) + return reconciler.Result{Requeue: true}, err } + } else { + // object is gone. } return reconciler.Result{}, nil } -func (c *controller) reconcilePVCCreate(cluster, namespace, name string, pvc *v1.PersistentVolumeClaim) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - _, err := c.pvcLister.PersistentVolumeClaims(targetNamespace).Get(name) - if err == nil { - return c.reconcilePVCUpdate(cluster, namespace, name, pvc) - } - - newObj, err := conversion.BuildMetadata(cluster, targetNamespace, pvc) +func (c *controller) reconcilePVCCreate(clusterName, targetNamespace string, pvc *v1.PersistentVolumeClaim) error { + newObj, err := conversion.BuildMetadata(clusterName, targetNamespace, pvc) if err != nil { return err } @@ -80,23 +95,14 @@ func (c *controller) reconcilePVCCreate(cluster, namespace, name string, pvc *v1 _, err = c.pvcClient.PersistentVolumeClaims(targetNamespace).Create(pPVC) if errors.IsAlreadyExists(err) { - klog.Infof("pvc %s/%s of cluster %s already exist in super master", namespace, name, cluster) + klog.Infof("pvc %s/%s of cluster %s already exist in super master", targetNamespace, pPVC.Name, clusterName) return nil } return err } -func (c *controller) reconcilePVCUpdate(cluster, namespace, name string, vPVC *v1.PersistentVolumeClaim) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - pPVC, err := c.pvcLister.PersistentVolumeClaims(targetNamespace).Get(name) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - return err - } - - spec, err := c.multiClusterPersistentVolumeClaimController.GetSpec(cluster) +func (c *controller) reconcilePVCUpdate(clusterName, targetNamespace string, pPVC, vPVC *v1.PersistentVolumeClaim) error { + spec, err := c.multiClusterPersistentVolumeClaimController.GetSpec(clusterName) if err != nil { return err } @@ -110,14 +116,13 @@ func (c *controller) reconcilePVCUpdate(cluster, namespace, name string, vPVC *v return nil } -func (c *controller) reconcilePVCRemove(cluster, namespace, name string) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcilePVCRemove(clusterName, targetNamespace, name string) error { opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, } err := c.pvcClient.PersistentVolumeClaims(targetNamespace).Delete(name, opts) if errors.IsNotFound(err) { - klog.Warningf("pvc %s/%s of cluster not found in super master", namespace, name) + klog.Warningf("pvc %s/%s of cluster %s not found in super master", targetNamespace, name, clusterName) return nil } return err diff --git a/incubator/virtualcluster/pkg/syncer/resources/pod/checker.go b/incubator/virtualcluster/pkg/syncer/resources/pod/checker.go index 08f8c2eed..63d40f7d3 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/pod/checker.go +++ b/incubator/virtualcluster/pkg/syncer/resources/pod/checker.go @@ -174,34 +174,40 @@ func (c *controller) checkPods() { continue } + shouldDelete := false vPodObj, err := c.multiClusterPodController.Get(clusterName, vNamespace, pPod.Name) - if err != nil { - if errors.IsNotFound(err) { - if pPod.DeletionTimestamp != nil { - // pPod is under deletion, waiting for UWS bock populate the pod status. - continue - } - // vPod not found and pPod not under deletion, we need to delete pPod manually - gracePeriod := int64(minimumGracePeriodInSeconds) - if pPod.Spec.TerminationGracePeriodSeconds != nil { - gracePeriod = *pPod.Spec.TerminationGracePeriodSeconds - } - deleteOptions := metav1.NewDeleteOptions(gracePeriod) - deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(pPod.UID)) - if err = c.client.Pods(pPod.Namespace).Delete(pPod.Name, deleteOptions); err != nil { - klog.Errorf("error deleting pPod %v/%v in super master: %v", pPod.Namespace, pPod.Name, err) - } else { - metrics.CheckerRemedyStats.WithLabelValues("numDeletedOrphanSuperMasterPods").Inc() + if errors.IsNotFound(err) && pPod.DeletionTimestamp == nil { + shouldDelete = true + } + if err == nil { + vPod := vPodObj.(*v1.Pod) + if pPod.Annotations[constants.LabelUID] != string(vPod.UID) { + shouldDelete = true + klog.Warningf("Found pPod %s/%s delegated UID is different from tenant object.", pPod.Namespace, pPod.Name) + } else { + if !equality.Semantic.DeepEqual(vPod.Status, pPod.Status) { + numStatusMissMatchedPods++ + klog.Warningf("status of pod %v/%v diff in super&tenant master", pPod.Namespace, pPod.Name) } } - continue } - vPod := vPodObj.(*v1.Pod) - - // pod has been updated by super master - if !equality.Semantic.DeepEqual(vPod.Status, pPod.Status) { - numStatusMissMatchedPods++ - klog.Warningf("status of pod %v/%v diff in super&tenant master", pPod.Namespace, pPod.Name) + if shouldDelete { + if pPod.DeletionTimestamp != nil { + // pPod is under deletion, waiting for UWS bock populate the pod status. + continue + } + // vPod not found and pPod not under deletion, we need to delete pPod manually + gracePeriod := int64(minimumGracePeriodInSeconds) + if pPod.Spec.TerminationGracePeriodSeconds != nil { + gracePeriod = *pPod.Spec.TerminationGracePeriodSeconds + } + deleteOptions := metav1.NewDeleteOptions(gracePeriod) + deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(pPod.UID)) + if err = c.client.Pods(pPod.Namespace).Delete(pPod.Name, deleteOptions); err != nil { + klog.Errorf("error deleting pPod %v/%v in super master: %v", pPod.Namespace, pPod.Name, err) + } else { + metrics.CheckerRemedyStats.WithLabelValues("numDeletedOrphanSuperMasterPods").Inc() + } } } @@ -255,6 +261,11 @@ func (c *controller) checkPodsOfTenantCluster(clusterName string) { klog.Errorf("error getting pPod %s/%s from super master cache: %v", targetNamespace, vPod.Name, err) continue } + if pPod.Annotations[constants.LabelUID] != string(vPod.UID) { + klog.Errorf("Found pPod %s/%s delegated UID is different from tenant object.", targetNamespace, pPod.Name) + continue + } + spec, err := c.multiClusterPodController.GetSpec(clusterName) if err != nil { klog.Errorf("fail to get cluster spec : %s", clusterName) diff --git a/incubator/virtualcluster/pkg/syncer/resources/pod/controller.go b/incubator/virtualcluster/pkg/syncer/resources/pod/controller.go index 277f2cf79..272cb2954 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/pod/controller.go +++ b/incubator/virtualcluster/pkg/syncer/resources/pod/controller.go @@ -23,7 +23,6 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" coreinformers "k8s.io/client-go/informers/core/v1" v1core "k8s.io/client-go/kubernetes/typed/core/v1" @@ -62,7 +61,7 @@ type controller struct { periodCheckerPeriod time.Duration // Cluster vNode PodMap and GCMap, needed for vNode garbage collection sync.Mutex - clusterVNodePodMap map[string]map[string]map[types.UID]struct{} + clusterVNodePodMap map[string]map[string]map[string]struct{} clusterVNodeGCMap map[string]map[string]VNodeGCStatus } @@ -91,7 +90,7 @@ func Register( queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "super_master_pod"), workers: constants.UwsControllerWorkerHigh, periodCheckerPeriod: 60 * time.Second, - clusterVNodePodMap: make(map[string]map[string]map[types.UID]struct{}), + clusterVNodePodMap: make(map[string]map[string]map[string]struct{}), clusterVNodeGCMap: make(map[string]map[string]VNodeGCStatus), } @@ -196,44 +195,38 @@ func (c *controller) removeQuiescingNodeFromClusterVNodeGCMap(cluster string, no return true } -func (c *controller) updateClusterVNodePodMap(cluster string, vPod *v1.Pod, event reconciler.EventType) { - nodeName := vPod.Spec.NodeName - if nodeName == "" || !isPodScheduled(vPod) { - return - } - func() { - c.Lock() - defer c.Unlock() - if event == reconciler.AddEvent || event == reconciler.UpdateEvent { - if _, exist := c.clusterVNodePodMap[cluster]; !exist { - c.clusterVNodePodMap[cluster] = make(map[string]map[types.UID]struct{}) - } - if _, exist := c.clusterVNodePodMap[cluster][nodeName]; !exist { - c.clusterVNodePodMap[cluster][nodeName] = make(map[types.UID]struct{}) - } - c.clusterVNodePodMap[cluster][nodeName][vPod.UID] = struct{}{} - if !c.removeQuiescingNodeFromClusterVNodeGCMap(cluster, nodeName) { - // We have consistency issue here. TODO: add to metrics - klog.Errorf("Cluster %s has vPods in vNode %s which is being GCed!", cluster, nodeName) +func (c *controller) updateClusterVNodePodMap(clusterName, nodeName, requestUID string, event reconciler.EventType) { + c.Lock() + defer c.Unlock() + if event == reconciler.UpdateEvent { + if _, exist := c.clusterVNodePodMap[clusterName]; !exist { + c.clusterVNodePodMap[clusterName] = make(map[string]map[string]struct{}) + } + if _, exist := c.clusterVNodePodMap[clusterName][nodeName]; !exist { + c.clusterVNodePodMap[clusterName][nodeName] = make(map[string]struct{}) + } + c.clusterVNodePodMap[clusterName][nodeName][requestUID] = struct{}{} + if !c.removeQuiescingNodeFromClusterVNodeGCMap(clusterName, nodeName) { + // We have consistency issue here. TODO: add to metrics + klog.Errorf("Cluster %s has vPods in vNode %s which is being GCed!", clusterName, nodeName) + } + } else { // delete + if _, exist := c.clusterVNodePodMap[clusterName][nodeName]; exist { + if _, exist := c.clusterVNodePodMap[clusterName][nodeName][requestUID]; exist { + delete(c.clusterVNodePodMap[clusterName][nodeName], requestUID) + } else { + klog.Warningf("Deleted pod %s of cluster (%s) is not found in clusterVNodePodMap", requestUID, clusterName) } - } else { // delete - if _, exist := c.clusterVNodePodMap[cluster][nodeName]; exist { - if _, exist := c.clusterVNodePodMap[cluster][nodeName][vPod.UID]; exist { - delete(c.clusterVNodePodMap[cluster][nodeName], vPod.UID) - } else { - klog.Warningf("Deleted pod %s of cluster (%s) is not found in clusterVNodePodMap", vPod.Name, cluster) - } - // If vNode does not have any Pod left, put it into gc map - if len(c.clusterVNodePodMap[cluster][nodeName]) == 0 { - c.addToClusterVNodeGCMap(cluster, nodeName) - delete(c.clusterVNodePodMap[cluster], nodeName) - } - } else { - klog.Warningf("The nodename %s of deleted pod %s in cluster (%s) is not found in clusterVNodePodMap", nodeName, vPod.Name, cluster) + // If vNode does not have any Pod left, put it into gc map + if len(c.clusterVNodePodMap[clusterName][nodeName]) == 0 { + c.addToClusterVNodeGCMap(clusterName, nodeName) + delete(c.clusterVNodePodMap[clusterName], nodeName) } + } else { + klog.Warningf("The nodename %s of deleted pod %s in cluster (%s) is not found in clusterVNodePodMap", nodeName, requestUID, clusterName) } - }() + } } func (c *controller) AddCluster(cluster mc.ClusterInterface) { diff --git a/incubator/virtualcluster/pkg/syncer/resources/pod/dws.go b/incubator/virtualcluster/pkg/syncer/resources/pod/dws.go index 9a07853db..df673f2fd 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/pod/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/pod/dws.go @@ -21,7 +21,6 @@ import ( "time" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -43,39 +42,58 @@ func (c *controller) StartDWS(stopCh <-chan struct{}) error { } func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) { - klog.V(4).Infof("reconcile pod %s/%s %s event for cluster %s", request.Namespace, request.Name, request.Event, request.Cluster.Name) - vPod := request.Obj.(*v1.Pod) - c.updateClusterVNodePodMap(request.Cluster.Name, vPod, request.Event) + klog.V(4).Infof("reconcile pod %s/%s for cluster %s", request.Namespace, request.Name, request.ClusterName) + targetNamespace := conversion.ToSuperMasterNamespace(request.ClusterName, request.Namespace) + pPod, err := c.podLister.Pods(targetNamespace).Get(request.Name) + pExists := true + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + pExists = false + } + vExists := true + vPodObj, err := c.multiClusterPodController.Get(request.ClusterName, request.Namespace, request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false + } var operation string - switch request.Event { - case reconciler.AddEvent: + if vExists && !pExists { operation = "pod_add" defer recordOperation(operation, time.Now()) - err := c.reconcilePodCreate(request.Cluster.Name, request.Namespace, request.Name, vPod) - recordError(operation, err) + vPod := vPodObj.(*v1.Pod) + err := c.reconcilePodCreate(request.ClusterName, targetNamespace, request.UID, vPod) if err != nil { - klog.Errorf("failed reconcile pod %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile Pod %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.UpdateEvent: - operation = "pod_update" + } else if !vExists && pExists { + operation = "pod_delete" defer recordOperation(operation, time.Now()) - err := c.reconcilePodUpdate(request.Cluster.Name, request.Namespace, request.Name, vPod) - recordError(operation, err) + // FIXME: For Pod, this should not be reached. So we need to call updateClusterVNodePodMap to remove pod from node in UWS. + err := c.reconcilePodRemove(request.ClusterName, targetNamespace, request.UID, request.Name, pPod) if err != nil { - klog.Errorf("failed reconcile pod %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile Pod %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.DeleteEvent: - operation = "pod_delete" + } else if vExists && pExists { + operation = "pod_update" defer recordOperation(operation, time.Now()) - err := c.reconcilePodRemove(request.Cluster.Name, request.Namespace, request.Name, vPod) - recordError(operation, err) + vPod := vPodObj.(*v1.Pod) + err := c.reconcilePodUpdate(request.ClusterName, targetNamespace, request.UID, pPod, vPod) if err != nil { - klog.Errorf("failed reconcile pod %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile Pod %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } + if vPod.Spec.NodeName != "" && isPodScheduled(vPod) { + c.updateClusterVNodePodMap(request.ClusterName, vPod.Spec.NodeName, request.UID, reconciler.UpdateEvent) + } + } else { + // object is gone. } return reconciler.Result{}, nil } @@ -105,31 +123,25 @@ func createNotSupportEvent(pod *v1.Pod) *v1.Event { } } -func (c *controller) reconcilePodCreate(cluster, namespace, name string, vPod *v1.Pod) error { +func (c *controller) reconcilePodCreate(clusterName, targetNamespace, requestUID string, vPod *v1.Pod) error { // load deleting pod, don't create any pod on super master. if vPod.DeletionTimestamp != nil { - return c.reconcilePodUpdate(cluster, namespace, name, vPod) - } - - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - _, err := c.podLister.Pods(targetNamespace).Get(name) - if err == nil { - return c.reconcilePodUpdate(cluster, namespace, name, vPod) + return nil } if vPod.Spec.NodeName != "" && !isPodScheduled(vPod) { // For now, we skip vPod that has NodeName set to prevent tenant from deploying DaemonSet or DaemonSet alike CRDs. - tenantClient, err := c.multiClusterPodController.GetClusterClient(cluster) + tenantClient, err := c.multiClusterPodController.GetClusterClient(clusterName) if err != nil { - return fmt.Errorf("failed to create client from cluster %s config: %v", cluster, err) + return fmt.Errorf("failed to create client from cluster %s config: %v", clusterName, err) } event := createNotSupportEvent(vPod) - vEvent := conversion.BuildVirtualPodEvent(cluster, event, vPod) + vEvent := conversion.BuildVirtualPodEvent(clusterName, event, vPod) _, err = tenantClient.CoreV1().Events(vPod.Namespace).Create(vEvent) return err } - newObj, err := conversion.BuildMetadata(cluster, targetNamespace, vPod) + newObj, err := conversion.BuildMetadata(clusterName, targetNamespace, vPod) if err != nil { return err } @@ -138,15 +150,15 @@ func (c *controller) reconcilePodCreate(cluster, namespace, name string, vPod *v pSecret, err := c.getPodServiceAccountSecret(pPod) if err != nil { - return fmt.Errorf("failed to get service account secret from cluster %s cache: %v", cluster, err) + return fmt.Errorf("failed to get service account secret from cluster %s cache: %v", clusterName, err) } - services, err := c.getPodRelatedServices(cluster, pPod) + services, err := c.getPodRelatedServices(clusterName, pPod) if err != nil { - return fmt.Errorf("failed to list services from cluster %s cache: %v", cluster, err) + return fmt.Errorf("failed to list services from cluster %s cache: %v", clusterName, err) } - nameServer, err := c.getClusterNameServer(cluster) + nameServer, err := c.getClusterNameServer(clusterName) if err != nil { return fmt.Errorf("failed to find nameserver: %v", err) } @@ -158,14 +170,18 @@ func (c *controller) reconcilePodCreate(cluster, namespace, name string, vPod *v //conversion.PodAddExtensionMeta(vPod), } - err = conversion.VC(c.multiClusterPodController, cluster).Pod(pPod).Mutate(ms...) + err = conversion.VC(c.multiClusterPodController, clusterName).Pod(pPod).Mutate(ms...) if err != nil { return fmt.Errorf("failed to mutate pod: %v", err) } - _, err = c.client.Pods(targetNamespace).Create(pPod) + pPod, err = c.client.Pods(targetNamespace).Create(pPod) if errors.IsAlreadyExists(err) { - klog.Infof("pod %s/%s of cluster %s already exist in super master", namespace, name, cluster) - return nil + if pPod.Annotations[constants.LabelUID] == requestUID { + klog.Infof("pod %s/%s of cluster %s already exist in super master", targetNamespace, pPod.Name, clusterName) + return nil + } else { + return fmt.Errorf("pPod %s/%s exists but the UID is different from tenant master.", targetNamespace, pPod.Name) + } } return err @@ -219,17 +235,9 @@ func (c *controller) getPodRelatedServices(cluster string, pPod *v1.Pod) ([]*v1. return services, nil } -func (c *controller) reconcilePodUpdate(cluster, namespace, name string, vPod *v1.Pod) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - pPod, err := c.podLister.Pods(targetNamespace).Get(name) - if err != nil { - if errors.IsNotFound(err) { - // if the pod on super master has been deleted and syncer has not - // deleted virtual pod with 0 grace period second successfully. - // we depends on periodic check to do gc. - return nil - } - return err +func (c *controller) reconcilePodUpdate(clusterName, targetNamespace, requestUID string, pPod, vPod *v1.Pod) error { + if pPod.Annotations[constants.LabelUID] != requestUID { + return fmt.Errorf("pPod %s/%s delegated UID is different from updated object.", targetNamespace, pPod.Name) } if vPod.DeletionTimestamp != nil { @@ -239,14 +247,14 @@ func (c *controller) reconcilePodUpdate(cluster, namespace, name string, vPod *v } deleteOptions := metav1.NewDeleteOptions(*vPod.DeletionGracePeriodSeconds) deleteOptions.Preconditions = metav1.NewUIDPreconditions(string(pPod.UID)) - err = c.client.Pods(targetNamespace).Delete(name, deleteOptions) + err := c.client.Pods(targetNamespace).Delete(pPod.Name, deleteOptions) if errors.IsNotFound(err) { return nil } return err } - spec, err := c.multiClusterPodController.GetSpec(cluster) + spec, err := c.multiClusterPodController.GetSpec(clusterName) if err != nil { return err } @@ -257,23 +265,21 @@ func (c *controller) reconcilePodUpdate(cluster, namespace, name string, vPod *v return err } } - - // pod has been updated by tenant controller - if !equality.Semantic.DeepEqual(vPod.Status, pPod.Status) { - c.enqueuePod(pPod) - } - return nil } -func (c *controller) reconcilePodRemove(cluster, namespace, name string, vPod *v1.Pod) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcilePodRemove(clusterName, targetNamespace, requestUID, name string, pPod *v1.Pod) error { + if pPod.Annotations[constants.LabelUID] != requestUID { + return fmt.Errorf("To be deleted pPod %s/%s delegated UID is different from deleted object.", targetNamespace, name) + } + opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, + Preconditions: metav1.NewUIDPreconditions(string(pPod.UID)), } err := c.client.Pods(targetNamespace).Delete(name, opts) if errors.IsNotFound(err) { - klog.Warningf("pod %s/%s of cluster (%s) is not found in super master", namespace, name, cluster) + klog.Warningf("To be deleted pod %s/%s of cluster (%s) is not found in super master", targetNamespace, name, clusterName) return nil } return err diff --git a/incubator/virtualcluster/pkg/syncer/resources/pod/uws.go b/incubator/virtualcluster/pkg/syncer/resources/pod/uws.go index 9b056d445..106622eb3 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/pod/uws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/pod/uws.go @@ -130,6 +130,9 @@ func (c *controller) backPopulate(key string) error { return fmt.Errorf("could not find pPod %s/%s's vPod in controller cache %v", vNamespace, pName, err) } vPod := vPodObj.(*v1.Pod) + if pPod.Annotations[constants.LabelUID] != string(vPod.UID) { + return fmt.Errorf("BackPopulated pPod %s/%s delegated UID is different from updated object.", pPod.Namespace, pPod.Name) + } tenantClient, err := c.multiClusterPodController.GetClusterClient(clusterName) if err != nil { diff --git a/incubator/virtualcluster/pkg/syncer/resources/secret/dws.go b/incubator/virtualcluster/pkg/syncer/resources/secret/dws.go index 7317d8980..b3e574548 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/secret/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/secret/dws.go @@ -23,7 +23,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/klog" @@ -41,100 +40,93 @@ func (c *controller) StartDWS(stopCh <-chan struct{}) error { // The reconcile logic for tenant master secret informer func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) { - klog.V(4).Infof("reconcile secret %s/%s %s event for cluster %s", request.Namespace, request.Name, request.Event, request.Cluster.Name) + klog.V(4).Infof("reconcile secret %s/%s for cluster %s", request.Namespace, request.Name, request.ClusterName) + targetNamespace := conversion.ToSuperMasterNamespace(request.ClusterName, request.Namespace) + vSecretObj, err := c.multiClusterSecretController.Get(request.ClusterName, request.Namespace, request.Name) + vExists := true + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false + } + // FIXME: Do we need to add sa name in the selector? + pExists := true + var pSecret *v1.Secret + secretList, err := c.secretLister.Secrets(targetNamespace).List(labels.SelectorFromSet(map[string]string{ + constants.LabelSecretName: request.Name, + })) + if err != nil && !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + if len(secretList) != 0 { + // This is service account vSecret + pSecret = secretList[0] + } else { + // We need to use name to search again for normal vScrect + pSecret, err = c.secretLister.Secrets(targetNamespace).Get(request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + pExists = false + } + } - switch request.Event { - case reconciler.AddEvent: - err := c.reconcileSecretCreate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Secret)) + if vExists && !pExists { + vSecret := vSecretObj.(*v1.Secret) + err := c.reconcileSecretCreate(request.ClusterName, targetNamespace, vSecret) if err != nil { - klog.Errorf("failed reconcile secret %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile secret %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.UpdateEvent: - err := c.reconcileSecretUpdate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Secret)) + } else if !vExists && pExists { + err := c.reconcileSecretRemove(request.ClusterName, targetNamespace, request.Name, pSecret) if err != nil { - klog.Errorf("failed reconcile secret %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile secret %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.DeleteEvent: - err := c.reconcileSecretRemove(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Secret)) + } else if vExists && pExists { + vSecret := vSecretObj.(*v1.Secret) + err := c.reconcileSecretUpdate(request.ClusterName, targetNamespace, pSecret, vSecret) if err != nil { - klog.Errorf("failed reconcile secret %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile secret %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } + } else { + // object is gone. } return reconciler.Result{}, nil } -func (c *controller) reconcileSecretCreate(cluster, namespace, name string, secret *v1.Secret) error { +func (c *controller) reconcileSecretCreate(clusterName, targetNamespace string, secret *v1.Secret) error { switch secret.Type { case v1.SecretTypeServiceAccountToken: - return c.reconcileServiceAccountSecretCreate(cluster, namespace, name, secret) + return c.reconcileServiceAccountSecretCreate(clusterName, targetNamespace, secret) default: - return c.reconcileNormalSecretCreate(cluster, namespace, name, secret) + return c.reconcileNormalSecretCreate(clusterName, targetNamespace, secret) } } -func (c *controller) reconcileServiceAccountSecretCreate(cluster, namespace, name string, vSecret *v1.Secret) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - saName := vSecret.GetAnnotations()[v1.ServiceAccountNameKey] - - secretList, err := c.secretLister.Secrets(targetNamespace).List(labels.SelectorFromSet(map[string]string{ - constants.LabelServiceAccountName: saName, - constants.LabelSecretName: vSecret.Name, - })) - if err != nil && !errors.IsNotFound(err) { - return err - } - if len(secretList) > 0 { - return serviceAccountSecretUpdate(c.secretClient.Secrets(targetNamespace), secretList, vSecret) - } - - newObj, err := conversion.BuildMetadata(cluster, targetNamespace, vSecret) +func (c *controller) reconcileServiceAccountSecretCreate(clusterName, targetNamespace string, vSecret *v1.Secret) error { + newObj, err := conversion.BuildMetadata(clusterName, targetNamespace, vSecret) if err != nil { return err } pSecret := newObj.(*v1.Secret) - conversion.VC(c.multiClusterSecretController, "").ServiceAccountTokenSecret(pSecret).Mutate(vSecret, cluster) + conversion.VC(c.multiClusterSecretController, "").ServiceAccountTokenSecret(pSecret).Mutate(vSecret, clusterName) _, err = c.secretClient.Secrets(targetNamespace).Create(pSecret) if errors.IsAlreadyExists(err) { - klog.Infof("secret %s/%s of cluster %s already exist in super master", namespace, name, cluster) + klog.Infof("secret %s/%s of cluster %s already exist in super master", targetNamespace, pSecret.Name, clusterName) return nil } return err } -func (c *controller) reconcileServiceAccountSecretUpdate(cluster, namespace, name string, vSecret *v1.Secret) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - - saName := vSecret.Annotations[v1.ServiceAccountNameKey] - - secretList, err := c.secretLister.Secrets(targetNamespace).List(labels.SelectorFromSet(map[string]string{ - constants.LabelServiceAccountName: saName, - constants.LabelSecretName: vSecret.Name, - })) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - return err - } - if len(secretList) == 0 { - return nil - } - - return serviceAccountSecretUpdate(c.secretClient.Secrets(targetNamespace), secretList, vSecret) -} - -func serviceAccountSecretUpdate(secretClient corev1.SecretInterface, secretList []*v1.Secret, vSecret *v1.Secret) error { - if len(secretList) == 0 { - return nil - } - pSecret := secretList[0] - +func (c *controller) reconcileServiceAccountSecretUpdate(clusterName, targetNamespace string, pSecret, vSecret *v1.Secret) error { updatedBinaryData, equal := conversion.Equality(nil).CheckBinaryDataEquality(pSecret.Data, vSecret.Data) if equal { return nil @@ -142,7 +134,7 @@ func serviceAccountSecretUpdate(secretClient corev1.SecretInterface, secretList updatedSecret := pSecret.DeepCopy() updatedSecret.Data = updatedBinaryData - _, err := secretClient.Update(pSecret) + _, err := c.secretClient.Secrets(targetNamespace).Update(updatedSecret) if err != nil { return err } @@ -150,47 +142,32 @@ func serviceAccountSecretUpdate(secretClient corev1.SecretInterface, secretList return nil } -func (c *controller) reconcileNormalSecretCreate(cluster, namespace, name string, secret *v1.Secret) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - _, err := c.secretLister.Secrets(targetNamespace).Get(name) - if err == nil { - return c.reconcileNormalSecretUpdate(cluster, namespace, name, secret) - } - - newObj, err := conversion.BuildMetadata(cluster, targetNamespace, secret) +func (c *controller) reconcileNormalSecretCreate(clusterName, targetNamespace string, secret *v1.Secret) error { + newObj, err := conversion.BuildMetadata(clusterName, targetNamespace, secret) if err != nil { return err } _, err = c.secretClient.Secrets(targetNamespace).Create(newObj.(*v1.Secret)) if errors.IsAlreadyExists(err) { - klog.Infof("secret %s/%s of cluster %s already exist in super master", namespace, name, cluster) + klog.Infof("secret %s/%s of cluster %s already exist in super master", targetNamespace, secret.Name, clusterName) return nil } return err } -func (c *controller) reconcileSecretUpdate(cluster, namespace, name string, secret *v1.Secret) error { - switch secret.Type { +func (c *controller) reconcileSecretUpdate(clusterName, targetNamespace string, pSecret, vSecret *v1.Secret) error { + switch vSecret.Type { case v1.SecretTypeServiceAccountToken: - return c.reconcileServiceAccountSecretUpdate(cluster, namespace, name, secret) + return c.reconcileServiceAccountSecretUpdate(clusterName, targetNamespace, pSecret, vSecret) default: - return c.reconcileNormalSecretUpdate(cluster, namespace, name, secret) + return c.reconcileNormalSecretUpdate(clusterName, targetNamespace, pSecret, vSecret) } } -func (c *controller) reconcileNormalSecretUpdate(cluster, namespace, name string, vSecret *v1.Secret) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - pSecret, err := c.secretLister.Secrets(targetNamespace).Get(name) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - return err - } - - spec, err := c.multiClusterSecretController.GetSpec(cluster) +func (c *controller) reconcileNormalSecretUpdate(clusterName, targetNamespace string, pSecret, vSecret *v1.Secret) error { + spec, err := c.multiClusterSecretController.GetSpec(clusterName) if err != nil { return err } @@ -205,30 +182,28 @@ func (c *controller) reconcileNormalSecretUpdate(cluster, namespace, name string return nil } -func (c *controller) reconcileSecretRemove(cluster, namespace, name string, secret *v1.Secret) error { +func (c *controller) reconcileSecretRemove(clusterName, targetNamespace, name string, secret *v1.Secret) error { switch secret.Type { case v1.SecretTypeServiceAccountToken: - return c.reconcileServiceAccountTokenSecretRemove(cluster, namespace, name, secret) + return c.reconcileServiceAccountTokenSecretRemove(clusterName, targetNamespace, name) default: - return c.reconcileNormalSecretRemove(cluster, namespace, name, secret) + return c.reconcileNormalSecretRemove(clusterName, targetNamespace, name) } } -func (c *controller) reconcileNormalSecretRemove(cluster, namespace, name string, secret *v1.Secret) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcileNormalSecretRemove(clusterName, targetNamespace, name string) error { opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, } err := c.secretClient.Secrets(targetNamespace).Delete(name, opts) if errors.IsNotFound(err) { - klog.Warningf("secret %s/%s of cluster is not found in super master", namespace, name) + klog.Warningf("secret %s/%s of cluster is not found in super master", targetNamespace, name) return nil } return err } -func (c *controller) reconcileServiceAccountTokenSecretRemove(cluster, namespace, name string, vSecret *v1.Secret) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcileServiceAccountTokenSecretRemove(clusterName, targetNamespace, name string) error { opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, } @@ -238,7 +213,7 @@ func (c *controller) reconcileServiceAccountTokenSecretRemove(cluster, namespace }).String(), }) if errors.IsNotFound(err) { - klog.Warningf("secret %s/%s of cluster is not found in super master", namespace, name) + klog.Warningf("secret %s/%s of cluster is not found in super master", targetNamespace, name) return nil } return err diff --git a/incubator/virtualcluster/pkg/syncer/resources/service/checker.go b/incubator/virtualcluster/pkg/syncer/resources/service/checker.go index b157c57c7..6bce3dbf2 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/service/checker.go +++ b/incubator/virtualcluster/pkg/syncer/resources/service/checker.go @@ -30,6 +30,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog" + "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/constants" "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/conversion" "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/metrics" "github.com/kubernetes-sigs/multi-tenancy/incubator/virtualcluster/pkg/syncer/reconciler" @@ -78,16 +79,26 @@ func (c *controller) checkServices() { if len(clusterName) == 0 || len(vNamespace) == 0 { continue } - - _, err := c.multiClusterServiceController.Get(clusterName, vNamespace, pService.Name) + shouldDelete := false + vServiceObj, err := c.multiClusterServiceController.Get(clusterName, vNamespace, pService.Name) if errors.IsNotFound(err) { + shouldDelete = true + } + if err == nil { + vService := vServiceObj.(*v1.Service) + if pService.Annotations[constants.LabelUID] != string(vService.UID) { + shouldDelete = true + klog.Warningf("Found pService %s/%s delegated UID is different from tenant object.", pService.Namespace, pService.Name) + } + + } + if shouldDelete { deleteOptions := metav1.NewPreconditionDeleteOptions(string(pService.UID)) if err = c.serviceClient.Services(pService.Namespace).Delete(pService.Name, deleteOptions); err != nil { klog.Errorf("error deleting pService %s/%s in super master: %v", pService.Namespace, pService.Name, err) } else { metrics.CheckerRemedyStats.WithLabelValues("numDeletedOrphanSuperMasterServices").Inc() } - continue } } @@ -118,6 +129,12 @@ func (c *controller) checkServicesOfTenantCluster(clusterName string) { klog.Errorf("failed to get pService %s/%s from super master cache: %v", targetNamespace, vService.Name, err) continue } + + if pService.Annotations[constants.LabelUID] != string(vService.UID) { + klog.Errorf("Found pService %s/%s delegated UID is different from tenant object.", targetNamespace, pService.Name) + continue + } + spec, err := c.multiClusterServiceController.GetSpec(clusterName) if err != nil { klog.Errorf("fail to get cluster spec : %s", clusterName) diff --git a/incubator/virtualcluster/pkg/syncer/resources/service/dws.go b/incubator/virtualcluster/pkg/syncer/resources/service/dws.go index 024c8ce2c..22c35b063 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/service/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/service/dws.go @@ -38,39 +38,53 @@ func (c *controller) StartDWS(stopCh <-chan struct{}) error { } func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) { - klog.V(4).Infof("reconcile service %s/%s %s event for cluster %s", request.Namespace, request.Name, request.Event, request.Cluster.Name) + klog.V(4).Infof("reconcile service %s/%s for cluster %s", request.Namespace, request.Name, request.ClusterName) + targetNamespace := conversion.ToSuperMasterNamespace(request.ClusterName, request.Namespace) + pService, err := c.serviceLister.Services(targetNamespace).Get(request.Name) + pExists := true + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + pExists = false + } + vExists := true + vServiceObj, err := c.multiClusterServiceController.Get(request.ClusterName, request.Namespace, request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false + } - switch request.Event { - case reconciler.AddEvent: - err := c.reconcileServiceCreate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Service)) + if vExists && !pExists { + vService := vServiceObj.(*v1.Service) + err := c.reconcileServiceCreate(request.ClusterName, targetNamespace, request.UID, vService) if err != nil { - klog.Errorf("failed reconcile service %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile service %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.UpdateEvent: - err := c.reconcileServiceUpdate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Service)) + } else if !vExists && pExists { + err := c.reconcileServiceRemove(request.ClusterName, targetNamespace, request.UID, request.Name, pService) if err != nil { - klog.Errorf("failed reconcile service %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile service %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.DeleteEvent: - err := c.reconcileServiceRemove(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.Service)) + } else if vExists && pExists { + vService := vServiceObj.(*v1.Service) + err := c.reconcileServiceUpdate(request.ClusterName, targetNamespace, request.UID, pService, vService) if err != nil { - klog.Errorf("failed reconcile service %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile service %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } + } else { + // object is gone. } return reconciler.Result{}, nil } -func (c *controller) reconcileServiceCreate(cluster, namespace, name string, service *v1.Service) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - _, err := c.serviceLister.Services(targetNamespace).Get(name) - if err == nil { - return c.reconcileServiceUpdate(cluster, namespace, name, service) - } - - newObj, err := conversion.BuildMetadata(cluster, targetNamespace, service) +func (c *controller) reconcileServiceCreate(clusterName, targetNamespace, requestUID string, service *v1.Service) error { + newObj, err := conversion.BuildMetadata(clusterName, targetNamespace, service) if err != nil { return err } @@ -78,25 +92,24 @@ func (c *controller) reconcileServiceCreate(cluster, namespace, name string, ser pService := newObj.(*v1.Service) conversion.VC(nil, "").Service(pService).Mutate(service) - _, err = c.serviceClient.Services(targetNamespace).Create(pService) + pService, err = c.serviceClient.Services(targetNamespace).Create(pService) if errors.IsAlreadyExists(err) { - klog.Infof("service %s/%s of cluster %s already exist in super master", namespace, name, cluster) - return nil + if pService.Annotations[constants.LabelUID] == requestUID { + klog.Infof("service %s/%s of cluster %s already exist in super master", targetNamespace, pService.Name, clusterName) + return nil + } else { + return fmt.Errorf("pService %s/%s exists but its delegated object UID is different.", targetNamespace, pService.Name) + } } return err } -func (c *controller) reconcileServiceUpdate(cluster, namespace, name string, vService *v1.Service) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - pService, err := c.serviceLister.Services(targetNamespace).Get(name) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - return err +func (c *controller) reconcileServiceUpdate(clusterName, targetNamespace, requestUID string, pService, vService *v1.Service) error { + if pService.Annotations[constants.LabelUID] != requestUID { + return fmt.Errorf("pService %s/%s delegated UID is different from updated object.", targetNamespace, pService.Name) } - spec, err := c.multiClusterServiceController.GetSpec(cluster) + spec, err := c.multiClusterServiceController.GetSpec(clusterName) if err != nil { return err } @@ -110,14 +123,18 @@ func (c *controller) reconcileServiceUpdate(cluster, namespace, name string, vSe return nil } -func (c *controller) reconcileServiceRemove(cluster, namespace, name string, service *v1.Service) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcileServiceRemove(clusterName, targetNamespace, requestUID, name string, pService *v1.Service) error { + if pService.Annotations[constants.LabelUID] != requestUID { + return fmt.Errorf("To be deleted pService %s/%s delegated UID is different from deleted object.", targetNamespace, name) + } + opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, + Preconditions: metav1.NewUIDPreconditions(string(pService.UID)), } err := c.serviceClient.Services(targetNamespace).Delete(name, opts) if errors.IsNotFound(err) { - klog.Warningf("service %s/%s of cluster not found in super master", namespace, name) + klog.Warningf("To be deleted service %s/%s not found in super master", targetNamespace, name) return nil } return err diff --git a/incubator/virtualcluster/pkg/syncer/resources/service/uws.go b/incubator/virtualcluster/pkg/syncer/resources/service/uws.go index 8c1d241cd..609f54583 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/service/uws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/service/uws.go @@ -128,6 +128,9 @@ func (c *controller) backPopulate(key string) error { return fmt.Errorf("could not find pService %s/%s's vService in controller cache %v", vNamespace, pName, err) } vService := vServiceObj.(*v1.Service) + if pService.Annotations[constants.LabelUID] != string(vService.UID) { + return fmt.Errorf("BackPopulated pService %s/%s delegated UID is different from updated object.", pService.Namespace, pService.Name) + } tenantClient, err := c.multiClusterServiceController.GetClusterClient(clusterName) if err != nil { diff --git a/incubator/virtualcluster/pkg/syncer/resources/serviceaccount/dws.go b/incubator/virtualcluster/pkg/syncer/resources/serviceaccount/dws.go index 4d80148ee..12f59cfcc 100644 --- a/incubator/virtualcluster/pkg/syncer/resources/serviceaccount/dws.go +++ b/incubator/virtualcluster/pkg/syncer/resources/serviceaccount/dws.go @@ -39,58 +39,53 @@ func (c *controller) StartDWS(stopCh <-chan struct{}) error { // The reconcile logic for tenant master service account informer func (c *controller) Reconcile(request reconciler.Request) (reconciler.Result, error) { - klog.V(4).Infof("reconcile service account %s/%s %s event for cluster %s", request.Namespace, request.Name, request.Event, request.Cluster.Name) + klog.V(4).Infof("reconcile service account %s/%s for cluster %s", request.Namespace, request.Name, request.ClusterName) + targetNamespace := conversion.ToSuperMasterNamespace(request.ClusterName, request.Namespace) + pSa, err := c.saLister.ServiceAccounts(targetNamespace).Get(request.Name) + pExists := true + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + pExists = false + } + vExists := true + vSaObj, err := c.multiClusterServiceAccountController.Get(request.ClusterName, request.Namespace, request.Name) + if err != nil { + if !errors.IsNotFound(err) { + return reconciler.Result{Requeue: true}, err + } + vExists = false + } - switch request.Event { - case reconciler.AddEvent: - err := c.reconcileServiceAccountCreate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount)) + if vExists && !pExists { + vSa := vSaObj.(*v1.ServiceAccount) + err := c.reconcileServiceAccountCreate(request.ClusterName, targetNamespace, vSa) if err != nil { - klog.Errorf("failed reconcile service account %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile serviceaccount %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.UpdateEvent: - err := c.reconcileServiceAccountUpdate(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount)) + } else if !vExists && pExists { + err := c.reconcileServiceAccountRemove(request.ClusterName, targetNamespace, request.Name) if err != nil { - klog.Errorf("failed reconcile service account %s/%s CREATE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile serviceaccount %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } - case reconciler.DeleteEvent: - err := c.reconcileServiceAccountRemove(request.Cluster.Name, request.Namespace, request.Name, request.Obj.(*v1.ServiceAccount)) + } else if vExists && pExists { + vSa := vSaObj.(*v1.ServiceAccount) + err := c.reconcileServiceAccountUpdate(request.ClusterName, targetNamespace, pSa, vSa) if err != nil { - klog.Errorf("failed reconcile service account %s/%s DELETE of cluster %s %v", request.Namespace, request.Name, request.Cluster.Name, err) + klog.Errorf("failed reconcile serviceaccount %s/%s UPDATE of cluster %s %v", request.Namespace, request.Name, request.ClusterName, err) return reconciler.Result{Requeue: true}, err } + } else { + // object is gone. } return reconciler.Result{}, nil } -func (c *controller) reconcileServiceAccountCreate(cluster, namespace, name string, secret *v1.ServiceAccount) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) - // Just mark the default service account of super master namespace, created by super master service account controller, as a tenant related resource. - if name == "default" { - sa, err := c.saLister.ServiceAccounts(targetNamespace).Get("default") - if err != nil { - // maybe the sa is not created, retry - return err - } - - if len(sa.Annotations) == 0 { - sa.Annotations = make(map[string]string) - } - if sa.Annotations[constants.LabelCluster] != cluster { - sa.Annotations[constants.LabelCluster] = cluster - _, err = c.saClient.ServiceAccounts(targetNamespace).Update(sa) - } - return err - } - - _, err := c.saLister.ServiceAccounts(targetNamespace).Get(name) - if err == nil { - // sa already exists. - return nil - } - - newObj, err := conversion.BuildMetadata(cluster, targetNamespace, secret) +func (c *controller) reconcileServiceAccountCreate(clusterName, targetNamespace string, secret *v1.ServiceAccount) error { + newObj, err := conversion.BuildMetadata(clusterName, targetNamespace, secret) if err != nil { return err } @@ -100,25 +95,37 @@ func (c *controller) reconcileServiceAccountCreate(cluster, namespace, name stri _, err = c.saClient.ServiceAccounts(targetNamespace).Create(pServiceAccount) if errors.IsAlreadyExists(err) { - klog.Infof("service account %s/%s of cluster %s already exist in super master", namespace, name, cluster) + klog.Infof("service account %s/%s of cluster %s already exist in super master", targetNamespace, pServiceAccount.Name, clusterName) return nil } return err } -func (c *controller) reconcileServiceAccountUpdate(cluster, namespace, name string, secret *v1.ServiceAccount) error { +func (c *controller) reconcileServiceAccountUpdate(clusterName, targetNamespace string, pSa, vSa *v1.ServiceAccount) error { + // Just mark the default service account of super master namespace, created by super master service account controller, as a tenant related resource. + if vSa.Name == "default" { + if len(pSa.Annotations) == 0 { + pSa.Annotations = make(map[string]string) + } + var err error + if pSa.Annotations[constants.LabelCluster] != clusterName { + // FIXME: How about ns/UID? + pSa.Annotations[constants.LabelCluster] = clusterName + _, err = c.saClient.ServiceAccounts(targetNamespace).Update(pSa) + } + return err + } // do nothing. return nil } -func (c *controller) reconcileServiceAccountRemove(cluster, namespace, name string, secret *v1.ServiceAccount) error { - targetNamespace := conversion.ToSuperMasterNamespace(cluster, namespace) +func (c *controller) reconcileServiceAccountRemove(clusterName, targetNamespace, name string) error { opts := &metav1.DeleteOptions{ PropagationPolicy: &constants.DefaultDeletionPolicy, } err := c.saClient.ServiceAccounts(targetNamespace).Delete(name, opts) if errors.IsNotFound(err) { - klog.Warningf("service account %s/%s of cluster %s not found in super master", namespace, name, cluster) + klog.Warningf("service account %s/%s of cluster %s not found in super master", targetNamespace, name, clusterName) return nil } return err