diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..e69de29b diff --git a/clusterkubevirtadm/cmd/get/kubeconfig/kubeconfig_test.go b/clusterkubevirtadm/cmd/get/kubeconfig/kubeconfig_test.go index c984df5d..e705acd9 100644 --- a/clusterkubevirtadm/cmd/get/kubeconfig/kubeconfig_test.go +++ b/clusterkubevirtadm/cmd/get/kubeconfig/kubeconfig_test.go @@ -157,7 +157,7 @@ var _ = Describe("test kubeconfig function", func() { Expect(found).ShouldNot(BeNil()) Expect(found.Secrets).ToNot(BeEmpty()) - Eventually(doneUpdatingSa) + Eventually(doneUpdatingSa).Should(BeClosed()) }) It("should fail after 10 seconds if the secret was not created", func() { diff --git a/config/kccm/kustomization.yaml b/config/kccm/kustomization.yaml index 61123d26..98d77f53 100644 --- a/config/kccm/kustomization.yaml +++ b/config/kccm/kustomization.yaml @@ -14,6 +14,8 @@ patchesJson6902: name: kubevirt-cloud-controller-manager bases: - https://github.com/kubevirt/cloud-provider-kubevirt/config/isolated?ref=v0.3.2 +- ../rbac + commonLabels: cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" capk.cluster.x-k8s.io/template-kind: "extra-resource" diff --git a/controllers/kubevirtmachine_controller.go b/controllers/kubevirtmachine_controller.go index 3bca8249..be2a1284 100644 --- a/controllers/kubevirtmachine_controller.go +++ b/controllers/kubevirtmachine_controller.go @@ -65,7 +65,7 @@ type KubevirtMachineReconciler struct { // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;machines,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachines;,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances;,verbs=get;list;watch +// +kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances;,verbs=get;list;watch;patch;update;delete // Reconcile handles KubevirtMachine events. func (r *KubevirtMachineReconciler) Reconcile(goctx gocontext.Context, req ctrl.Request) (_ ctrl.Result, rerr error) { @@ -289,6 +289,14 @@ func (r *KubevirtMachineReconciler) reconcileNormal(ctx *context.MachineContext) return ctrl.Result{RequeueAfter: 20 * time.Second}, nil } + retryDuration, err := externalMachine.DrainNodeIfNeeded(r.WorkloadCluster) + if err != nil { + return ctrl.Result{RequeueAfter: retryDuration}, errors.Wrap(err, "failed to drain node") + } + if retryDuration > 0 { + return ctrl.Result{RequeueAfter: retryDuration}, nil + } + if externalMachine.SupportsCheckingIsBootstrapped() && !conditions.IsTrue(ctx.KubevirtMachine, infrav1.BootstrapExecSucceededCondition) { if !externalMachine.IsBootstrapped() { ctx.Logger.Info("Waiting for underlying VM to bootstrap...") diff --git a/controllers/kubevirtmachine_controller_test.go b/controllers/kubevirtmachine_controller_test.go index 0381dbe6..9fa33d99 100644 --- a/controllers/kubevirtmachine_controller_test.go +++ b/controllers/kubevirtmachine_controller_test.go @@ -18,6 +18,7 @@ package controllers import ( gocontext "context" + "fmt" "time" "github.com/golang/mock/gomock" @@ -395,6 +396,8 @@ var _ = Describe("reconcile a kubevirt machine", func() { machineMock.EXPECT().Address().Return("1.1.1.1").AnyTimes() machineMock.EXPECT().SupportsCheckingIsBootstrapped().Return(false).AnyTimes() machineMock.EXPECT().GenerateProviderID().Return("abc", nil).AnyTimes() + machineMock.EXPECT().GenerateProviderID().Return("abc", nil).AnyTimes() + machineMock.EXPECT().DrainNodeIfNeeded(gomock.Any()).Return(time.Duration(0), nil).AnyTimes() machineFactoryMock.EXPECT().NewMachine(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(machineMock, nil).Times(1) infraClusterMock.EXPECT().GenerateInfraClusterClient(kubevirtMachine.Spec.InfraClusterSecretRef, kubevirtMachine.Namespace, machineContext.Context).Return(fakeClient, kubevirtMachine.Namespace, nil).Times(3) @@ -898,6 +901,7 @@ var _ = Describe("reconcile a kubevirt machine", func() { machineMock.EXPECT().Exists().Return(true).Times(1) machineMock.EXPECT().Address().Return("1.1.1.1").Times(1) machineMock.EXPECT().SupportsCheckingIsBootstrapped().Return(false).Times(1) + machineMock.EXPECT().DrainNodeIfNeeded(gomock.Any()).Return(time.Duration(0), nil) machineFactoryMock.EXPECT().NewMachine(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(machineMock, nil).Times(1) infraClusterMock.EXPECT().GenerateInfraClusterClient(kubevirtMachine.Spec.InfraClusterSecretRef, kubevirtMachine.Namespace, machineContext.Context).Return(fakeClient, kubevirtMachine.Namespace, nil) @@ -943,6 +947,7 @@ var _ = Describe("reconcile a kubevirt machine", func() { machineMock.EXPECT().GenerateProviderID().Return("abc", nil).AnyTimes() machineMock.EXPECT().SupportsCheckingIsBootstrapped().Return(true) machineMock.EXPECT().IsBootstrapped().Return(false) + machineMock.EXPECT().DrainNodeIfNeeded(gomock.Any()).Return(time.Duration(0), nil) machineFactoryMock.EXPECT().NewMachine(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(machineMock, nil).Times(1) @@ -992,6 +997,7 @@ var _ = Describe("reconcile a kubevirt machine", func() { machineMock.EXPECT().GenerateProviderID().Return("abc", nil).Times(1) machineMock.EXPECT().SupportsCheckingIsBootstrapped().Return(true) machineMock.EXPECT().IsBootstrapped().Return(true) + machineMock.EXPECT().DrainNodeIfNeeded(gomock.Any()).Return(time.Duration(0), nil) machineFactoryMock.EXPECT().NewMachine(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(machineMock, nil).Times(1) @@ -1007,6 +1013,97 @@ var _ = Describe("reconcile a kubevirt machine", func() { Expect(conditions[0].Type).To(Equal(infrav1.BootstrapExecSucceededCondition)) Expect(conditions[0].Status).To(Equal(corev1.ConditionTrue)) }) + + It("should requeue on node draining", func() { + vmiReadyCondition := kubevirtv1.VirtualMachineInstanceCondition{ + Type: kubevirtv1.VirtualMachineInstanceReady, + Status: corev1.ConditionTrue, + } + vmi.Status.Conditions = append(vmi.Status.Conditions, vmiReadyCondition) + vmi.Status.Interfaces = []kubevirtv1.VirtualMachineInstanceNetworkInterface{ + + { + IP: "1.1.1.1", + }, + } + sshKeySecret.Data["pub"] = []byte("shell") + + objects := []client.Object{ + cluster, + kubevirtCluster, + machine, + kubevirtMachine, + bootstrapSecret, + bootstrapUserDataSecret, + sshKeySecret, + vm, + vmi, + } + + const requeueDurationSeconds = 3 + machineMock.EXPECT().IsTerminal().Return(false, "", nil).Times(1) + machineMock.EXPECT().Exists().Return(true).Times(1) + machineMock.EXPECT().IsReady().Return(true).Times(1) + machineMock.EXPECT().Address().Return("1.1.1.1").Times(1) + machineMock.EXPECT().DrainNodeIfNeeded(gomock.Any()).Return(time.Second*requeueDurationSeconds, nil).Times(1) + + machineFactoryMock.EXPECT().NewMachine(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(machineMock, nil).Times(1) + + setupClient(machineFactoryMock, objects) + + infraClusterMock.EXPECT().GenerateInfraClusterClient(kubevirtMachine.Spec.InfraClusterSecretRef, kubevirtMachine.Namespace, machineContext.Context).Return(fakeClient, kubevirtMachine.Namespace, nil) + + res, err := kubevirtMachineReconciler.reconcileNormal(machineContext) + Expect(err).ShouldNot(HaveOccurred()) + + Expect(res.RequeueAfter).To(Equal(time.Second * requeueDurationSeconds)) + }) + + It("should requeue on node draining error + requeue duration", func() { + vmiReadyCondition := kubevirtv1.VirtualMachineInstanceCondition{ + Type: kubevirtv1.VirtualMachineInstanceReady, + Status: corev1.ConditionTrue, + } + vmi.Status.Conditions = append(vmi.Status.Conditions, vmiReadyCondition) + vmi.Status.Interfaces = []kubevirtv1.VirtualMachineInstanceNetworkInterface{ + + { + IP: "1.1.1.1", + }, + } + sshKeySecret.Data["pub"] = []byte("shell") + + objects := []client.Object{ + cluster, + kubevirtCluster, + machine, + kubevirtMachine, + bootstrapSecret, + bootstrapUserDataSecret, + sshKeySecret, + vm, + vmi, + } + + const requeueDurationSeconds = 3 + machineMock.EXPECT().IsTerminal().Return(false, "", nil).Times(1) + machineMock.EXPECT().Exists().Return(true).Times(1) + machineMock.EXPECT().IsReady().Return(true).Times(1) + machineMock.EXPECT().Address().Return("1.1.1.1").Times(1) + machineMock.EXPECT().DrainNodeIfNeeded(gomock.Any()).Return(time.Second*requeueDurationSeconds, fmt.Errorf("mock error")).Times(1) + + machineFactoryMock.EXPECT().NewMachine(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(machineMock, nil).Times(1) + + setupClient(machineFactoryMock, objects) + + infraClusterMock.EXPECT().GenerateInfraClusterClient(kubevirtMachine.Spec.InfraClusterSecretRef, kubevirtMachine.Namespace, machineContext.Context).Return(fakeClient, kubevirtMachine.Namespace, nil) + + res, err := kubevirtMachineReconciler.reconcileNormal(machineContext) + Expect(err).Should(HaveOccurred()) + Expect(errors.Unwrap(err).Error()).Should(ContainSubstring("failed to drain node: mock error")) + + Expect(res.RequeueAfter).To(Equal(time.Second * requeueDurationSeconds)) + }) }) }) It("should detect when a previous Ready KubeVirtMachine is no longer ready due to vmi ready condition being false", func() { diff --git a/controllers/vmi_eviction_controller.go b/controllers/vmi_eviction_controller.go deleted file mode 100644 index 3f91eb0b..00000000 --- a/controllers/vmi_eviction_controller.go +++ /dev/null @@ -1,276 +0,0 @@ -package controllers - -import ( - goContext "context" - "fmt" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "time" - - "github.com/go-logr/logr" - "github.com/pkg/errors" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubedrain "k8s.io/kubectl/pkg/drain" - kubevirtv1 "kubevirt.io/api/core/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/noderefutil" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - infrav1 "sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha1" - context "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" - "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster" -) - -const ( - vmiDeleteGraceTimeoutDurationSeconds = 600 // 10 minutes -) - -type VmiEvictionReconciler struct { - client.Client - workloadCluster workloadcluster.WorkloadCluster -} - -// NewVmiEvictionReconciler creates a new VmiEvictionReconciler -func NewVmiEvictionReconciler(cl client.Client) *VmiEvictionReconciler { - return &VmiEvictionReconciler{Client: cl, workloadCluster: workloadcluster.New(cl)} -} - -// SetupWithManager will add watches for this controller. -func (r *VmiEvictionReconciler) SetupWithManager(ctx goContext.Context, mgr ctrl.Manager) error { - selector, err := getLabelPredicate() - - if err != nil { - return fmt.Errorf("can't setup the VMI eviction controller; %w", err) - } - - _, err = ctrl.NewControllerManagedBy(mgr). - For(&kubevirtv1.VirtualMachineInstance{}). - WithEventFilter(selector). - Build(r) - - return err -} - -// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;machines,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=secrets;,verbs=get;list;watch -// +kubebuilder:rbac:groups=kubevirt.io,resources=virtualmachineinstances;,verbs=get;list;watch;patch;update;delete - -// Reconcile handles VMI events. -func (r VmiEvictionReconciler) Reconcile(ctx goContext.Context, req ctrl.Request) (ctrl.Result, error) { - logger := ctrl.LoggerFrom(ctx) - - vmi := &kubevirtv1.VirtualMachineInstance{} - err := r.Get(ctx, req.NamespacedName, vmi) - if err != nil { - if apierrors.IsNotFound(err) { - logger.V(4).Info(fmt.Sprintf("Can't find virtualMachineInstance %s; it was already deleted.", req.NamespacedName)) - return ctrl.Result{}, nil - } - logger.Error(err, fmt.Sprintf("failed to read VMI %s", req.Name)) - return ctrl.Result{}, err - } - - if !shouldGracefulDeleteVMI(vmi, logger, req.NamespacedName) { - return ctrl.Result{}, nil - } - - exceeded, err := r.drainGracePeriodExceeded(ctx, vmi, logger) - if err != nil { - return ctrl.Result{}, err - } - - if !exceeded { - cluster, err := r.getCluster(ctx, vmi) - if err != nil { - logger.Error(err, "Can't get the cluster form the VirtualMachineInstance", "VirtualMachineInstance name", req.NamespacedName) - return ctrl.Result{}, err - } - - nodeDrained, retryDuration, err := r.drainNode(ctx, cluster, vmi.Status.EvacuationNodeName, logger) - if err != nil { - return ctrl.Result{RequeueAfter: retryDuration}, err - } - - if !nodeDrained { - return ctrl.Result{RequeueAfter: retryDuration}, nil - } - } - - // now, when the node is drained (or vmiDeleteGraceTimeoutDurationSeconds has passed), we can delete the VMI - propagationPolicy := metav1.DeletePropagationForeground - err = r.Delete(ctx, vmi, &client.DeleteOptions{PropagationPolicy: &propagationPolicy}) - if err != nil { - logger.Error(err, "failed to delete VirtualMachineInstance", "VirtualMachineInstance name", req.NamespacedName) - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func shouldGracefulDeleteVMI(vmi *kubevirtv1.VirtualMachineInstance, logger logr.Logger, namespacedName types.NamespacedName) bool { - if vmi.DeletionTimestamp != nil { - logger.V(4).Info("The virtualMachineInstance is already in deletion process. Nothing to do here", "VirtualMachineInstance name", namespacedName) - return false - } - - if vmi.Spec.EvictionStrategy == nil || *vmi.Spec.EvictionStrategy != kubevirtv1.EvictionStrategyExternal { - logger.V(4).Info("Graceful deletion is not supported for virtualMachineInstance. Nothing to do here", "VirtualMachineInstance name", namespacedName) - return false - } - - // KubeVirt will set the EvacuationNodeName field in case of guest node eviction. If the field is not set, there is - // nothing to do. - if len(vmi.Status.EvacuationNodeName) == 0 { - logger.V(4).Info("The virtualMachineInstance is not marked for deletion. Nothing to do here", "VirtualMachineInstance name", namespacedName) - return false - } - - return true -} - -func (r VmiEvictionReconciler) getCluster(ctx goContext.Context, vmi *kubevirtv1.VirtualMachineInstance) (*clusterv1.Cluster, error) { - // get cluster from vmi - clusterNS, ok := vmi.Labels[infrav1.KubevirtMachineNamespaceLabel] - if !ok { - return nil, fmt.Errorf("can't find the cluster namespace from the VM; missing %s label", infrav1.KubevirtMachineNamespaceLabel) - } - - clusterName, ok := vmi.Labels[clusterv1.ClusterLabelName] - if !ok { - return nil, fmt.Errorf("can't find the cluster name from the VM; missing %s label", clusterv1.ClusterLabelName) - } - - cluster := &clusterv1.Cluster{} - err := r.Get(ctx, client.ObjectKey{Namespace: clusterNS, Name: clusterName}, cluster) - if err != nil { - return nil, fmt.Errorf("can't find the cluster %s/%s; %w", clusterNS, clusterName, err) - } - - return cluster, nil -} - -// This functions drains a node from a tenant cluster. -// The function returns 3 values: -// * drain done - boolean -// * retry time, or 0 if not needed -// * error - to be returned if we want to retry -func (r VmiEvictionReconciler) drainNode(goctx goContext.Context, cluster *clusterv1.Cluster, nodeName string, logger logr.Logger) (bool, time.Duration, error) { - ctx := &context.MachineContext{Context: goctx, KubevirtCluster: &infrav1.KubevirtCluster{ObjectMeta: metav1.ObjectMeta{Namespace: cluster.Namespace, Name: cluster.Name}}} - kubeClient, err := r.workloadCluster.GenerateWorkloadClusterK8sClient(ctx) - if err != nil { - logger.Error(err, "Error creating a remote client while deleting Machine, won't retry") - return false, 0, nil - } - - node, err := kubeClient.CoreV1().Nodes().Get(goctx, nodeName, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - // If an admin deletes the node directly, we'll end up here. - logger.Error(err, "Could not find node from noderef, it may have already been deleted") - return true, 0, nil - } - return false, 0, fmt.Errorf("unable to get node %q: %w", nodeName, err) - } - - drainer := &kubedrain.Helper{ - Client: kubeClient, - Ctx: ctx, - Force: true, - IgnoreAllDaemonSets: true, - DeleteEmptyDirData: true, - GracePeriodSeconds: -1, - // If a pod is not evicted in 20 seconds, retry the eviction next time the - // machine gets reconciled again (to allow other machines to be reconciled). - Timeout: 20 * time.Second, - OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) { - verbStr := "Deleted" - if usingEviction { - verbStr = "Evicted" - } - logger.Info(fmt.Sprintf("%s pod from Node", verbStr), - "pod", fmt.Sprintf("%s/%s", pod.Name, pod.Namespace)) - }, - Out: writer{logger.Info}, - ErrOut: writer{func(msg string, keysAndValues ...interface{}) { - logger.Error(nil, msg, keysAndValues...) - }}, - } - - if noderefutil.IsNodeUnreachable(node) { - // When the node is unreachable and some pods are not evicted for as long as this timeout, we ignore them. - drainer.SkipWaitForDeleteTimeoutSeconds = 60 * 5 // 5 minutes - } - - if err = kubedrain.RunCordonOrUncordon(drainer, node, true); err != nil { - // Machine will be re-reconciled after a cordon failure. - logger.Error(err, "Cordon failed") - return false, 0, errors.Errorf("unable to cordon node %s: %v", nodeName, err) - } - - if err = kubedrain.RunNodeDrain(drainer, node.Name); err != nil { - // Machine will be re-reconciled after a drain failure. - logger.Error(err, "Drain failed, retry in 20s", "node name", nodeName) - return false, 20 * time.Second, nil - } - - logger.Info("Drain successful", "node name", nodeName) - return true, 0, nil -} - -// wait vmiDeleteGraceTimeoutDurationSeconds to the node to be drained. If this time had passed, don't wait anymore. -func (r VmiEvictionReconciler) drainGracePeriodExceeded(ctx goContext.Context, vmi *kubevirtv1.VirtualMachineInstance, logger logr.Logger) (bool, error) { - if graceTime, found := vmi.Annotations[infrav1.VmiDeletionGraceTime]; found { - deletionGraceTime, err := time.Parse(time.RFC3339, graceTime) - if err != nil { // wrong format - rewrite - if err = r.setVmiDeletionGraceTime(ctx, vmi, logger); err != nil { - return false, err - } - } else { - return time.Now().UTC().After(deletionGraceTime), nil - } - } else { - if err := r.setVmiDeletionGraceTime(ctx, vmi, logger); err != nil { - return false, err - } - } - - return false, nil -} - -func (r VmiEvictionReconciler) setVmiDeletionGraceTime(ctx goContext.Context, vmi *kubevirtv1.VirtualMachineInstance, logger logr.Logger) error { - logger.V(2).Info(fmt.Sprintf("setting the %s annotation", infrav1.VmiDeletionGraceTime)) - graceTime := time.Now().Add(vmiDeleteGraceTimeoutDurationSeconds * time.Second).UTC().Format(time.RFC3339) - patch := fmt.Sprintf(`{"metadata":{"annotations":{"%s": "%s"}}}`, infrav1.VmiDeletionGraceTime, graceTime) - patchRequest := client.RawPatch(types.MergePatchType, []byte(patch)) - - if err := r.Patch(ctx, vmi, patchRequest); err != nil { - return fmt.Errorf("failed to add the %s annotation to the VMI; %w", infrav1.VmiDeletionGraceTime, err) - } - - return nil -} - -func getLabelPredicate() (predicate.Predicate, error) { - return predicate.LabelSelectorPredicate( - metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{{ - Key: infrav1.KubevirtMachineNameLabel, - Operator: metav1.LabelSelectorOpExists, - Values: nil, - }}, - }) -} - -// writer implements io.Writer interface as a pass-through for klog. -type writer struct { - logFunc func(msg string, keysAndValues ...interface{}) -} - -// Write passes string(p) into writer's logFunc and always returns len(p). -func (w writer) Write(p []byte) (n int, err error) { - w.logFunc(string(p)) - return len(p), nil -} diff --git a/controllers/vmi_eviction_controller_test.go b/controllers/vmi_eviction_controller_test.go deleted file mode 100644 index 9940128d..00000000 --- a/controllers/vmi_eviction_controller_test.go +++ /dev/null @@ -1,466 +0,0 @@ -package controllers - -import ( - gocontext "context" - "errors" - "time" - - "github.com/golang/mock/gomock" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - k8sfake "k8s.io/client-go/kubernetes/fake" - kubevirtv1 "kubevirt.io/api/core/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/event" - - infrav1 "sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha1" - "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/testing" - "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster/mock" -) - -var _ = Describe("Test VMI Controller", func() { - - const ( - clusterName = "test" - clusterNamespace = clusterName + "-cluster" - clusterInstanceName = clusterName + "-1234" - nodeName = "worker-node-1" - ) - - Context("Test VmiEviction reconciler", func() { - var ( - mockCtrl *gomock.Controller - fakeClient client.Client - vmi *kubevirtv1.VirtualMachineInstance - cluster *clusterv1.Cluster - wlCluster *mock.MockWorkloadCluster - ) - - BeforeEach(func() { - mockCtrl = gomock.NewController(GinkgoT()) - - vmi = &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test-cluster", - Name: nodeName, - Labels: map[string]string{ - infrav1.KubevirtMachineNamespaceLabel: clusterNamespace, - clusterv1.ClusterLabelName: clusterInstanceName, - }, - Annotations: make(map[string]string), - }, - } - - cluster = &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: clusterNamespace, - Name: clusterInstanceName, - }, - Spec: clusterv1.ClusterSpec{ - InfrastructureRef: &corev1.ObjectReference{ - Kind: "Secret", - Namespace: clusterNamespace, - Name: clusterInstanceName, - }, - }, - } - - wlCluster = mock.NewMockWorkloadCluster(mockCtrl) - }) - - It("Should ignore vmi if it already deleted", func() { - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).Build() - - // make sure we never get into darin process, but exit earlier - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - - Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) - }) - - It("Should ignore vmi if its deletion process already started", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - now := metav1.Now() - vmi.DeletionTimestamp = &now - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - // make sure we never get into darin process, but exit earlier - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - - Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) - }) - - It("Should ignore vmi with no eviction strategy", func() { - vmi.Spec.EvictionStrategy = nil - vmi.Status.EvacuationNodeName = nodeName - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - // make sure we never get into darin process, but exit earlier - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - - Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) - }) - - It("Should ignore vmi with no eviction strategy != external", func() { - es := kubevirtv1.EvictionStrategyLiveMigrate - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - // make sure we never get into darin process, but exit earlier - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - - Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) - }) - - It("Should ignore non-evicted VMIs", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - // make sure we never get into darin process, but exit earlier - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - - Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) - }) - - It("Should drain node", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - } - - Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) - cl := k8sfake.NewSimpleClientset(node) - - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Return(cl, nil).Times(1) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - - Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) - - // check that the node was drained - readNode, err := cl.CoreV1().Nodes().Get(gocontext.TODO(), nodeName, metav1.GetOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(readNode.Spec.Unschedulable).To(BeTrue()) - - // check that the VMI was removed - readVMI := &kubevirtv1.VirtualMachineInstance{} - err = fakeClient.Get(gocontext.TODO(), client.ObjectKey{Namespace: clusterNamespace, Name: nodeName}, readVMI) - Expect(apierrors.IsNotFound(err)).Should(BeTrue()) - }) - - It("Should skip drain if the node already deleted", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - - vmi.Status.EvacuationNodeName = nodeName - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) - cl := k8sfake.NewSimpleClientset() - - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Return(cl, nil).Times(1) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - - Expect(r.Reconcile(gocontext.TODO(), req)).Should(Equal(ctrl.Result{})) - }) - - Context("Error cases", func() { - It("Should return error if the 'capk.cluster.x-k8s.io/kubevirt-machine-namespace' label is missing", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - delete(vmi.Labels, infrav1.KubevirtMachineNamespaceLabel) - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - // make sure we never get into darin process, but exit earlier - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - _, err := r.Reconcile(gocontext.TODO(), req) - Expect(err).Should(HaveOccurred()) - }) - - It("Should return error if the 'cluster.x-k8s.io/cluster-name' label is missing", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - delete(vmi.Labels, clusterv1.ClusterLabelName) - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - // make sure we never get into darin process, but exit earlier - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - _, err := r.Reconcile(gocontext.TODO(), req) - Expect(err).Should(HaveOccurred()) - }) - - It("Should return error if the cluster is missing", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi).Build() - - // make sure we never get into darin process, but exit earlier - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - _, err := r.Reconcile(gocontext.TODO(), req) - Expect(err).Should(HaveOccurred()) - }) - - It("Should return not error if can't get the external cluster client, but do not remove the VMI", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - - Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) - - node := &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeName, - }, - } - - Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) - cl := k8sfake.NewSimpleClientset(node) - - wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Return(nil, errors.New("fake error")).Times(1) - - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - req := ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "test-cluster", Name: nodeName}} - - _, err := r.Reconcile(gocontext.TODO(), req) - Expect(err).ShouldNot(HaveOccurred()) - - // check that the node was not drained - readNode, err := cl.CoreV1().Nodes().Get(gocontext.TODO(), nodeName, metav1.GetOptions{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(readNode.Spec.Unschedulable).To(BeFalse()) - - // check that the VMI was not deleted - readVMI := &kubevirtv1.VirtualMachineInstance{} - Expect( - fakeClient.Get(gocontext.TODO(), client.ObjectKey{Namespace: clusterNamespace, Name: nodeName}, readVMI), - ).To(Succeed()) - Expect(readVMI).ToNot(BeNil()) - }) - }) - - Context("test drainGracePeriodExceeded", func() { - It("should add the annotation", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - ctx := gocontext.Background() - Expect(r.drainGracePeriodExceeded(ctx, vmi, ctrl.LoggerFrom(ctx))).To(BeFalse()) - timeoutAnnotation, found := vmi.Annotations[infrav1.VmiDeletionGraceTime] - Expect(found).To(BeTrue()) - timeout, err := time.Parse(time.RFC3339, timeoutAnnotation) - Expect(err).ToNot(HaveOccurred()) - Expect(timeout).To(And( - BeTemporally(">", time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds-1)*time.Second)), - BeTemporally("<", time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds+1)*time.Second)))) - }) - - It("should return false if timeout was not exceeded", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - - timeout := time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds / 2) * time.Second).Format(time.RFC3339) - vmi.Annotations[infrav1.VmiDeletionGraceTime] = timeout - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - ctx := gocontext.Background() - Expect(r.drainGracePeriodExceeded(ctx, vmi, ctrl.LoggerFrom(ctx))).To(BeFalse()) - timeoutAnnotation, found := vmi.Annotations[infrav1.VmiDeletionGraceTime] - Expect(found).To(BeTrue()) - Expect(timeoutAnnotation).To(Equal(timeout)) - }) - - It("should return true if timeout was exceeded", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - - timeout := time.Now().UTC().Add(-(time.Millisecond)).Format(time.RFC3339) - vmi.Annotations[infrav1.VmiDeletionGraceTime] = timeout - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - ctx := gocontext.Background() - Expect(r.drainGracePeriodExceeded(ctx, vmi, ctrl.LoggerFrom(ctx))).To(BeTrue()) - timeoutAnnotation, found := vmi.Annotations[infrav1.VmiDeletionGraceTime] - Expect(found).To(BeTrue()) - Expect(timeoutAnnotation).To(Equal(timeout)) - }) - - It("should fix the annotation if it's with a wrong format", func() { - es := kubevirtv1.EvictionStrategyExternal - vmi.Spec.EvictionStrategy = &es - vmi.Status.EvacuationNodeName = nodeName - - origTimeout := time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds / 2) * time.Second).Format(time.RFC850) - vmi.Annotations[infrav1.VmiDeletionGraceTime] = origTimeout - - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(vmi, cluster).Build() - r := &VmiEvictionReconciler{Client: fakeClient, workloadCluster: wlCluster} - ctx := gocontext.Background() - Expect(r.drainGracePeriodExceeded(ctx, vmi, ctrl.LoggerFrom(ctx))).To(BeFalse()) - timeoutAnnotation, found := vmi.Annotations[infrav1.VmiDeletionGraceTime] - Expect(found).To(BeTrue()) - timeout, err := time.Parse(time.RFC3339, timeoutAnnotation) - Expect(err).ToNot(HaveOccurred()) - Expect(timeout).To(And( - BeTemporally(">", time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds-1)*time.Second)), - BeTemporally("<", time.Now().UTC().Add((vmiDeleteGraceTimeoutDurationSeconds+1)*time.Second)))) - }) - }) - }) - - Context("check the label predicate", func() { - sel, err := getLabelPredicate() - It("should successfully create the predicate", func() { - Expect(err).ToNot(HaveOccurred()) - }) - - It("should select if the label exist", func() { - Expect(sel.Create(event.CreateEvent{ - Object: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, - }, - }, - })).To(BeTrue()) - }) - - It("should select if the label exist and empty", func() { - Expect(sel.Create(event.CreateEvent{ - Object: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{infrav1.KubevirtMachineNameLabel: ""}, - }, - }, - })).To(BeTrue()) - }) - - It("should select if the label does not exist", func() { - Expect(sel.Create(event.CreateEvent{ - Object: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: nil, - }, - }, - })).To(BeFalse()) - }) - - It("should select if the label exist", func() { - Expect(sel.Update(event.UpdateEvent{ - ObjectOld: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, - }, - }, - ObjectNew: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, - }, - }, - })).To(BeTrue()) - }) - - It("should select if the label now exist", func() { - Expect(sel.Update(event.UpdateEvent{ - ObjectOld: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"foo": "bar"}, - }, - }, - ObjectNew: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, - }, - }, - })).To(BeTrue()) - - }) - - It("should select if the label now not exist", func() { - Expect(sel.Update(event.UpdateEvent{ - ObjectOld: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{infrav1.KubevirtMachineNameLabel: "machine-name"}, - }, - }, - ObjectNew: &kubevirtv1.VirtualMachineInstance{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"foo": "bar"}, - }, - }, - })).To(BeFalse()) - }) - }) - -}) - -func setupRemoteScheme() *runtime.Scheme { - s := runtime.NewScheme() - if err := corev1.AddToScheme(s); err != nil { - panic(err) - } - return s -} diff --git a/e2e/create-cluster_test.go b/e2e/create-cluster_test.go index 821fdfbd..b97a6b39 100644 --- a/e2e/create-cluster_test.go +++ b/e2e/create-cluster_test.go @@ -1,9 +1,11 @@ package e2e_test import ( + "bytes" "context" "fmt" "io/ioutil" + "k8s.io/apimachinery/pkg/util/json" "os" "os/exec" "path/filepath" @@ -201,6 +203,12 @@ var _ = Describe("CreateCluster", func() { } waitForMachineReadiness := func(numExpectedReady int, numExpectedNotReady int) { + // debug + buff := &bytes.Buffer{} + enc := json.NewEncoder(buff) + enc.SetIndent("", " ") + // + Eventually(func(g Gomega) { readyCount := 0 notReadyCount := 0 @@ -217,11 +225,14 @@ var _ = Describe("CreateCluster", func() { notReadyCount++ } } - - g.Expect(readyCount).To(Equal(numExpectedReady), "Expected %d ready, but got %d", numExpectedReady, readyCount) + // debug + buff.Reset() + _ = enc.Encode(machineList.Items) + // + g.Expect(readyCount).To(Equal(numExpectedReady), "Expected %d ready, but got %d, machineList.Items:\n %s\n", numExpectedReady, readyCount, buff.String()) g.Expect(notReadyCount).To(Equal(numExpectedNotReady), "Expected %d not ready, but got %d", numExpectedNotReady, notReadyCount) }).WithOffset(1). - WithTimeout(5*time.Minute). + WithTimeout(10*time.Minute). WithPolling(5*time.Second). Should(Succeed(), "waiting for expected readiness.") } @@ -892,7 +903,7 @@ func waitForVMIDraining(vmiName, namespace string) { var err error By("wait for VMI is marked for deletion") - Eventually(func(g Gomega) bool { + Eventually(func(g Gomega) { vmi, err = virtClient.VirtualMachineInstance(namespace).Get(vmiName, &metav1.GetOptions{}) g.Expect(err).ShouldNot(HaveOccurred()) @@ -900,12 +911,10 @@ func waitForVMIDraining(vmiName, namespace string) { g.Expect(vmi.Status.EvacuationNodeName).ShouldNot(BeEmpty()) g.Expect(vmi.DeletionTimestamp).ShouldNot(BeNil()) - - return true }).WithOffset(1). - WithTimeout(time.Minute * 2). - WithPolling(time.Second). - Should(BeTrue()) + WithTimeout(time.Minute * 5). + WithPolling(time.Second * 5). + Should(Succeed()) } func evictNode(pod *corev1.Pod) { diff --git a/main.go b/main.go index c67e9407..d8046981 100644 --- a/main.go +++ b/main.go @@ -172,11 +172,6 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { setupLog.Error(err, "unable to create controller", "controller", "KubevirtCluster") os.Exit(1) } - - if err := (controllers.NewVmiEvictionReconciler(mgr.GetClient())).SetupWithManager(ctx, mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "VirtualMachineInstance") - os.Exit(1) - } } func setupWebhooks(mgr ctrl.Manager) { diff --git a/pkg/kubevirt/machine.go b/pkg/kubevirt/machine.go index 18b77736..2c111c2b 100644 --- a/pkg/kubevirt/machine.go +++ b/pkg/kubevirt/machine.go @@ -19,21 +19,29 @@ package kubevirt import ( gocontext "context" "fmt" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + kubedrain "k8s.io/kubectl/pkg/drain" kubevirtv1 "kubevirt.io/api/core/v1" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/noderefutil" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "time" infrav1 "sigs.k8s.io/cluster-api-provider-kubevirt/api/v1alpha1" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/ssh" ) +const ( + vmiDeleteGraceTimeoutDurationSeconds = 600 // 10 minutes +) + // Machine implement a service for managing the KubeVirt VM hosting a kubernetes node. type Machine struct { client client.Client @@ -85,7 +93,7 @@ func NewMachine(ctx *context.MachineContext, client client.Client, namespace str return machine, nil } -// Reports back if the VM is either being requested to terminate or is terminate +// IsTerminal Reports back if the VM is either being requested to terminate or is terminate // in a way that it will never recover from. func (m *Machine) IsTerminal() (bool, string, error) { if m.vmInstance == nil || m.vmiInstance == nil { @@ -287,3 +295,173 @@ func (m *Machine) Delete() error { return nil } + +func (m *Machine) DrainNodeIfNeeded(wrkldClstr workloadcluster.WorkloadCluster) (time.Duration, error) { + if m.vmiInstance == nil { + return 0, nil + } + + if !m.shouldGracefulDeleteVMI() { + return 0, nil + } + + exceeded, err := m.drainGracePeriodExceeded() + if err != nil { + return 0, err + } + + if !exceeded { + retryDuration, err := m.drainNode(wrkldClstr) + if err != nil { + return 0, err + } + + if retryDuration > 0 { + return retryDuration, nil + } + } + + // now, when the node is drained (or vmiDeleteGraceTimeoutDurationSeconds has passed), we can delete the VMI + propagationPolicy := metav1.DeletePropagationForeground + err = m.client.Delete(m.machineContext, m.vmiInstance, &client.DeleteOptions{PropagationPolicy: &propagationPolicy}) + if err != nil { + m.machineContext.Logger.Error(err, "failed to delete VirtualMachineInstance") + return 0, err + } + + // requeue to force reading the VMI again + return time.Second * 10, nil +} + +func (m *Machine) shouldGracefulDeleteVMI() bool { + if m.vmiInstance.DeletionTimestamp != nil { + m.machineContext.Logger.V(4).Info("DrainNode: the virtualMachineInstance is already in deletion process. Nothing to do here") + return false + } + + if m.vmiInstance.Spec.EvictionStrategy == nil || *m.vmiInstance.Spec.EvictionStrategy != kubevirtv1.EvictionStrategyExternal { + m.machineContext.Logger.V(4).Info("DrainNode: graceful deletion is not supported for virtualMachineInstance. Nothing to do here") + return false + } + + // KubeVirt will set the EvacuationNodeName field in case of guest node eviction. If the field is not set, there is + // nothing to do. + if len(m.vmiInstance.Status.EvacuationNodeName) == 0 { + m.machineContext.Logger.V(4).Info("DrainNode: the virtualMachineInstance is not marked for deletion. Nothing to do here") + return false + } + + return true +} + +// wait vmiDeleteGraceTimeoutDurationSeconds to the node to be drained. If this time had passed, don't wait anymore. +func (m *Machine) drainGracePeriodExceeded() (bool, error) { + if graceTime, found := m.vmiInstance.Annotations[infrav1.VmiDeletionGraceTime]; found { + deletionGraceTime, err := time.Parse(time.RFC3339, graceTime) + if err != nil { // wrong format - rewrite + if err = m.setVmiDeletionGraceTime(); err != nil { + return false, err + } + } else { + return time.Now().UTC().After(deletionGraceTime), nil + } + } else { + if err := m.setVmiDeletionGraceTime(); err != nil { + return false, err + } + } + + return false, nil +} + +func (m *Machine) setVmiDeletionGraceTime() error { + m.machineContext.Logger.V(2).Info(fmt.Sprintf("setting the %s annotation", infrav1.VmiDeletionGraceTime)) + graceTime := time.Now().Add(vmiDeleteGraceTimeoutDurationSeconds * time.Second).UTC().Format(time.RFC3339) + patch := fmt.Sprintf(`{"metadata":{"annotations":{"%s": "%s"}}}`, infrav1.VmiDeletionGraceTime, graceTime) + patchRequest := client.RawPatch(types.MergePatchType, []byte(patch)) + + if err := m.client.Patch(m.machineContext, m.vmiInstance, patchRequest); err != nil { + return fmt.Errorf("failed to add the %s annotation to the VMI; %w", infrav1.VmiDeletionGraceTime, err) + } + + return nil +} + +// This functions drains a node from a tenant cluster. +// The function returns 3 values: +// * drain done - boolean +// * retry time, or 0 if not needed +// * error - to be returned if we want to retry +func (m *Machine) drainNode(wrkldClstr workloadcluster.WorkloadCluster) (time.Duration, error) { + kubeClient, err := wrkldClstr.GenerateWorkloadClusterK8sClient(m.machineContext) + if err != nil { + m.machineContext.Logger.Error(err, "Error creating a remote client while deleting Machine, won't retry") + return 0, nil + } + + nodeName := m.vmiInstance.Status.EvacuationNodeName + node, err := kubeClient.CoreV1().Nodes().Get(m.machineContext, nodeName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // If an admin deletes the node directly, we'll end up here. + m.machineContext.Logger.Error(err, "Could not find node from noderef, it may have already been deleted") + return 0, nil + } + return 0, fmt.Errorf("unable to get node %q: %w", nodeName, err) + } + + drainer := &kubedrain.Helper{ + Client: kubeClient, + Ctx: m.machineContext, + Force: true, + IgnoreAllDaemonSets: true, + DeleteEmptyDirData: true, + GracePeriodSeconds: -1, + // If a pod is not evicted in 20 seconds, retry the eviction next time the + // machine gets reconciled again (to allow other machines to be reconciled). + Timeout: 20 * time.Second, + OnPodDeletedOrEvicted: func(pod *corev1.Pod, usingEviction bool) { + verbStr := "Deleted" + if usingEviction { + verbStr = "Evicted" + } + m.machineContext.Logger.Info(fmt.Sprintf("%s pod from Node", verbStr), + "pod", fmt.Sprintf("%s/%s", pod.Name, pod.Namespace)) + }, + Out: writer{m.machineContext.Logger.Info}, + ErrOut: writer{func(msg string, keysAndValues ...interface{}) { + m.machineContext.Logger.Error(nil, msg, keysAndValues...) + }}, + } + + if noderefutil.IsNodeUnreachable(node) { + // When the node is unreachable and some pods are not evicted for as long as this timeout, we ignore them. + drainer.SkipWaitForDeleteTimeoutSeconds = 60 * 5 // 5 minutes + } + + if err = kubedrain.RunCordonOrUncordon(drainer, node, true); err != nil { + // Machine will be re-reconciled after a cordon failure. + m.machineContext.Logger.Error(err, "Cordon failed") + return 0, errors.Errorf("unable to cordon node %s: %v", nodeName, err) + } + + if err = kubedrain.RunNodeDrain(drainer, node.Name); err != nil { + // Machine will be re-reconciled after a drain failure. + m.machineContext.Logger.Error(err, "Drain failed, retry in a second", "node name", nodeName) + return time.Second, nil + } + + m.machineContext.Logger.Info("Drain successful", "node name", nodeName) + return 0, nil +} + +// writer implements io.Writer interface as a pass-through for klog. +type writer struct { + logFunc func(msg string, keysAndValues ...interface{}) +} + +// Write passes string(p) into writer's logFunc and always returns len(p). +func (w writer) Write(p []byte) (n int, err error) { + w.logFunc(string(p)) + return len(p), nil +} diff --git a/pkg/kubevirt/machine_factory.go b/pkg/kubevirt/machine_factory.go index acf8f431..f4ddc2bc 100644 --- a/pkg/kubevirt/machine_factory.go +++ b/pkg/kubevirt/machine_factory.go @@ -1,16 +1,17 @@ package kubevirt -//go:generate mockgen -source=./machine_factory.go -destination=./mock/machine_factory_generated.go -package=mock +//go:generate mockgen -source=./machine_factory.go -destination=./mock/machine_factory_generated.go -package=mock_kubevirt import ( gocontext "context" + "time" + "github.com/pkg/errors" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/ssh" - - "github.com/pkg/errors" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster" ) // MachineInterface abstracts the functions that the kubevirt.machine interface implements. @@ -35,6 +36,8 @@ type MachineInterface interface { GenerateProviderID() (string, error) // IsTerminal reports back if a VM is in a permanent terminal state IsTerminal() (bool, string, error) + + DrainNodeIfNeeded(workloadcluster.WorkloadCluster) (time.Duration, error) } // MachineFactory allows creating new instances of kubevirt.machine diff --git a/pkg/kubevirt/machine_test.go b/pkg/kubevirt/machine_test.go index 6d27e5e5..9815f9e8 100644 --- a/pkg/kubevirt/machine_test.go +++ b/pkg/kubevirt/machine_test.go @@ -19,6 +19,11 @@ package kubevirt import ( gocontext "context" "fmt" + "github.com/golang/mock/gomock" + "k8s.io/apimachinery/pkg/runtime" + k8sfake "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster/mock" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -218,6 +223,10 @@ var _ = Describe("With KubeVirt VM running", func() { Status: corev1.ConditionTrue, }, } + + fakeVMCommandExecutor = FakeVMCommandExecutor{true} + }) + JustBeforeEach(func() { objects := []client.Object{ cluster, kubevirtCluster, @@ -226,10 +235,7 @@ var _ = Describe("With KubeVirt VM running", func() { virtualMachineInstance, virtualMachine, } - fakeClient = fake.NewClientBuilder().WithScheme(testing.SetupScheme()).WithObjects(objects...).Build() - - fakeVMCommandExecutor = FakeVMCommandExecutor{true} }) AfterEach(func() {}) @@ -311,6 +317,128 @@ var _ = Describe("With KubeVirt VM running", func() { Expect(externalMachine.Delete()).To(Succeed()) validateVMNotExist(virtualMachine, fakeClient, machineContext) }) + + Context("test DrainNodeIfNeeded", func() { + const nodeName = "control-plane1" + + var ( + wlCluster *mock.MockWorkloadCluster + ) + + BeforeEach(func() { + virtualMachineInstance = testing.NewVirtualMachineInstance(kubevirtMachine) + strategy := kubevirtv1.EvictionStrategyExternal + virtualMachineInstance.Spec.EvictionStrategy = &strategy + virtualMachineInstance.Status.EvacuationNodeName = nodeName + if virtualMachineInstance.Annotations == nil { + virtualMachineInstance.Annotations = make(map[string]string) + } + + mockCtrl := gomock.NewController(GinkgoT()) + wlCluster = mock.NewMockWorkloadCluster(mockCtrl) + }) + + When("VMI is not evicted", func() { + BeforeEach(func() { + virtualMachineInstance.Spec.EvictionStrategy = nil + virtualMachineInstance.Status.EvacuationNodeName = "" + }) + + It("Should do nothing", func() { + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + externalMachine, err := defaultTestMachine(machineContext, namespace, fakeClient, fakeVMCommandExecutor, []byte(sshKey)) + Expect(err).NotTo(HaveOccurred()) + + requeueDuration, err := externalMachine.DrainNodeIfNeeded(wlCluster) + Expect(err).NotTo(HaveOccurred()) + Expect(requeueDuration).Should(BeZero()) + + vmi := &kubevirtv1.VirtualMachineInstance{} + err = fakeClient.Get(gocontext.Background(), client.ObjectKey{Namespace: virtualMachineInstance.Namespace, Name: virtualMachineInstance.Name}, vmi) + Expect(err).ToNot(HaveOccurred()) + Expect(vmi).ToNot(BeNil()) + }) + }) + + When("VMI is already deleted", func() { + BeforeEach(func() { + deletionTimeStamp := metav1.NewTime(time.Now().UTC().Add(-5 * time.Second)) + virtualMachineInstance.DeletionTimestamp = &deletionTimeStamp + }) + + It("Should do nothing", func() { + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + externalMachine, err := defaultTestMachine(machineContext, namespace, fakeClient, fakeVMCommandExecutor, []byte(sshKey)) + Expect(err).NotTo(HaveOccurred()) + + requeueDuration, err := externalMachine.DrainNodeIfNeeded(wlCluster) + Expect(err).NotTo(HaveOccurred()) + Expect(requeueDuration).Should(BeZero()) + + vmi := &kubevirtv1.VirtualMachineInstance{} + err = fakeClient.Get(gocontext.Background(), client.ObjectKey{Namespace: virtualMachineInstance.Namespace, Name: virtualMachineInstance.Name}, vmi) + Expect(err).ToNot(HaveOccurred()) + Expect(vmi).ToNot(BeNil()) + }) + }) + + When("grace not expired (wrap for BeforeEach)", func() { + BeforeEach(func() { + graceTime := time.Now().UTC().Add(5 * time.Minute).Format(time.RFC3339) + virtualMachineInstance.Annotations[v1alpha1.VmiDeletionGraceTime] = graceTime + }) + + It("Should drain the node", func() { + node := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + }, + } + + Expect(k8sfake.AddToScheme(setupRemoteScheme())).ToNot(HaveOccurred()) + cl := k8sfake.NewSimpleClientset(node) + + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Return(cl, nil).Times(1) + + externalMachine, err := defaultTestMachine(machineContext, namespace, fakeClient, fakeVMCommandExecutor, []byte(sshKey)) + Expect(err).NotTo(HaveOccurred()) + + requeueDuration, err := externalMachine.DrainNodeIfNeeded(wlCluster) + Expect(err).NotTo(HaveOccurred()) + Expect(requeueDuration).To(Equal(time.Duration(10 * time.Second))) + + vmi := &kubevirtv1.VirtualMachineInstance{} + err = fakeClient.Get(gocontext.Background(), client.ObjectKey{Namespace: virtualMachineInstance.Namespace, Name: virtualMachineInstance.Name}, vmi) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + }) + + When("grace not expired (wrap for BeforeEach)", func() { + BeforeEach(func() { + graceTime := time.Now().UTC().Format(time.RFC3339) + virtualMachineInstance.Annotations[v1alpha1.VmiDeletionGraceTime] = graceTime + }) + + It("Should delete the VMI after grace period", func() { + wlCluster.EXPECT().GenerateWorkloadClusterK8sClient(gomock.Any()).Times(0) + + externalMachine, err := defaultTestMachine(machineContext, namespace, fakeClient, fakeVMCommandExecutor, []byte(sshKey)) + Expect(err).NotTo(HaveOccurred()) + + requeueDuration, err := externalMachine.DrainNodeIfNeeded(nil) + Expect(err).NotTo(HaveOccurred()) + Expect(requeueDuration).Should(Equal(10 * time.Second)) + + vmi := &kubevirtv1.VirtualMachineInstance{} + err = fakeClient.Get(gocontext.Background(), client.ObjectKey{Namespace: virtualMachineInstance.Namespace, Name: virtualMachineInstance.Name}, vmi) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + }) + }) }) var _ = Describe("util functions", func() { @@ -573,3 +701,11 @@ func defaultTestMachine(ctx *context.MachineContext, namespace string, client cl return machine, err } + +func setupRemoteScheme() *runtime.Scheme { + s := runtime.NewScheme() + if err := corev1.AddToScheme(s); err != nil { + panic(err) + } + return s +} diff --git a/pkg/kubevirt/mock/machine_factory_generated.go b/pkg/kubevirt/mock/machine_factory_generated.go index 45cba747..0652f0e5 100644 --- a/pkg/kubevirt/mock/machine_factory_generated.go +++ b/pkg/kubevirt/mock/machine_factory_generated.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: pkg/kubevirt/machine_factory.go +// Source: ./machine_factory.go // Package mock_kubevirt is a generated GoMock package. package mock_kubevirt @@ -7,11 +7,13 @@ package mock_kubevirt import ( context "context" reflect "reflect" + time "time" gomock "github.com/golang/mock/gomock" context0 "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/context" kubevirt "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/kubevirt" ssh "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/ssh" + workloadcluster "sigs.k8s.io/cluster-api-provider-kubevirt/pkg/workloadcluster" client "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -80,6 +82,21 @@ func (mr *MockMachineInterfaceMockRecorder) Delete() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockMachineInterface)(nil).Delete)) } +// DrainNodeIfNeeded mocks base method. +func (m *MockMachineInterface) DrainNodeIfNeeded(arg0 workloadcluster.WorkloadCluster) (time.Duration, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DrainNodeIfNeeded", arg0) + ret0, _ := ret[0].(time.Duration) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DrainNodeIfNeeded indicates an expected call of DrainNodeIfNeeded. +func (mr *MockMachineInterfaceMockRecorder) DrainNodeIfNeeded(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DrainNodeIfNeeded", reflect.TypeOf((*MockMachineInterface)(nil).DrainNodeIfNeeded), arg0) +} + // Exists mocks base method. func (m *MockMachineInterface) Exists() bool { m.ctrl.T.Helper() diff --git a/templates/cluster-template-kccm.yaml b/templates/cluster-template-kccm.yaml index cb71cb05..c89233de 100644 --- a/templates/cluster-template-kccm.yaml +++ b/templates/cluster-template-kccm.yaml @@ -175,6 +175,15 @@ metadata: name: cloud-controller-manager namespace: ${NAMESPACE} --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager + namespace: ${NAMESPACE} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -217,6 +226,183 @@ rules: - '*' --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: leader-election-role + namespace: ${NAMESPACE} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + - serviceaccounts + verbs: + - delete + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - delete + - list +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + - machines + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtclusters/status + verbs: + - get + - patch + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtmachines/status + verbs: + - get + - patch + - update +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - delete + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -233,6 +419,39 @@ subjects: name: cloud-controller-manager namespace: ${NAMESPACE} --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: leader-election-rolebinding + namespace: ${NAMESPACE} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: manager + namespace: ${NAMESPACE} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: manager + namespace: ${NAMESPACE} +--- apiVersion: v1 data: cloud-config: | diff --git a/templates/cluster-template-lb-kccm.yaml b/templates/cluster-template-lb-kccm.yaml index c9ccf6a8..e84001a5 100644 --- a/templates/cluster-template-lb-kccm.yaml +++ b/templates/cluster-template-lb-kccm.yaml @@ -175,6 +175,15 @@ metadata: name: cloud-controller-manager namespace: ${NAMESPACE} --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager + namespace: ${NAMESPACE} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -217,6 +226,183 @@ rules: - '*' --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: leader-election-role + namespace: ${NAMESPACE} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + - serviceaccounts + verbs: + - delete + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - delete + - list +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + - machines + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtclusters/status + verbs: + - get + - patch + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtmachines/status + verbs: + - get + - patch + - update +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - delete + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -233,6 +419,39 @@ subjects: name: cloud-controller-manager namespace: ${NAMESPACE} --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: leader-election-rolebinding + namespace: ${NAMESPACE} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: manager + namespace: ${NAMESPACE} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: manager + namespace: ${NAMESPACE} +--- apiVersion: v1 data: cloud-config: | diff --git a/templates/cluster-template-passt-kccm.yaml b/templates/cluster-template-passt-kccm.yaml index f8da9b1a..6de500ad 100644 --- a/templates/cluster-template-passt-kccm.yaml +++ b/templates/cluster-template-passt-kccm.yaml @@ -207,10 +207,12 @@ rules: resources: - virtualmachineinstances verbs: + - delete - get - - watch - list + - patch - update + - watch - apiGroups: - "" resources: diff --git a/templates/cluster-template-persistent-storage-kccm.yaml b/templates/cluster-template-persistent-storage-kccm.yaml index 9d591e55..ec468539 100644 --- a/templates/cluster-template-persistent-storage-kccm.yaml +++ b/templates/cluster-template-persistent-storage-kccm.yaml @@ -199,6 +199,15 @@ metadata: name: cloud-controller-manager namespace: ${NAMESPACE} --- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager + namespace: ${NAMESPACE} +--- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: @@ -241,6 +250,183 @@ rules: - '*' --- apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: leader-election-role + namespace: ${NAMESPACE} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + - serviceaccounts + verbs: + - delete + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - delete + - list +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + - machines + verbs: + - get + - list + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtclusters/status + verbs: + - get + - patch + - update +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - kubevirtmachines/status + verbs: + - get + - patch + - update +- apiGroups: + - kubevirt.io + resources: + - virtualmachineinstances + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kubevirt.io + resources: + - virtualmachines + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - delete + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -257,6 +443,39 @@ subjects: name: cloud-controller-manager namespace: ${NAMESPACE} --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: leader-election-rolebinding + namespace: ${NAMESPACE} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: manager + namespace: ${NAMESPACE} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + capk.cluster.x-k8s.io/template-kind: extra-resource + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: manager + namespace: ${NAMESPACE} +--- apiVersion: v1 data: cloud-config: |