From cc237b85e6811358ce8e06d1aad240f817a69dca Mon Sep 17 00:00:00 2001 From: kmova Date: Fri, 10 Apr 2020 09:17:11 +0000 Subject: [PATCH 1/3] fix(localpv-hostpath):pv deletion caused panic, if node not found After the PV is created and node affinity is set based on kubernetes.io/hostname label, either: - hostname label changed on the node or - the node is deleted from the cluster. This PR fixes the panic, by logging the details of the node hostname that no longer exists in the cluster. The code doesn't force delete the PV, considering that the node might intermittently be out of the cluster. If the user decides that node is not going to come back, then the user can go ahead with force delete of pv using `kubectl delete pv`. An alternate case is that node hostname has really changed. In this case, user will have to perform a manual migration of the localpv-hostpath PV to the new node using the following steps: - Scale down the deployment or sts using the PVC. - Save the PVC and PV yamls files. - Delete the PVC and PV - Modify the saved PV yaml will the node hostname and apply Note: when re-applying the yamls, the uuid of pv and pvc objects will change, so the metadata around self-uuid,etc needs to be cleared. - Modify the saved PVC yaml for stale references and re-apply. - Update the PV yaml with the uuid of the newly created PVC - Scale up the deployment. Signed-off-by: kmova --- cmd/provisioner-localpv/app/provisioner_hostpath.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/provisioner-localpv/app/provisioner_hostpath.go b/cmd/provisioner-localpv/app/provisioner_hostpath.go index 2f6da47a6d..120c1ec181 100644 --- a/cmd/provisioner-localpv/app/provisioner_hostpath.go +++ b/cmd/provisioner-localpv/app/provisioner_hostpath.go @@ -134,8 +134,12 @@ func (p *Provisioner) GetNodeObjectFromHostName(hostName string) (*v1.Node, erro Limit: 1, } nodeList, err := p.kubeClient.CoreV1().Nodes().List(listOptions) - if err != nil { - return nil, errors.Errorf("Unable to get the Node with the NodeHostName") + if err != nil || nodeList.Items == nil || len(nodeList.Items) == 0 { + // After the PV is created and node affinity is set + // based on kubernetes.io/hostname label, either: + // - hostname label changed on the node or + // - the node is deleted from the cluster. + return nil, errors.Errorf("Unable to get the Node with the NodeHostName [%s]", hostName) } return &nodeList.Items[0], nil From 35f519ce44b8c3373d3dc2a62868c5eec1c7224d Mon Sep 17 00:00:00 2001 From: kmova Date: Fri, 10 Apr 2020 12:38:35 +0000 Subject: [PATCH 2/3] fix(localpv-hostpath): gofmt issue on provisioner_hostpath.go Signed-off-by: kmova --- cmd/provisioner-localpv/app/provisioner_hostpath.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/provisioner-localpv/app/provisioner_hostpath.go b/cmd/provisioner-localpv/app/provisioner_hostpath.go index 120c1ec181..4204da7175 100644 --- a/cmd/provisioner-localpv/app/provisioner_hostpath.go +++ b/cmd/provisioner-localpv/app/provisioner_hostpath.go @@ -135,7 +135,7 @@ func (p *Provisioner) GetNodeObjectFromHostName(hostName string) (*v1.Node, erro } nodeList, err := p.kubeClient.CoreV1().Nodes().List(listOptions) if err != nil || nodeList.Items == nil || len(nodeList.Items) == 0 { - // After the PV is created and node affinity is set + // After the PV is created and node affinity is set // based on kubernetes.io/hostname label, either: // - hostname label changed on the node or // - the node is deleted from the cluster. From 6a98dda11da99d345731ef68d84d00bfdf66f87e Mon Sep 17 00:00:00 2001 From: kmova Date: Fri, 10 Apr 2020 14:13:09 +0000 Subject: [PATCH 3/3] fix(localpv-hostpath): remove redundant nil check in provisioner_hostpath.go Signed-off-by: kmova --- cmd/provisioner-localpv/app/provisioner_hostpath.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/provisioner-localpv/app/provisioner_hostpath.go b/cmd/provisioner-localpv/app/provisioner_hostpath.go index 4204da7175..35484b131d 100644 --- a/cmd/provisioner-localpv/app/provisioner_hostpath.go +++ b/cmd/provisioner-localpv/app/provisioner_hostpath.go @@ -134,7 +134,7 @@ func (p *Provisioner) GetNodeObjectFromHostName(hostName string) (*v1.Node, erro Limit: 1, } nodeList, err := p.kubeClient.CoreV1().Nodes().List(listOptions) - if err != nil || nodeList.Items == nil || len(nodeList.Items) == 0 { + if err != nil || len(nodeList.Items) == 0 { // After the PV is created and node affinity is set // based on kubernetes.io/hostname label, either: // - hostname label changed on the node or