From d0f018f51c20788bc4efad6d5e36ac40fc5acafc Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Thu, 8 Jun 2023 09:18:56 -0500 Subject: [PATCH 1/7] fix(plugin): fix node id to node mapping Signed-off-by: Jesse Nelson --- pkg/driver/controller.go | 7 ++++++- pkg/mgmt/zfsnode/start.go | 24 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 154655641..23acb2b1c 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -918,7 +918,12 @@ func (cs *controller) GetCapacity( var availableCapacity int64 for _, nodeName := range nodeNames { - v, exists, err := zfsNodesCache.GetByKey(zfs.OpenEBSNamespace + "/" + nodeName) + mappedNodeId, mapErr := zfs.GetNodeID(nodeName) + if mapErr != nil { + klog.Warningf("Unable to find mapped node id for %s", nodeName) + mappedNodeId = nodeName + } + v, exists, err := zfsNodesCache.GetByKey(zfs.OpenEBSNamespace + "/" + mappedNodeId) if err != nil { klog.Warning("unexpected error after querying the zfsNode informer cache") continue diff --git a/pkg/mgmt/zfsnode/start.go b/pkg/mgmt/zfsnode/start.go index 395ae4f92..cf7b709fb 100644 --- a/pkg/mgmt/zfsnode/start.go +++ b/pkg/mgmt/zfsnode/start.go @@ -18,6 +18,11 @@ package zfsnode import ( "context" + "fmt" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/klog/v2" "sync" "time" @@ -27,7 +32,6 @@ import ( "github.com/openebs/zfs-localpv/pkg/zfs" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" ) @@ -60,10 +64,26 @@ func Start(controllerMtx *sync.RWMutex, stopCh <-chan struct{}) error { options.FieldSelector = fields.OneTermEqualSelector("metadata.name", zfs.NodeID).String() })) - k8sNode, err := kubeClient.CoreV1().Nodes().Get(context.TODO(), zfs.NodeID, metav1.GetOptions{}) + topologyRequirement, requirementError := labels.NewRequirement(zfs.ZFSTopologyKey, selection.Equals, []string{zfs.NodeID}) + if requirementError != nil { + return errors.Wrapf(requirementError, "Unable to retrieve node by %s for node id %s", zfs.ZFSTopologyKey, zfs.NodeID) + } + topologySelector := labels.NewSelector().Add(*topologyRequirement).String() + klog.Infof("The topology selector is %s", topologySelector) + + k8sNodeCandidates, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: topologySelector, + }) + if err != nil { return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) } + + if k8sNodeCandidates == nil || len(k8sNodeCandidates.Items) != 1 { + return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) + } + + k8sNode := k8sNodeCandidates.Items[0] isTrue := true // as object returned by client go clears all TypeMeta from it. nodeGVK := &schema.GroupVersionKind{ From e424c32caa714b70b80d39ec0b2019781df4ce0f Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Fri, 16 Jun 2023 13:48:05 -0500 Subject: [PATCH 2/7] add change log Signed-off-by: Jesse Nelson --- changelogs/unreleased/450-nodeid-fix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/unreleased/450-nodeid-fix.md diff --git a/changelogs/unreleased/450-nodeid-fix.md b/changelogs/unreleased/450-nodeid-fix.md new file mode 100644 index 000000000..eeef98d0d --- /dev/null +++ b/changelogs/unreleased/450-nodeid-fix.md @@ -0,0 +1 @@ +fix regression introduced with v2.0.0 that caused the plugin code to not be able to start when a user sets openebs.io/nodeid \ No newline at end of file From bd689b3a7cb6a320fc800700c7f831e8f537840b Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Wed, 21 Jun 2023 10:20:02 -0500 Subject: [PATCH 3/7] fix case when nodeid not set Signed-off-by: Jesse Nelson --- pkg/mgmt/zfsnode/start.go | 51 +++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/pkg/mgmt/zfsnode/start.go b/pkg/mgmt/zfsnode/start.go index cf7b709fb..633330cc2 100644 --- a/pkg/mgmt/zfsnode/start.go +++ b/pkg/mgmt/zfsnode/start.go @@ -19,10 +19,12 @@ package zfsnode import ( "context" "fmt" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/klog/v2" + "os" "sync" "time" @@ -64,26 +66,39 @@ func Start(controllerMtx *sync.RWMutex, stopCh <-chan struct{}) error { options.FieldSelector = fields.OneTermEqualSelector("metadata.name", zfs.NodeID).String() })) - topologyRequirement, requirementError := labels.NewRequirement(zfs.ZFSTopologyKey, selection.Equals, []string{zfs.NodeID}) - if requirementError != nil { - return errors.Wrapf(requirementError, "Unable to retrieve node by %s for node id %s", zfs.ZFSTopologyKey, zfs.NodeID) + var k8sNode *v1.Node + nodeName := os.Getenv("OPENEBS_NODE_NAME") + if nodeName != zfs.NodeID { + topologyRequirement, requirementError := labels.NewRequirement(zfs.ZFSTopologyKey, selection.Equals, []string{zfs.NodeID}) + if requirementError != nil { + return errors.Wrapf(requirementError, "Unable to retrieve node by %s for node id %s", zfs.ZFSTopologyKey, zfs.NodeID) + } + topologySelector := labels.NewSelector().Add(*topologyRequirement).String() + klog.Infof("The topology selector is %s", topologySelector) + + k8sNodeCandidates, listError := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: topologySelector, + }) + + if listError != nil { + return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) + } + + if k8sNodeCandidates == nil || len(k8sNodeCandidates.Items) != 1 { + return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) + } + + k8sNode = &k8sNodeCandidates.Items[0] + } else { + byName, byNameErr := kubeClient.CoreV1().Nodes().Get(context.TODO(), zfs.NodeID, metav1.GetOptions{}) + + if byNameErr != nil { + return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) + } + + k8sNode = byName } - topologySelector := labels.NewSelector().Add(*topologyRequirement).String() - klog.Infof("The topology selector is %s", topologySelector) - k8sNodeCandidates, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: topologySelector, - }) - - if err != nil { - return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) - } - - if k8sNodeCandidates == nil || len(k8sNodeCandidates.Items) != 1 { - return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) - } - - k8sNode := k8sNodeCandidates.Items[0] isTrue := true // as object returned by client go clears all TypeMeta from it. nodeGVK := &schema.GroupVersionKind{ From 5715846d2a9febf77931dcc8307066e0fd7315a7 Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Wed, 21 Jun 2023 11:50:33 -0500 Subject: [PATCH 4/7] simplify logic Signed-off-by: Jesse Nelson --- pkg/mgmt/zfsnode/start.go | 49 +++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/pkg/mgmt/zfsnode/start.go b/pkg/mgmt/zfsnode/start.go index 633330cc2..8508d7404 100644 --- a/pkg/mgmt/zfsnode/start.go +++ b/pkg/mgmt/zfsnode/start.go @@ -19,7 +19,6 @@ package zfsnode import ( "context" "fmt" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" @@ -66,39 +65,33 @@ func Start(controllerMtx *sync.RWMutex, stopCh <-chan struct{}) error { options.FieldSelector = fields.OneTermEqualSelector("metadata.name", zfs.NodeID).String() })) - var k8sNode *v1.Node nodeName := os.Getenv("OPENEBS_NODE_NAME") - if nodeName != zfs.NodeID { - topologyRequirement, requirementError := labels.NewRequirement(zfs.ZFSTopologyKey, selection.Equals, []string{zfs.NodeID}) - if requirementError != nil { - return errors.Wrapf(requirementError, "Unable to retrieve node by %s for node id %s", zfs.ZFSTopologyKey, zfs.NodeID) - } - topologySelector := labels.NewSelector().Add(*topologyRequirement).String() - klog.Infof("The topology selector is %s", topologySelector) - - k8sNodeCandidates, listError := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: topologySelector, - }) - - if listError != nil { - return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) - } - - if k8sNodeCandidates == nil || len(k8sNodeCandidates.Items) != 1 { - return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) - } - - k8sNode = &k8sNodeCandidates.Items[0] + var searchLabel string + if nodeName == zfs.NodeID { + searchLabel = zfs.ZFSTopoNodenameKey } else { - byName, byNameErr := kubeClient.CoreV1().Nodes().Get(context.TODO(), zfs.NodeID, metav1.GetOptions{}) + searchLabel = zfs.ZFSTopologyKey + } + topologyRequirement, requirementError := labels.NewRequirement(searchLabel, selection.Equals, []string{zfs.NodeID}) + if requirementError != nil { + return errors.Wrapf(requirementError, "Unable to retrieve node by %s for node id %s", zfs.ZFSTopologyKey, zfs.NodeID) + } + topologySelector := labels.NewSelector().Add(*topologyRequirement).String() + klog.Infof("The topology selector is %s", topologySelector) + + k8sNodeCandidates, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: topologySelector, + }) - if byNameErr != nil { - return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) - } + if err != nil { + return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) + } - k8sNode = byName + if k8sNodeCandidates == nil || len(k8sNodeCandidates.Items) != 1 { + return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) } + k8sNode := k8sNodeCandidates.Items[0] isTrue := true // as object returned by client go clears all TypeMeta from it. nodeGVK := &schema.GroupVersionKind{ From 704698cdeb89393eda9ebf3b219d1a01ece6f78d Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Wed, 1 Nov 2023 15:07:47 -0500 Subject: [PATCH 5/7] update doc Signed-off-by: Jesse Nelson --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 4f11a4002..7c8a88d4c 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,8 @@ Configure the custom topology keys (if needed). This can be used for many purpos https://github.com/openebs/zfs-localpv/blob/HEAD/docs/faq.md#6-how-to-add-custom-topology-key ### Installation +In order to support moving data to a new node later on, you must label each node with a unique value for `openebs.io/nodeid`. +For more information on migrating data, please [see here](docs/faq.md#8-how-to-migrate-pvs-to-the-new-node-in-case-old-node-is-not-accessible) We can install the latest release of OpenEBS ZFS driver by running the following command: ```bash From 3e769740f7f8a9580cf0ebbb2fd8a024415dfa5f Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Wed, 1 Nov 2023 17:27:48 -0500 Subject: [PATCH 6/7] fix cases where nodeid not set Signed-off-by: Jesse Nelson --- pkg/mgmt/zfsnode/start.go | 49 ++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/pkg/mgmt/zfsnode/start.go b/pkg/mgmt/zfsnode/start.go index 8508d7404..8fb164bf4 100644 --- a/pkg/mgmt/zfsnode/start.go +++ b/pkg/mgmt/zfsnode/start.go @@ -19,11 +19,13 @@ package zfsnode import ( "context" "fmt" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/klog/v2" "os" + "strings" "sync" "time" @@ -66,32 +68,37 @@ func Start(controllerMtx *sync.RWMutex, stopCh <-chan struct{}) error { })) nodeName := os.Getenv("OPENEBS_NODE_NAME") - var searchLabel string - if nodeName == zfs.NodeID { - searchLabel = zfs.ZFSTopoNodenameKey - } else { - searchLabel = zfs.ZFSTopologyKey - } - topologyRequirement, requirementError := labels.NewRequirement(searchLabel, selection.Equals, []string{zfs.NodeID}) - if requirementError != nil { - return errors.Wrapf(requirementError, "Unable to retrieve node by %s for node id %s", zfs.ZFSTopologyKey, zfs.NodeID) - } - topologySelector := labels.NewSelector().Add(*topologyRequirement).String() - klog.Infof("The topology selector is %s", topologySelector) + var k8sNode v1.Node - k8sNodeCandidates, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: topologySelector, - }) + if len(strings.TrimSpace(zfs.NodeID)) == 0 || nodeName == zfs.NodeID { + k8sNodeCandidate, err := kubeClient.CoreV1().Nodes().Get(context.TODO(), zfs.NodeID, metav1.GetOptions{}) - if err != nil { - return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) - } + if err != nil { + return errors.Wrapf(err, "fetch k8s node %s", zfs.NodeID) + } + + k8sNode = *k8sNodeCandidate - if k8sNodeCandidates == nil || len(k8sNodeCandidates.Items) != 1 { - return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) + } else { + topologyRequirement, requirementError := labels.NewRequirement(zfs.ZFSTopologyKey, selection.Equals, []string{zfs.NodeID}) + if requirementError != nil { + return errors.Wrapf(requirementError, "Unable to generate topology requirement by %s for node id %s", zfs.ZFSTopologyKey, zfs.NodeID) + } + topologySelector := labels.NewSelector().Add(*topologyRequirement).String() + klog.Infof("The topology selector is %s", topologySelector) + + k8sNodeCandidate, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: topologySelector, + }) + if k8sNodeCandidate == nil || len(k8sNodeCandidate.Items) != 1 { + return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) + } + if err != nil { + return errors.Wrapf(err, "error trying to find node with label %s having value %s", zfs.ZFSTopologyKey, zfs.NodeID) + } + k8sNode = k8sNodeCandidate.Items[0] } - k8sNode := k8sNodeCandidates.Items[0] isTrue := true // as object returned by client go clears all TypeMeta from it. nodeGVK := &schema.GroupVersionKind{ From 1058228e0670be55c98d1b76d2b76d7996621651 Mon Sep 17 00:00:00 2001 From: Jesse Nelson Date: Thu, 2 Nov 2023 08:50:35 -0500 Subject: [PATCH 7/7] check for error first Signed-off-by: Jesse Nelson --- pkg/mgmt/zfsnode/start.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/mgmt/zfsnode/start.go b/pkg/mgmt/zfsnode/start.go index 8fb164bf4..20cf6bdad 100644 --- a/pkg/mgmt/zfsnode/start.go +++ b/pkg/mgmt/zfsnode/start.go @@ -90,12 +90,12 @@ func Start(controllerMtx *sync.RWMutex, stopCh <-chan struct{}) error { k8sNodeCandidate, err := kubeClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ LabelSelector: topologySelector, }) - if k8sNodeCandidate == nil || len(k8sNodeCandidate.Items) != 1 { - return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) - } if err != nil { return errors.Wrapf(err, "error trying to find node with label %s having value %s", zfs.ZFSTopologyKey, zfs.NodeID) } + if k8sNodeCandidate == nil || len(k8sNodeCandidate.Items) != 1 { + return fmt.Errorf("unable to retrieve a single node by %s for %s", zfs.ZFSTopologyKey, zfs.NodeID) + } k8sNode = k8sNodeCandidate.Items[0] }