Add joinSubnets config for UDNs

go-controller/pkg/clustermanager/clustermanager_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
 	ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
 	"github.com/urfave/cli/v2"
@@ -28,9 +29,6 @@ const (
 	// ovnNodeIDAnnotaton is the node annotation name used to store the node id.
 	ovnNodeIDAnnotaton = "k8s.ovn.org/node-id"
 
-	// ovnNodeGRLRPAddrAnnotaton is the node annotation name used to store the node gateway router port ips.
-	ovnNodeGRLRPAddrAnnotaton = "k8s.ovn.org/node-gateway-router-lrp-ifaddr"
-
 	// ovnTransitSwitchPortAddrAnnotation is the node annotation name to store the transit switch port ips.
 	ovnTransitSwitchPortAddrAnnotation = "k8s.ovn.org/node-transit-switch-port-ifaddr"
 )
@@ -916,7 +914,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() {
 							return err
 						}
 
-						gwLRPAddrs, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode)
+						gwLRPAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName)
 						if err != nil {
 							return err
 						}
@@ -989,13 +987,13 @@ var _ = ginkgo.Describe("Cluster Manager", func() {
 							return err
 						}
 
-						gwLRPAddrs, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode)
+						gwLRPAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName)
 						if err != nil {
 							return err
 						}
 						gomega.Expect(gwLRPAddrs).NotTo(gomega.BeNil())
 						gomega.Expect(len(gwLRPAddrs)).To(gomega.Equal(2))
-						nodeAddrs[n.Name] = updatedNode.Annotations[ovnNodeGRLRPAddrAnnotaton]
+						nodeAddrs[n.Name] = updatedNode.Annotations[util.OVNNodeGRLRPAddrs]
 						return nil
 					}).ShouldNot(gomega.HaveOccurred())
 				}
@@ -1009,7 +1007,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() {
 					for k, v := range nodeAnnotations {
 						nodeAnnotator.Set(k, v)
 					}
-					nodeAnnotator.Delete(ovnNodeGRLRPAddrAnnotaton)
+					nodeAnnotator.Delete(util.OVNNodeGRLRPAddrs)
 					err = nodeAnnotator.Run()
 					gomega.Expect(err).NotTo(gomega.HaveOccurred())
 				}
@@ -1021,7 +1019,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() {
 							return err
 						}
 
-						nodeGWRPIPs, ok := updatedNode.Annotations[ovnNodeGRLRPAddrAnnotaton]
+						nodeGWRPIPs, ok := updatedNode.Annotations[util.OVNNodeGRLRPAddrs]
 						if !ok {
 							return fmt.Errorf("expected node annotation for node %s to have node gateway-router-lrp-ifaddr allocated", n.Name)
 						}
@@ -1092,7 +1090,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() {
 							return err
 						}
 
-						gwLRPAddrs, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode)
+						gwLRPAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName)
 						if err != nil {
 							return err
 						}
@@ -1101,7 +1099,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() {
 
 						// Store the node 3's gw router port addresses
 						if updatedNode.Name == "node3" {
-							node3GWRPAnnotation = updatedNode.Annotations[ovnNodeGRLRPAddrAnnotaton]
+							node3GWRPAnnotation = updatedNode.Annotations[util.OVNNodeGRLRPAddrs]
 						}
 						return nil
 					}).ShouldNot(gomega.HaveOccurred())
@@ -1149,10 +1147,10 @@ var _ = ginkgo.Describe("Cluster Manager", func() {
 						return err
 					}
 
-					node3UpdatedGWRPAnnotation := updatedNode.Annotations[ovnNodeGRLRPAddrAnnotaton]
+					node3UpdatedGWRPAnnotation := updatedNode.Annotations[util.OVNNodeGRLRPAddrs]
 					gomega.Expect(node3UpdatedGWRPAnnotation).NotTo(gomega.Equal(node3GWRPAnnotation))
 
-					gwLRPAddrs, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode)
+					gwLRPAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName)
 					if err != nil {
 						return err
 					}

go-controller/pkg/clustermanager/network_cluster_controller_test.go

@@ -54,6 +54,7 @@ var _ = ginkgo.Describe("Network Cluster Controller", func() {
 						ObjectMeta: metav1.ObjectMeta{
 							Name: "node1",
 							Annotations: map[string]string{
+								ovnNodeIDAnnotaton:         "3",
 								"k8s.ovn.org/node-subnets": "{\"default\":[\"10.128.0.0/24\", \"fd02:0:0:2::2895/64\"]}",
 							},
 						},
@@ -104,6 +105,9 @@ var _ = ginkgo.Describe("Network Cluster Controller", func() {
 					{
 						ObjectMeta: metav1.ObjectMeta{
 							Name: "node1",
+							Annotations: map[string]string{
+								ovnNodeIDAnnotaton: "3",
+							},
 						},
 					},
 				}
@@ -154,6 +158,7 @@ var _ = ginkgo.Describe("Network Cluster Controller", func() {
 							Name: "node1",
 							Annotations: map[string]string{
 								"k8s.ovn.org/node-subnets": "{\"default\":[\"10.128.0.0/24\", \"1.2.3.0/24\"]}",
+								ovnNodeIDAnnotaton:         "3",
 							},
 						},
 					},

go-controller/pkg/clustermanager/node/node_allocator.go

@@ -11,9 +11,12 @@ import (
 	"k8s.io/klog/v2"
 	utilnet "k8s.io/utils/net"
 
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
 	houtil "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/util"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
+	ipgenerator "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/generator/ip"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
@@ -33,6 +36,9 @@ type NodeAllocator struct {
 
 	clusterSubnetAllocator       SubnetAllocator
 	hybridOverlaySubnetAllocator SubnetAllocator
+	// node gateway router port IP generators (connecting to the join switch)
+	nodeGWRouterLRPIPv4Generator *ipgenerator.IPGenerator
+	nodeGWRouterLRPIPv6Generator *ipgenerator.IPGenerator
 
 	// unique id of the network
 	networkID int
@@ -62,6 +68,24 @@ func NewNodeAllocator(networkID int, netInfo util.NetInfo, nodeLister listers.No
 }
 
 func (na *NodeAllocator) Init() error {
+	if na.hasJoinSubnetAllocation() {
+		if config.IPv4Mode {
+			nodeGWRouterLRPIPv4Generator, err := ipgenerator.NewIPGenerator(na.netInfo.JoinSubnetV4().String())
+			if err != nil {
+				return fmt.Errorf("error creating IP Generator for v4 join subnet %s: %w", na.netInfo.JoinSubnetV4().String(), err)
+			}
+			na.nodeGWRouterLRPIPv4Generator = nodeGWRouterLRPIPv4Generator
+		}
+
+		if config.IPv6Mode {
+			nodeGWRouterLRPIPv6Generator, err := ipgenerator.NewIPGenerator(na.netInfo.JoinSubnetV6().String())
+			if err != nil {
+				return fmt.Errorf("error creating IP Generator for v6 join subnet %s: %w", na.netInfo.JoinSubnetV6().String(), err)
+			}
+			na.nodeGWRouterLRPIPv6Generator = nodeGWRouterLRPIPv6Generator
+		}
+	}
+
 	if !na.hasNodeSubnetAllocation() {
 		return nil
 	}
@@ -185,7 +209,47 @@ func (na *NodeAllocator) syncNodeNetworkAnnotations(node *corev1.Node) error {
 	}
 
 	updatedSubnetsMap := map[string][]*net.IPNet{}
-	var validExistingSubnets, allocatedSubnets []*net.IPNet
+	var validExistingSubnets, allocatedSubnets, allocatedJoinSubnets []*net.IPNet
+	if na.hasJoinSubnetAllocation() {
+		var joinAddr []*net.IPNet
+		existingSubnets, err := util.ParseNodeGatewayRouterJoinAddrs(node, networkName)
+		if err != nil && !util.IsAnnotationNotSetError(err) {
+			// Log the error and try to allocate new subnets
+			klog.Warningf("Failed to get node %s join subnets annotations for network %s: %v", node.Name, networkName, err)
+		}
+		// Allocate the IP address(es) for the node Gateway router port connecting
+		// to the Join switch
+		nodeID := util.GetNodeID(node)
+		if nodeID == -1 {
+			// Don't consider this node as cluster-manager has not allocated node id yet.
+			return fmt.Errorf("failed to get node id for node - %s", node.Name)
+		}
+
+		if config.IPv4Mode {
+			joinV4Addr, err := na.nodeGWRouterLRPIPv4Generator.GenerateIP(nodeID)
+			if err != nil {
+				return fmt.Errorf("failed to generate gateway router port IPv4 address for node %s : err - %w", node.Name, err)
+			}
+			joinAddr = append(joinAddr, joinV4Addr)
+		}
+
+		if config.IPv6Mode {
+			joinV6Addr, err := na.nodeGWRouterLRPIPv6Generator.GenerateIP(nodeID)
+			if err != nil {
+				return fmt.Errorf("failed to generate gateway router port IPv6 address for node %s : err - %w", node.Name, err)
+			}
+			joinAddr = append(joinAddr, joinV6Addr)
+		}
+		// If the existing subnets weren't OK, or new ones were allocated, update the node annotation.
+		// This happens in a couple cases:
+		// 1) new node: no existing subnets and one or more new subnets were allocated
+		// 2) dual-stack/single-stack conversion: two existing subnets but only one will be valid, and no allocated subnets
+		// 3) bad subnet annotation: one more existing subnets will be invalid and might have allocated a correct one; let us reset it
+		lessIPNet := func(a, b net.IPNet) bool { return a.String() < b.String() }
+		if !cmp.Equal(existingSubnets, joinAddr, cmpopts.SortSlices(lessIPNet)) {
+			allocatedJoinSubnets = joinAddr
+		}
+	}
 	if na.hasNodeSubnetAllocation() {
 		existingSubnets, err := util.ParseNodeHostSubnetAnnotation(node, networkName)
 		if err != nil && !util.IsAnnotationNotSetError(err) {
@@ -214,8 +278,8 @@ func (na *NodeAllocator) syncNodeNetworkAnnotations(node *corev1.Node) error {
 	}
 
 	// Also update the node annotation if the networkID doesn't match
-	if len(updatedSubnetsMap) > 0 || na.networkID != networkID {
-		err = na.updateNodeNetworkAnnotationsWithRetry(node.Name, updatedSubnetsMap, na.networkID)
+	if len(updatedSubnetsMap) > 0 || na.networkID != networkID || len(allocatedJoinSubnets) > 0 {
+		err = na.updateNodeNetworkAnnotationsWithRetry(node.Name, updatedSubnetsMap, na.networkID, allocatedJoinSubnets)
 		if err != nil {
 			if errR := na.clusterSubnetAllocator.ReleaseNetworks(node.Name, allocatedSubnets...); errR != nil {
 				klog.Warningf("Error releasing node %s subnets: %v", node.Name, errR)
@@ -287,7 +351,7 @@ func (na *NodeAllocator) Sync(nodes []interface{}) error {
 }
 
 // updateNodeNetworkAnnotationsWithRetry will update the node's subnet annotation and network id annotation
-func (na *NodeAllocator) updateNodeNetworkAnnotationsWithRetry(nodeName string, hostSubnetsMap map[string][]*net.IPNet, networkId int) error {
+func (na *NodeAllocator) updateNodeNetworkAnnotationsWithRetry(nodeName string, hostSubnetsMap map[string][]*net.IPNet, networkId int, joinAddr []*net.IPNet) error {
 	// Retry if it fails because of potential conflict which is transient. Return error in the
 	// case of other errors (say temporary API server down), and it will be taken care of by the
 	// retry mechanism.
@@ -309,6 +373,12 @@ func (na *NodeAllocator) updateNodeNetworkAnnotationsWithRetry(nodeName string,
 
 		networkName := na.netInfo.GetNetworkName()
 
+		cnode.Annotations, err = util.UpdateNodeGatewayRouterLRPAddrsAnnotation(cnode.Annotations, joinAddr, networkName)
+		if err != nil {
+			return fmt.Errorf("failed to update node %q annotation LRPAddrAnnotation %s",
+				node.Name, util.JoinIPNets(joinAddr, ","))
+		}
+
 		cnode.Annotations, err = util.UpdateNetworkIDAnnotation(cnode.Annotations, networkName, networkId)
 		if err != nil {
 			return fmt.Errorf("failed to update node %q network id annotation %d for network %s",
@@ -344,7 +414,7 @@ func (na *NodeAllocator) Cleanup() error {
 
 		hostSubnetsMap := map[string][]*net.IPNet{networkName: nil}
 		// passing util.InvalidNetworkID deletes the network id annotation for the network.
-		err = na.updateNodeNetworkAnnotationsWithRetry(node.Name, hostSubnetsMap, util.InvalidNetworkID)
+		err = na.updateNodeNetworkAnnotationsWithRetry(node.Name, hostSubnetsMap, util.InvalidNetworkID, nil)
 		if err != nil {
 			return fmt.Errorf("failed to clear node %q subnet annotation for network %s",
 				node.Name, networkName)
@@ -468,3 +538,8 @@ func (na *NodeAllocator) hasNodeSubnetAllocation() bool {
 	// we only allocate subnets for L3 secondary network or default network
 	return na.netInfo.TopologyType() == types.Layer3Topology || !na.netInfo.IsSecondary()
 }
+
+func (na *NodeAllocator) hasJoinSubnetAllocation() bool {
+	// we allocate join subnets for L3/L2 primary user defined networks or default network
+	return na.netInfo.IsDefault() || (util.IsNetworkSegmentationSupportEnabled() && na.netInfo.IsPrimaryNetwork())
+}

go-controller/pkg/clustermanager/zone_cluster_controller.go

@@ -41,10 +41,6 @@ type zoneClusterController struct {
 	// ID allocator for the nodes
 	nodeIDAllocator id.Allocator
 
-	// node gateway router port IP generators (connecting to the join switch)
-	nodeGWRouterLRPIPv4Generator *ipgenerator.IPGenerator
-	nodeGWRouterLRPIPv6Generator *ipgenerator.IPGenerator
-
 	// Transit switch IP generator. This is required if EnableInterconnect feature is enabled.
 	transitSwitchIPv4Generator *ipgenerator.IPGenerator
 	transitSwitchIPv6Generator *ipgenerator.IPGenerator
@@ -70,22 +66,6 @@ func newZoneClusterController(ovnClient *util.OVNClusterManagerClientset, wf *fa
 	}
 	wg := &sync.WaitGroup{}
 
-	var nodeGWRouterLRPIPv4Generator, nodeGWRouterLRPIPv6Generator *ipgenerator.IPGenerator
-
-	if config.IPv4Mode {
-		nodeGWRouterLRPIPv4Generator, err = ipgenerator.NewIPGenerator(config.Gateway.V4JoinSubnet)
-		if err != nil {
-			return nil, fmt.Errorf("error creating IP Generator for v4 join subnet %s: %w", config.Gateway.V4JoinSubnet, err)
-		}
-	}
-
-	if config.IPv6Mode {
-		nodeGWRouterLRPIPv6Generator, err = ipgenerator.NewIPGenerator(config.Gateway.V6JoinSubnet)
-		if err != nil {
-			return nil, fmt.Errorf("error creating IP Generator for v6 join subnet %s: %w", config.Gateway.V6JoinSubnet, err)
-		}
-	}
-
 	var transitSwitchIPv4Generator, transitSwitchIPv6Generator *ipgenerator.IPGenerator
 
 	if config.OVNKubernetesFeature.EnableInterconnect {
@@ -105,15 +85,13 @@ func newZoneClusterController(ovnClient *util.OVNClusterManagerClientset, wf *fa
 	}
 
 	zcc := &zoneClusterController{
-		kube:                         kube,
-		watchFactory:                 wf,
-		stopChan:                     make(chan struct{}),
-		wg:                           wg,
-		nodeIDAllocator:              nodeIDAllocator,
-		nodeGWRouterLRPIPv4Generator: nodeGWRouterLRPIPv4Generator,
-		nodeGWRouterLRPIPv6Generator: nodeGWRouterLRPIPv6Generator,
-		transitSwitchIPv4Generator:   transitSwitchIPv4Generator,
-		transitSwitchIPv6Generator:   transitSwitchIPv6Generator,
+		kube:                       kube,
+		watchFactory:               wf,
+		stopChan:                   make(chan struct{}),
+		wg:                         wg,
+		nodeIDAllocator:            nodeIDAllocator,
+		transitSwitchIPv4Generator: transitSwitchIPv4Generator,
+		transitSwitchIPv6Generator: transitSwitchIPv6Generator,
 	}
 
 	zcc.initRetryFramework()
@@ -173,25 +151,6 @@ func (zcc *zoneClusterController) handleAddUpdateNodeEvent(node *corev1.Node) er
 	// Allocate the IP address(es) for the node Gateway router port connecting
 	// to the Join switch
 	var v4Addr, v6Addr *net.IPNet
-	if config.IPv4Mode {
-		v4Addr, err = zcc.nodeGWRouterLRPIPv4Generator.GenerateIP(allocatedNodeID)
-		if err != nil {
-			return fmt.Errorf("failed to generate gateway router port IPv4 address for node %s : err - %w", node.Name, err)
-		}
-	}
-
-	if config.IPv6Mode {
-		v6Addr, err = zcc.nodeGWRouterLRPIPv6Generator.GenerateIP(allocatedNodeID)
-		if err != nil {
-			return fmt.Errorf("failed to generate gateway router port IPv6 address for node %s : err - %w", node.Name, err)
-		}
-	}
-
-	nodeAnnotations, err = util.CreateNodeGatewayRouterLRPAddrAnnotation(nodeAnnotations, v4Addr, v6Addr)
-	if err != nil {
-		return fmt.Errorf("failed to marshal node %q annotation for Gateway LRP IPs, err : %v",
-			node.Name, err)
-	}
 
 	if config.OVNKubernetesFeature.EnableInterconnect {
 		v4Addr = nil
@@ -369,7 +328,7 @@ func (h *zoneClusterControllerEventHandler) AreResourcesEqual(obj1, obj2 interfa
 		if util.NodeIDAnnotationChanged(node1, node2) {
 			return false, nil
 		}
-		if util.NodeGatewayRouterLRPAddrAnnotationChanged(node1, node2) {
+		if util.NodeGatewayRouterLRPAddrsAnnotationChanged(node1, node2) {
 			return false, nil
 		}
 		if util.NodeTransitSwitchPortAddrAnnotationChanged(node1, node2) {

go-controller/pkg/cni/types/types.go

@@ -32,6 +32,16 @@ type NetConf struct {
 	// valid for layer2 and localnet network topology
 	// eg. "10.1.130.0/27, 10.1.130.122/32"
 	ExcludeSubnets string `json:"excludeSubnets,omitempty"`
+	// join subnet cidr is required for supporting
+	// services and ingress for user defined networks
+	// in case of dualstack cluster, please do a comma-seperated list
+	// expected format:
+	// 1) V4 single stack: "v4CIDR" (eg: "100.65.0.0/16")
+	// 2) V6 single stack: "v6CIDR" (eg: "fd99::/64")
+	// 3) dualstack: "v4CIDR,v6CIDR" (eg: "100.65.0.0/16,fd99::/64")
+	// valid for UDN layer3/layer2 network topology
+	// default value: 100.65.0.0/16,fd99::/64 if not provided
+	JoinSubnet string `json:"joinSubnet,omitempty"`
 	// VLANID, valid in localnet topology network only
 	VLANID int `json:"vlanID,omitempty"`
 	// AllowPersistentIPs is valid on both localnet / layer topologies.