diff --git a/go-controller/pkg/clustermanager/clustermanager_test.go b/go-controller/pkg/clustermanager/clustermanager_test.go index c54a5973489..b36d805e54a 100644 --- a/go-controller/pkg/clustermanager/clustermanager_test.go +++ b/go-controller/pkg/clustermanager/clustermanager_test.go @@ -18,6 +18,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/urfave/cli/v2" @@ -28,9 +29,6 @@ const ( // ovnNodeIDAnnotaton is the node annotation name used to store the node id. ovnNodeIDAnnotaton = "k8s.ovn.org/node-id" - // ovnNodeGRLRPAddrAnnotaton is the node annotation name used to store the node gateway router port ips. - ovnNodeGRLRPAddrAnnotaton = "k8s.ovn.org/node-gateway-router-lrp-ifaddr" - // ovnTransitSwitchPortAddrAnnotation is the node annotation name to store the transit switch port ips. ovnTransitSwitchPortAddrAnnotation = "k8s.ovn.org/node-transit-switch-port-ifaddr" ) @@ -916,7 +914,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() { return err } - gwLRPAddrs, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode) + gwLRPAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName) if err != nil { return err } @@ -989,13 +987,13 @@ var _ = ginkgo.Describe("Cluster Manager", func() { return err } - gwLRPAddrs, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode) + gwLRPAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName) if err != nil { return err } gomega.Expect(gwLRPAddrs).NotTo(gomega.BeNil()) gomega.Expect(len(gwLRPAddrs)).To(gomega.Equal(2)) - nodeAddrs[n.Name] = updatedNode.Annotations[ovnNodeGRLRPAddrAnnotaton] + nodeAddrs[n.Name] = updatedNode.Annotations[util.OVNNodeGRLRPAddrs] return nil }).ShouldNot(gomega.HaveOccurred()) } @@ -1009,7 +1007,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() { for k, v := range nodeAnnotations { nodeAnnotator.Set(k, v) } - nodeAnnotator.Delete(ovnNodeGRLRPAddrAnnotaton) + nodeAnnotator.Delete(util.OVNNodeGRLRPAddrs) err = nodeAnnotator.Run() gomega.Expect(err).NotTo(gomega.HaveOccurred()) } @@ -1021,7 +1019,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() { return err } - nodeGWRPIPs, ok := updatedNode.Annotations[ovnNodeGRLRPAddrAnnotaton] + nodeGWRPIPs, ok := updatedNode.Annotations[util.OVNNodeGRLRPAddrs] if !ok { return fmt.Errorf("expected node annotation for node %s to have node gateway-router-lrp-ifaddr allocated", n.Name) } @@ -1092,7 +1090,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() { return err } - gwLRPAddrs, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode) + gwLRPAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName) if err != nil { return err } @@ -1101,7 +1099,7 @@ var _ = ginkgo.Describe("Cluster Manager", func() { // Store the node 3's gw router port addresses if updatedNode.Name == "node3" { - node3GWRPAnnotation = updatedNode.Annotations[ovnNodeGRLRPAddrAnnotaton] + node3GWRPAnnotation = updatedNode.Annotations[util.OVNNodeGRLRPAddrs] } return nil }).ShouldNot(gomega.HaveOccurred()) @@ -1149,10 +1147,10 @@ var _ = ginkgo.Describe("Cluster Manager", func() { return err } - node3UpdatedGWRPAnnotation := updatedNode.Annotations[ovnNodeGRLRPAddrAnnotaton] + node3UpdatedGWRPAnnotation := updatedNode.Annotations[util.OVNNodeGRLRPAddrs] gomega.Expect(node3UpdatedGWRPAnnotation).NotTo(gomega.Equal(node3GWRPAnnotation)) - gwLRPAddrs, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode) + gwLRPAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName) if err != nil { return err } diff --git a/go-controller/pkg/clustermanager/network_cluster_controller_test.go b/go-controller/pkg/clustermanager/network_cluster_controller_test.go index f98ac13d9f2..44c529a6f58 100644 --- a/go-controller/pkg/clustermanager/network_cluster_controller_test.go +++ b/go-controller/pkg/clustermanager/network_cluster_controller_test.go @@ -54,6 +54,7 @@ var _ = ginkgo.Describe("Network Cluster Controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: "node1", Annotations: map[string]string{ + ovnNodeIDAnnotaton: "3", "k8s.ovn.org/node-subnets": "{\"default\":[\"10.128.0.0/24\", \"fd02:0:0:2::2895/64\"]}", }, }, @@ -104,6 +105,9 @@ var _ = ginkgo.Describe("Network Cluster Controller", func() { { ObjectMeta: metav1.ObjectMeta{ Name: "node1", + Annotations: map[string]string{ + ovnNodeIDAnnotaton: "3", + }, }, }, } @@ -154,6 +158,7 @@ var _ = ginkgo.Describe("Network Cluster Controller", func() { Name: "node1", Annotations: map[string]string{ "k8s.ovn.org/node-subnets": "{\"default\":[\"10.128.0.0/24\", \"1.2.3.0/24\"]}", + ovnNodeIDAnnotaton: "3", }, }, }, diff --git a/go-controller/pkg/clustermanager/node/node_allocator.go b/go-controller/pkg/clustermanager/node/node_allocator.go index 14c853af2ff..c1277c59fae 100644 --- a/go-controller/pkg/clustermanager/node/node_allocator.go +++ b/go-controller/pkg/clustermanager/node/node_allocator.go @@ -11,9 +11,12 @@ import ( "k8s.io/klog/v2" utilnet "k8s.io/utils/net" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types" houtil "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/util" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" + ipgenerator "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/generator/ip" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" @@ -33,6 +36,9 @@ type NodeAllocator struct { clusterSubnetAllocator SubnetAllocator hybridOverlaySubnetAllocator SubnetAllocator + // node gateway router port IP generators (connecting to the join switch) + nodeGWRouterLRPIPv4Generator *ipgenerator.IPGenerator + nodeGWRouterLRPIPv6Generator *ipgenerator.IPGenerator // unique id of the network networkID int @@ -62,6 +68,24 @@ func NewNodeAllocator(networkID int, netInfo util.NetInfo, nodeLister listers.No } func (na *NodeAllocator) Init() error { + if na.hasJoinSubnetAllocation() { + if config.IPv4Mode { + nodeGWRouterLRPIPv4Generator, err := ipgenerator.NewIPGenerator(na.netInfo.JoinSubnetV4().String()) + if err != nil { + return fmt.Errorf("error creating IP Generator for v4 join subnet %s: %w", na.netInfo.JoinSubnetV4().String(), err) + } + na.nodeGWRouterLRPIPv4Generator = nodeGWRouterLRPIPv4Generator + } + + if config.IPv6Mode { + nodeGWRouterLRPIPv6Generator, err := ipgenerator.NewIPGenerator(na.netInfo.JoinSubnetV6().String()) + if err != nil { + return fmt.Errorf("error creating IP Generator for v6 join subnet %s: %w", na.netInfo.JoinSubnetV6().String(), err) + } + na.nodeGWRouterLRPIPv6Generator = nodeGWRouterLRPIPv6Generator + } + } + if !na.hasNodeSubnetAllocation() { return nil } @@ -185,7 +209,47 @@ func (na *NodeAllocator) syncNodeNetworkAnnotations(node *corev1.Node) error { } updatedSubnetsMap := map[string][]*net.IPNet{} - var validExistingSubnets, allocatedSubnets []*net.IPNet + var validExistingSubnets, allocatedSubnets, allocatedJoinSubnets []*net.IPNet + if na.hasJoinSubnetAllocation() { + var joinAddr []*net.IPNet + existingSubnets, err := util.ParseNodeGatewayRouterJoinAddrs(node, networkName) + if err != nil && !util.IsAnnotationNotSetError(err) { + // Log the error and try to allocate new subnets + klog.Warningf("Failed to get node %s join subnets annotations for network %s: %v", node.Name, networkName, err) + } + // Allocate the IP address(es) for the node Gateway router port connecting + // to the Join switch + nodeID := util.GetNodeID(node) + if nodeID == -1 { + // Don't consider this node as cluster-manager has not allocated node id yet. + return fmt.Errorf("failed to get node id for node - %s", node.Name) + } + + if config.IPv4Mode { + joinV4Addr, err := na.nodeGWRouterLRPIPv4Generator.GenerateIP(nodeID) + if err != nil { + return fmt.Errorf("failed to generate gateway router port IPv4 address for node %s : err - %w", node.Name, err) + } + joinAddr = append(joinAddr, joinV4Addr) + } + + if config.IPv6Mode { + joinV6Addr, err := na.nodeGWRouterLRPIPv6Generator.GenerateIP(nodeID) + if err != nil { + return fmt.Errorf("failed to generate gateway router port IPv6 address for node %s : err - %w", node.Name, err) + } + joinAddr = append(joinAddr, joinV6Addr) + } + // If the existing subnets weren't OK, or new ones were allocated, update the node annotation. + // This happens in a couple cases: + // 1) new node: no existing subnets and one or more new subnets were allocated + // 2) dual-stack/single-stack conversion: two existing subnets but only one will be valid, and no allocated subnets + // 3) bad subnet annotation: one more existing subnets will be invalid and might have allocated a correct one; let us reset it + lessIPNet := func(a, b net.IPNet) bool { return a.String() < b.String() } + if !cmp.Equal(existingSubnets, joinAddr, cmpopts.SortSlices(lessIPNet)) { + allocatedJoinSubnets = joinAddr + } + } if na.hasNodeSubnetAllocation() { existingSubnets, err := util.ParseNodeHostSubnetAnnotation(node, networkName) if err != nil && !util.IsAnnotationNotSetError(err) { @@ -214,8 +278,8 @@ func (na *NodeAllocator) syncNodeNetworkAnnotations(node *corev1.Node) error { } // Also update the node annotation if the networkID doesn't match - if len(updatedSubnetsMap) > 0 || na.networkID != networkID { - err = na.updateNodeNetworkAnnotationsWithRetry(node.Name, updatedSubnetsMap, na.networkID) + if len(updatedSubnetsMap) > 0 || na.networkID != networkID || len(allocatedJoinSubnets) > 0 { + err = na.updateNodeNetworkAnnotationsWithRetry(node.Name, updatedSubnetsMap, na.networkID, allocatedJoinSubnets) if err != nil { if errR := na.clusterSubnetAllocator.ReleaseNetworks(node.Name, allocatedSubnets...); errR != nil { klog.Warningf("Error releasing node %s subnets: %v", node.Name, errR) @@ -287,7 +351,7 @@ func (na *NodeAllocator) Sync(nodes []interface{}) error { } // updateNodeNetworkAnnotationsWithRetry will update the node's subnet annotation and network id annotation -func (na *NodeAllocator) updateNodeNetworkAnnotationsWithRetry(nodeName string, hostSubnetsMap map[string][]*net.IPNet, networkId int) error { +func (na *NodeAllocator) updateNodeNetworkAnnotationsWithRetry(nodeName string, hostSubnetsMap map[string][]*net.IPNet, networkId int, joinAddr []*net.IPNet) error { // Retry if it fails because of potential conflict which is transient. Return error in the // case of other errors (say temporary API server down), and it will be taken care of by the // retry mechanism. @@ -309,6 +373,12 @@ func (na *NodeAllocator) updateNodeNetworkAnnotationsWithRetry(nodeName string, networkName := na.netInfo.GetNetworkName() + cnode.Annotations, err = util.UpdateNodeGatewayRouterLRPAddrsAnnotation(cnode.Annotations, joinAddr, networkName) + if err != nil { + return fmt.Errorf("failed to update node %q annotation LRPAddrAnnotation %s", + node.Name, util.JoinIPNets(joinAddr, ",")) + } + cnode.Annotations, err = util.UpdateNetworkIDAnnotation(cnode.Annotations, networkName, networkId) if err != nil { return fmt.Errorf("failed to update node %q network id annotation %d for network %s", @@ -344,7 +414,7 @@ func (na *NodeAllocator) Cleanup() error { hostSubnetsMap := map[string][]*net.IPNet{networkName: nil} // passing util.InvalidNetworkID deletes the network id annotation for the network. - err = na.updateNodeNetworkAnnotationsWithRetry(node.Name, hostSubnetsMap, util.InvalidNetworkID) + err = na.updateNodeNetworkAnnotationsWithRetry(node.Name, hostSubnetsMap, util.InvalidNetworkID, nil) if err != nil { return fmt.Errorf("failed to clear node %q subnet annotation for network %s", node.Name, networkName) @@ -468,3 +538,8 @@ func (na *NodeAllocator) hasNodeSubnetAllocation() bool { // we only allocate subnets for L3 secondary network or default network return na.netInfo.TopologyType() == types.Layer3Topology || !na.netInfo.IsSecondary() } + +func (na *NodeAllocator) hasJoinSubnetAllocation() bool { + // we allocate join subnets for L3/L2 primary user defined networks or default network + return na.netInfo.IsDefault() || (util.IsNetworkSegmentationSupportEnabled() && na.netInfo.IsPrimaryNetwork()) +} diff --git a/go-controller/pkg/clustermanager/zone_cluster_controller.go b/go-controller/pkg/clustermanager/zone_cluster_controller.go index 89411393a53..ae3301469f3 100644 --- a/go-controller/pkg/clustermanager/zone_cluster_controller.go +++ b/go-controller/pkg/clustermanager/zone_cluster_controller.go @@ -41,10 +41,6 @@ type zoneClusterController struct { // ID allocator for the nodes nodeIDAllocator id.Allocator - // node gateway router port IP generators (connecting to the join switch) - nodeGWRouterLRPIPv4Generator *ipgenerator.IPGenerator - nodeGWRouterLRPIPv6Generator *ipgenerator.IPGenerator - // Transit switch IP generator. This is required if EnableInterconnect feature is enabled. transitSwitchIPv4Generator *ipgenerator.IPGenerator transitSwitchIPv6Generator *ipgenerator.IPGenerator @@ -70,22 +66,6 @@ func newZoneClusterController(ovnClient *util.OVNClusterManagerClientset, wf *fa } wg := &sync.WaitGroup{} - var nodeGWRouterLRPIPv4Generator, nodeGWRouterLRPIPv6Generator *ipgenerator.IPGenerator - - if config.IPv4Mode { - nodeGWRouterLRPIPv4Generator, err = ipgenerator.NewIPGenerator(config.Gateway.V4JoinSubnet) - if err != nil { - return nil, fmt.Errorf("error creating IP Generator for v4 join subnet %s: %w", config.Gateway.V4JoinSubnet, err) - } - } - - if config.IPv6Mode { - nodeGWRouterLRPIPv6Generator, err = ipgenerator.NewIPGenerator(config.Gateway.V6JoinSubnet) - if err != nil { - return nil, fmt.Errorf("error creating IP Generator for v6 join subnet %s: %w", config.Gateway.V6JoinSubnet, err) - } - } - var transitSwitchIPv4Generator, transitSwitchIPv6Generator *ipgenerator.IPGenerator if config.OVNKubernetesFeature.EnableInterconnect { @@ -105,15 +85,13 @@ func newZoneClusterController(ovnClient *util.OVNClusterManagerClientset, wf *fa } zcc := &zoneClusterController{ - kube: kube, - watchFactory: wf, - stopChan: make(chan struct{}), - wg: wg, - nodeIDAllocator: nodeIDAllocator, - nodeGWRouterLRPIPv4Generator: nodeGWRouterLRPIPv4Generator, - nodeGWRouterLRPIPv6Generator: nodeGWRouterLRPIPv6Generator, - transitSwitchIPv4Generator: transitSwitchIPv4Generator, - transitSwitchIPv6Generator: transitSwitchIPv6Generator, + kube: kube, + watchFactory: wf, + stopChan: make(chan struct{}), + wg: wg, + nodeIDAllocator: nodeIDAllocator, + transitSwitchIPv4Generator: transitSwitchIPv4Generator, + transitSwitchIPv6Generator: transitSwitchIPv6Generator, } zcc.initRetryFramework() @@ -173,25 +151,6 @@ func (zcc *zoneClusterController) handleAddUpdateNodeEvent(node *corev1.Node) er // Allocate the IP address(es) for the node Gateway router port connecting // to the Join switch var v4Addr, v6Addr *net.IPNet - if config.IPv4Mode { - v4Addr, err = zcc.nodeGWRouterLRPIPv4Generator.GenerateIP(allocatedNodeID) - if err != nil { - return fmt.Errorf("failed to generate gateway router port IPv4 address for node %s : err - %w", node.Name, err) - } - } - - if config.IPv6Mode { - v6Addr, err = zcc.nodeGWRouterLRPIPv6Generator.GenerateIP(allocatedNodeID) - if err != nil { - return fmt.Errorf("failed to generate gateway router port IPv6 address for node %s : err - %w", node.Name, err) - } - } - - nodeAnnotations, err = util.CreateNodeGatewayRouterLRPAddrAnnotation(nodeAnnotations, v4Addr, v6Addr) - if err != nil { - return fmt.Errorf("failed to marshal node %q annotation for Gateway LRP IPs, err : %v", - node.Name, err) - } if config.OVNKubernetesFeature.EnableInterconnect { v4Addr = nil @@ -369,7 +328,7 @@ func (h *zoneClusterControllerEventHandler) AreResourcesEqual(obj1, obj2 interfa if util.NodeIDAnnotationChanged(node1, node2) { return false, nil } - if util.NodeGatewayRouterLRPAddrAnnotationChanged(node1, node2) { + if util.NodeGatewayRouterLRPAddrsAnnotationChanged(node1, node2) { return false, nil } if util.NodeTransitSwitchPortAddrAnnotationChanged(node1, node2) { diff --git a/go-controller/pkg/cni/types/types.go b/go-controller/pkg/cni/types/types.go index 0289c07dbf4..442b1a6c76d 100644 --- a/go-controller/pkg/cni/types/types.go +++ b/go-controller/pkg/cni/types/types.go @@ -32,6 +32,16 @@ type NetConf struct { // valid for layer2 and localnet network topology // eg. "10.1.130.0/27, 10.1.130.122/32" ExcludeSubnets string `json:"excludeSubnets,omitempty"` + // join subnet cidr is required for supporting + // services and ingress for user defined networks + // in case of dualstack cluster, please do a comma-seperated list + // expected format: + // 1) V4 single stack: "v4CIDR" (eg: "100.65.0.0/16") + // 2) V6 single stack: "v6CIDR" (eg: "fd99::/64") + // 3) dualstack: "v4CIDR,v6CIDR" (eg: "100.65.0.0/16,fd99::/64") + // valid for UDN layer3/layer2 network topology + // default value: 100.65.0.0/16,fd99::/64 if not provided + JoinSubnet string `json:"joinSubnet,omitempty"` // VLANID, valid in localnet topology network only VLANID int `json:"vlanID,omitempty"` // AllowPersistentIPs is valid on both localnet / layer topologies. diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index f47addd3840..32cbe098e38 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -162,6 +162,10 @@ type BaseNetworkController struct { // might have been already be released on startup releasedPodsBeforeStartup map[string]sets.Set[string] releasedPodsOnStartupMutex sync.Mutex + + // IP addresses of OVN Cluster logical router port ("GwRouterToJoinSwitchPrefix + OVNClusterRouter") + // connecting to the join switch + ovnClusterLRPToJoinIfAddrs []*net.IPNet } // BaseSecondaryNetworkController structure holds per-network fields and network specific @@ -256,6 +260,30 @@ func (bnc *BaseNetworkController) createOvnClusterRouter() (*nbdb.LogicalRouter, return &logicalRouter, nil } +// getOVNClusterRouterPortToJoinSwitchIPs returns the IP addresses for the +// logical router port "GwRouterToJoinSwitchPrefix + OVNClusterRouter" from the +// config.Gateway.V4JoinSubnet and config.Gateway.V6JoinSubnet. This will +// always be the first IP from these subnets. +func (bnc *BaseNetworkController) getOVNClusterRouterPortToJoinSwitchIfAddrs() (gwLRPIPs []*net.IPNet, err error) { + joinSubnetsConfig := []*net.IPNet{} + if config.IPv4Mode { + joinSubnetsConfig = append(joinSubnetsConfig, bnc.JoinSubnetV4()) + } + if config.IPv6Mode { + joinSubnetsConfig = append(joinSubnetsConfig, bnc.JoinSubnetV6()) + } + for _, joinSubnet := range joinSubnetsConfig { + joinSubnetBaseIP := utilnet.BigForIP(joinSubnet.IP) + ipnet := &net.IPNet{ + IP: utilnet.AddIPOffset(joinSubnetBaseIP, 1), + Mask: joinSubnet.Mask, + } + gwLRPIPs = append(gwLRPIPs, ipnet) + } + + return gwLRPIPs, nil +} + // syncNodeClusterRouterPort ensures a node's LS to the cluster router's LRP is created. // NOTE: We could have created the router port in createNodeLogicalSwitch() instead of here, // but chassis ID is not available at that moment. We need the chassis ID to set the @@ -847,8 +875,8 @@ func (bnc *BaseNetworkController) nodeZoneClusterChanged(oldNode, newNode *kapi. return true } - // NodeGatewayRouterLRPAddrAnnotationChanged would not affect local, nor layer3 secondary network - if !newNodeIsLocalZone && !bnc.IsSecondary() && util.NodeGatewayRouterLRPAddrAnnotationChanged(oldNode, newNode) { + // NodeGatewayRouterLRPAddrsAnnotationChanged would not affect local, nor localnet secondary network + if !newNodeIsLocalZone && bnc.NetInfo.TopologyType() != types.LocalnetTopology && util.NodeGatewayRouterLRPAddrsAnnotationChanged(oldNode, newNode) { return true } diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index bdc2843fa36..b6ac0a9252b 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -3,7 +3,6 @@ package ovn import ( "context" "fmt" - "net" "reflect" "sync" "time" @@ -132,10 +131,6 @@ type DefaultNetworkController struct { // updated atomically allInitialPodsProcessed uint32 - // IP addresses of OVN Cluster logical router port ("GwRouterToJoinSwitchPrefix + OVNClusterRouter") - // connecting to the join switch - ovnClusterLRPToJoinIfAddrs []*net.IPNet - // zoneChassisHandler handles the local node and remote nodes in creating or updating the chassis entries in the OVN Southbound DB. // Please see zone_interconnect/chassis_handler.go for more details. zoneChassisHandler *zoneic.ZoneChassisHandler diff --git a/go-controller/pkg/ovn/egressfirewall_test.go b/go-controller/pkg/ovn/egressfirewall_test.go index 3d93ae7a013..8e621536750 100644 --- a/go-controller/pkg/ovn/egressfirewall_test.go +++ b/go-controller/pkg/ovn/egressfirewall_test.go @@ -841,10 +841,10 @@ var _ = ginkgo.Describe("OVN EgressFirewall Operations", func() { Annotations: map[string]string{ // this will cause node add failure, stolen from // master_test:reconciles node host subnets after dual-stack to single-stack downgrade - "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\", \"fd02:0:0:2::2895/64\"]}", v4NodeSubnet), - util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s/24\",\"%s/24\",\"%s/64\"]", nodeIP, nodeIP2, nodeIP3), - "k8s.ovn.org/node-chassis-id": "2", - "k8s.ovn.org/node-gateway-router-lrp-ifaddr": "{\"ipv4\":\"100.64.0.2/16\"}", + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\", \"fd02:0:0:2::2895/64\"]}", v4NodeSubnet), + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s/24\",\"%s/24\",\"%s/64\"]", nodeIP, nodeIP2, nodeIP3), + "k8s.ovn.org/node-chassis-id": "2", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.2/16\"}}", }, }, } diff --git a/go-controller/pkg/ovn/hybrid_test.go b/go-controller/pkg/ovn/hybrid_test.go index 4aa14f7118e..651ecb22bf2 100644 --- a/go-controller/pkg/ovn/hybrid_test.go +++ b/go-controller/pkg/ovn/hybrid_test.go @@ -1638,10 +1638,10 @@ var _ = ginkgo.Describe("Hybrid SDN Master Operations", func() { } testNode := node1.k8sNode("2") testNode.Annotations = map[string]string{ - hotypes.HybridOverlayDRIP: nodeHOIP, - hotypes.HybridOverlayDRMAC: nodeHOMAC, - "k8s.ovn.org/ovn-node-id": "2", - "k8s.ovn.org/node-gateway-router-lrp-ifaddr": "{\"ipv4\": \"100.64.0.2/16\"}"} + hotypes.HybridOverlayDRIP: nodeHOIP, + hotypes.HybridOverlayDRMAC: nodeHOMAC, + "k8s.ovn.org/ovn-node-id": "2", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.2/16\"}}"} kubeFakeClient := fake.NewSimpleClientset(&v1.NodeList{ Items: []v1.Node{testNode}, diff --git a/go-controller/pkg/ovn/kubevirt_test.go b/go-controller/pkg/ovn/kubevirt_test.go index f7374b65d90..f0796b7f6fc 100644 --- a/go-controller/pkg/ovn/kubevirt_test.go +++ b/go-controller/pkg/ovn/kubevirt_test.go @@ -798,8 +798,8 @@ var _ = Describe("OVN Kubevirt Operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "newNode1", Annotations: map[string]string{ - "k8s.ovn.org/node-subnets": fmt.Sprintf(`{"default":[%q,%q]}`, nodeByName[t.replaceNode].subnetIPv4, nodeByName[t.replaceNode].subnetIPv6), - "k8s.ovn.org/node-gateway-router-lrp-ifaddr": `{"ipv4": "100.64.0.2/16"}`, + "k8s.ovn.org/node-subnets": fmt.Sprintf(`{"default":[%q,%q]}`, nodeByName[t.replaceNode].subnetIPv4, nodeByName[t.replaceNode].subnetIPv6), + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.2/16\"}}", }, }, } diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index d10bce47f6b..bbb51899b8a 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -171,9 +171,17 @@ func (oc *DefaultNetworkController) syncGatewayLogicalNetwork(node *kapi.Node, l clusterSubnets = append(clusterSubnets, clusterSubnet.CIDR) } - gwLRPIPs, err = util.ParseNodeGatewayRouterLRPAddrs(node) + gwLRPIPs, err = util.ParseNodeGatewayRouterJoinAddrs(node, oc.GetNetworkName()) if err != nil { - return fmt.Errorf("failed to get join switch port IP address for node %s: %v", node.Name, err) + if util.IsAnnotationNotSetError(err) { + // FIXME(tssurya): This is present for backwards compatibility + // Remove me a few months from now + var err1 error + gwLRPIPs, err1 = util.ParseNodeGatewayRouterLRPAddrs(node) + if err1 != nil { + return fmt.Errorf("failed to get join switch port IP address for node %s: %v/%v", node.Name, err, err1) + } + } } enableGatewayMTU := util.ParseNodeGatewayMTUSupport(node) @@ -781,34 +789,6 @@ func (oc *DefaultNetworkController) deleteOVNNodeEvent(node *kapi.Node) error { return nil } -// getOVNClusterRouterPortToJoinSwitchIPs returns the IP addresses for the -// logical router port "GwRouterToJoinSwitchPrefix + OVNClusterRouter" from the -// config.Gateway.V4JoinSubnet and config.Gateway.V6JoinSubnet. This will -// always be the first IP from these subnets. -func (oc *DefaultNetworkController) getOVNClusterRouterPortToJoinSwitchIfAddrs() (gwLRPIPs []*net.IPNet, err error) { - joinSubnetsConfig := []string{} - if config.IPv4Mode { - joinSubnetsConfig = append(joinSubnetsConfig, config.Gateway.V4JoinSubnet) - } - if config.IPv6Mode { - joinSubnetsConfig = append(joinSubnetsConfig, config.Gateway.V6JoinSubnet) - } - for _, joinSubnetString := range joinSubnetsConfig { - _, joinSubnet, err := net.ParseCIDR(joinSubnetString) - if err != nil { - return nil, fmt.Errorf("error parsing join subnet string %s: %v", joinSubnetString, err) - } - joinSubnetBaseIP := utilnet.BigForIP(joinSubnet.IP) - ipnet := &net.IPNet{ - IP: utilnet.AddIPOffset(joinSubnetBaseIP, 1), - Mask: joinSubnet.Mask, - } - gwLRPIPs = append(gwLRPIPs, ipnet) - } - - return gwLRPIPs, nil -} - // addUpdateHoNodeEvent reconsile ovn nodes when a hybrid overlay node is added. func (oc *DefaultNetworkController) addUpdateHoNodeEvent(node *kapi.Node) error { if subnets, _ := util.ParseNodeHostSubnetAnnotation(node, oc.GetNetworkName()); len(subnets) > 0 { diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index d4a3a4a5d01..bd3ebd26dbf 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -73,10 +73,7 @@ type tNode struct { const ( // ovnNodeID is the id (of type integer) of a node. It is set by cluster-manager. - ovnNodeID = "k8s.ovn.org/node-id" - - // ovnNodeGRLRPAddr is the CIDR form representation of Gate Router LRP IP address to join switch (i.e: 100.64.0.5/24) - ovnNodeGRLRPAddr = "k8s.ovn.org/node-gateway-router-lrp-ifaddr" + ovnNodeID = "k8s.ovn.org/node-id" ovnNodePrimaryIfAddr = "k8s.ovn.org/node-primary-ifaddr" ovnNodeSubnets = "k8s.ovn.org/node-subnets" ) @@ -86,10 +83,10 @@ func (n tNode) k8sNode(nodeID string) v1.Node { ObjectMeta: metav1.ObjectMeta{ Name: n.Name, Annotations: map[string]string{ - ovnNodeID: nodeID, - ovnNodeGRLRPAddr: "{\"ipv4\": \"100.64.0." + nodeID + "/16\"}", - util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", fmt.Sprintf("%s/24", n.NodeIP)), - ovnNodePrimaryIfAddr: fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", fmt.Sprintf("%s/24", n.NodeIP), ""), + ovnNodeID: nodeID, + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\": \"100.64.0." + nodeID + "/16\"}}", + util.OVNNodeHostCIDRs: fmt.Sprintf("[\"%s\"]", fmt.Sprintf("%s/24", n.NodeIP)), + ovnNodePrimaryIfAddr: fmt.Sprintf("{\"ipv4\": \"%s\", \"ipv6\": \"%s\"}", fmt.Sprintf("%s/24", n.NodeIP), ""), }, }, Status: v1.NodeStatus{ @@ -1604,9 +1601,9 @@ var _ = ginkgo.Describe("Default network controller operations", func() { ObjectMeta: metav1.ObjectMeta{ Name: "newNode", Annotations: map[string]string{ - "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\", \"fd02:0:0:2::2895/64\"]}", newNodeSubnet), - "k8s.ovn.org/node-chassis-id": "2", - "k8s.ovn.org/node-gateway-router-lrp-ifaddr": "{\"ipv4\":\"100.64.0.2/16\"}", + "k8s.ovn.org/node-subnets": fmt.Sprintf("{\"default\":[\"%s\", \"fd02:0:0:2::2895/64\"]}", newNodeSubnet), + "k8s.ovn.org/node-chassis-id": "2", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.2/16\"}}", }, }, } @@ -1793,7 +1790,7 @@ var _ = ginkgo.Describe("Default network controller operations", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) mgmt_ips := util.GetNodeManagementIfAddr(hostSubnets[0]) ips = append(ips, mgmt_ips.IP.String()) - lrpips, err := util.ParseNodeGatewayRouterLRPAddrs(updatedNode) + lrpips, err := util.ParseNodeGatewayRouterJoinAddrs(updatedNode, types.DefaultNetworkName) gomega.Expect(err).NotTo(gomega.HaveOccurred()) lrpip, _, _ := net.ParseCIDR(lrpips[0].String()) ips = append(ips, lrpip.String()) diff --git a/go-controller/pkg/ovn/namespace.go b/go-controller/pkg/ovn/namespace.go index 249025096bb..a804492a6ec 100644 --- a/go-controller/pkg/ovn/namespace.go +++ b/go-controller/pkg/ovn/namespace.go @@ -348,9 +348,17 @@ func (oc *DefaultNetworkController) getHostNamespaceAddressesForNode(node *kapi. } // for shared gateway mode we will use LRP IPs to SNAT host network traffic // so add these to the address set. - lrpIPs, err := util.ParseNodeGatewayRouterLRPAddrs(node) + lrpIPs, err := util.ParseNodeGatewayRouterJoinAddrs(node, oc.GetNetworkName()) if err != nil { - return nil, err + if util.IsAnnotationNotSetError(err) { + // FIXME(tssurya): This is present for backwards compatibility + // Remove me a few months from now + var err1 error + lrpIPs, err1 = util.ParseNodeGatewayRouterLRPAddrs(node) + if err1 != nil { + return nil, fmt.Errorf("failed to get join switch port IP address for node %s: %v/%v", node.Name, err, err1) + } + } } for _, lrpIP := range lrpIPs { diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index 32b47c8e5c1..e0ccc2e92ea 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -308,7 +308,8 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { // be in the addressset yet, depending on if the host subnets annotation of the node exists in the informer cache. The addressset // can only be deterministic when WatchNamespaces() handles this host network namespace. - gwLRPIPs, err := util.ParseNodeGatewayRouterLRPAddrs(&testNode) + gwLRPIPs, err := util.ParseNodeGatewayRouterJoinAddrs(&testNode, ovntypes.DefaultNetworkName) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) gomega.Expect(len(gwLRPIPs) != 0).To(gomega.BeTrue()) err = fakeOvn.controller.WatchNamespaces() diff --git a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go index 52db87604e2..7fedc11261a 100644 --- a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go +++ b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go @@ -446,7 +446,7 @@ func (zic *ZoneInterconnectHandler) createLocalZoneNodeResources(node *corev1.No // Its possible that node is moved from a remote zone to the local zone. Check and delete the remote zone routes // for this node as it's no longer needed. - return zic.deleteLocalNodeStaticRoutes(node, nodeID, nodeTransitSwitchPortIPs) + return zic.deleteLocalNodeStaticRoutes(node, nodeTransitSwitchPortIPs) } // createRemoteZoneNodeResources creates the remote zone node resources @@ -668,9 +668,17 @@ func (zic *ZoneInterconnectHandler) addRemoteNodeStaticRoutes(node *corev1.Node, return nil } - nodeGRPIPs, err := util.ParseNodeGatewayRouterLRPAddrs(node) + nodeGRPIPs, err := util.ParseNodeGatewayRouterJoinAddrs(node, zic.GetNetworkName()) if err != nil { - return fmt.Errorf("failed to parse node %s Gateway router LRP Addrs annotation %w", node.Name, err) + if util.IsAnnotationNotSetError(err) { + // FIXME(tssurya): This is present for backwards compatibility + // Remove me a few months from now + var err1 error + nodeGRPIPs, err1 = util.ParseNodeGatewayRouterLRPAddrs(node) + if err1 != nil { + return fmt.Errorf("failed to parse node %s Gateway router LRP Addrs annotation %w", node.Name, err1) + } + } } nodeGRPIPStaticRoutes := zic.getStaticRoutes(nodeGRPIPs, nodeTransitSwitchPortIPs, true) @@ -685,7 +693,7 @@ func (zic *ZoneInterconnectHandler) addRemoteNodeStaticRoutes(node *corev1.Node, } // deleteLocalNodeStaticRoutes deletes the static routes added by the function addRemoteNodeStaticRoutes -func (zic *ZoneInterconnectHandler) deleteLocalNodeStaticRoutes(node *corev1.Node, nodeID int, nodeTransitSwitchPortIPs []*net.IPNet) error { +func (zic *ZoneInterconnectHandler) deleteLocalNodeStaticRoutes(node *corev1.Node, nodeTransitSwitchPortIPs []*net.IPNet) error { deleteRoute := func(prefix, nexthop string) error { p := func(lrsr *nbdb.LogicalRouterStaticRoute) bool { return lrsr.IPPrefix == prefix && @@ -718,9 +726,17 @@ func (zic *ZoneInterconnectHandler) deleteLocalNodeStaticRoutes(node *corev1.Nod } // Clear the routes connecting to the GW Router for the default network - nodeGRPIPs, err := util.ParseNodeGatewayRouterLRPAddrs(node) + nodeGRPIPs, err := util.ParseNodeGatewayRouterJoinAddrs(node, zic.GetNetworkName()) if err != nil { - return fmt.Errorf("failed to parse node %s Gateway router LRP Addrs annotation %w", node.Name, err) + if util.IsAnnotationNotSetError(err) { + // FIXME(tssurya): This is present for backwards compatibility + // Remove me a few months from now + var err1 error + nodeGRPIPs, err1 = util.ParseNodeGatewayRouterLRPAddrs(node) + if err1 != nil { + return fmt.Errorf("failed to parse node %s Gateway router LRP Addrs annotation %w", node.Name, err1) + } + } } nodenodeGRPIPStaticRoutes := zic.getStaticRoutes(nodeGRPIPs, nodeTransitSwitchPortIPs, true) diff --git a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler_test.go b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler_test.go index dfacf8ee0e9..2225ff74d71 100644 --- a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler_test.go +++ b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler_test.go @@ -28,9 +28,6 @@ const ( // ovnNodeIDAnnotaton is the node annotation name used to store the node id. ovnNodeIDAnnotaton = "k8s.ovn.org/node-id" - // ovnNodeGRLRPAddrAnnotaton is the node annotation name used to store the node gateway router port ips. - ovnNodeGRLRPAddrAnnotaton = "k8s.ovn.org/node-gateway-router-lrp-ifaddr" - // ovnTransitSwitchPortAddrAnnotation is the node annotation name to store the transit switch port ips. ovnTransitSwitchPortAddrAnnotation = "k8s.ovn.org/node-transit-switch-port-ifaddr" @@ -295,7 +292,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ovnNodeIDAnnotaton: "2", ovnNodeSubnetsAnnotation: "{\"default\":[\"10.244.2.0/24\"]}", ovnTransitSwitchPortAddrAnnotation: "{\"ipv4\":\"100.88.0.2/16\"}", - ovnNodeGRLRPAddrAnnotaton: "{\"ipv4\":\"100.64.0.2/16\"}", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.2/16\"}}", ovnNodeNetworkIDsAnnotation: "{\"default\":\"0\"}", }, }, @@ -313,7 +310,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ovnNodeIDAnnotaton: "3", ovnNodeSubnetsAnnotation: "{\"default\":[\"10.244.3.0/24\"]}", ovnTransitSwitchPortAddrAnnotation: "{\"ipv4\":\"100.88.0.3/16\"}", - ovnNodeGRLRPAddrAnnotaton: "{\"ipv4\":\"100.64.0.3/16\"}", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.3/16\"}}", ovnNodeNetworkIDsAnnotation: "{\"default\":\"0\"}", }, }, @@ -331,7 +328,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ovnNodeIDAnnotaton: "4", ovnNodeSubnetsAnnotation: "{\"default\":[\"10.244.4.0/24\"]}", ovnTransitSwitchPortAddrAnnotation: "{\"ipv4\":\"100.88.0.4/16\"}", - ovnNodeGRLRPAddrAnnotaton: "{\"ipv4\":\"100.64.0.4/16\"}", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.4/16\"}}", ovnNodeNetworkIDsAnnotation: "{\"default\":\"0\"}", }, }, @@ -591,7 +588,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ovnNodeIDAnnotaton: "2", ovnNodeSubnetsAnnotation: "{\"blue\":[\"10.244.2.0/24\"]}", ovnTransitSwitchPortAddrAnnotation: "{\"ipv4\":\"100.88.0.2/16\"}", - ovnNodeGRLRPAddrAnnotaton: "{\"ipv4\":\"100.64.0.2/16\"}", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.2/16\"}}", ovnNodeNetworkIDsAnnotation: "{\"blue\":\"1\"}", }, }, @@ -609,7 +606,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ovnNodeIDAnnotaton: "3", ovnNodeSubnetsAnnotation: "{\"blue\":[\"10.244.3.0/24\"]}", ovnTransitSwitchPortAddrAnnotation: "{\"ipv4\":\"100.88.0.3/16\"}", - ovnNodeGRLRPAddrAnnotaton: "{\"ipv4\":\"100.64.0.3/16\"}", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.3/16\"}}", ovnNodeNetworkIDsAnnotation: "{\"blue\":\"1\"}", }, }, @@ -627,7 +624,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { ovnNodeIDAnnotaton: "4", ovnNodeSubnetsAnnotation: "{\"blue\":[\"10.244.4.0/24\"]}", ovnTransitSwitchPortAddrAnnotation: "{\"ipv4\":\"100.88.0.4/16\"}", - ovnNodeGRLRPAddrAnnotaton: "{\"ipv4\":\"100.64.0.4/16\"}", + util.OVNNodeGRLRPAddrs: "{\"default\":{\"ipv4\":\"100.64.0.4/16\"}}", ovnNodeNetworkIDsAnnotation: "{\"blue\":\"1\"}", }, }, @@ -805,7 +802,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { gomega.Expect(err).To(gomega.HaveOccurred(), "failed to parse node node4 GR IPs annotation") // Set node ovn-gw-router-port-ips annotation - testNode4.Annotations[ovnNodeGRLRPAddrAnnotaton] = "{\"ipv4\":\"100.64.0.5/16\"}" + testNode4.Annotations[util.OVNNodeGRLRPAddrs] = "{\"default\":{\"ipv4\":\"100.64.0.5/16\"}}" err = zoneICHandler.AddLocalZoneNode(&testNode4) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -908,7 +905,7 @@ var _ = ginkgo.Describe("Zone Interconnect Operations", func() { gomega.Expect(err).To(gomega.HaveOccurred(), "failed to parse node node4 GR IPs annotation") // Set node ovn-gw-router-port-ips annotation - testNode4.Annotations[ovnNodeGRLRPAddrAnnotaton] = "{\"ipv4\":\"100.64.0.5/16\"}" + testNode4.Annotations[util.OVNNodeGRLRPAddrs] = "{\"default\":{\"ipv4\":\"100.64.0.5/16\"}}" err = zoneICHandler.AddRemoteZoneNode(&testNode4) gomega.Expect(err).NotTo(gomega.HaveOccurred()) testNodesRouteInfo = map[string]map[string]string{ diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index 240cdf14936..8c78209ac71 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -113,15 +113,10 @@ const ( EgressIPNodeConnectionMark = "1008" EgressIPReplyTrafficConnectionMark = 42 - V6NodeLocalNATSubnet = "fd99::/64" - V6NodeLocalNATSubnetPrefix = 64 - V6NodeLocalNATSubnetNextHop = "fd99::1" - V6NodeLocalDistributedGWPortIP = "fd99::2" - - V4NodeLocalNATSubnet = "169.254.0.0/20" - V4NodeLocalNATSubnetPrefix = 20 - V4NodeLocalNATSubnetNextHop = "169.254.0.1" - V4NodeLocalDistributedGWPortIP = "169.254.0.2" + // primary user defined network's default join subnet value + // users can configure custom values using NADs + UserDefinedPrimaryNetworkJoinSubnetV4 = "100.65.0.0/16" + UserDefinedPrimaryNetworkJoinSubnetV6 = "fd99::/64" // OpenFlow and Networking constants RouteAdvertisementICMPType = 134 diff --git a/go-controller/pkg/util/multi_network.go b/go-controller/pkg/util/multi_network.go index 2d3d6646f31..43bb324fdee 100644 --- a/go-controller/pkg/util/multi_network.go +++ b/go-controller/pkg/util/multi_network.go @@ -37,6 +37,9 @@ type BasicNetInfo interface { IPMode() (bool, bool) Subnets() []config.CIDRNetworkEntry ExcludeSubnets() []*net.IPNet + JoinSubnetV4() *net.IPNet + JoinSubnetV6() *net.IPNet + JoinSubnets() []*net.IPNet Vlan() uint AllowsPersistentIPs() bool @@ -184,6 +187,47 @@ func (nInfo *DefaultNetInfo) ExcludeSubnets() []*net.IPNet { return nil } +// JoinSubnetV4 returns the defaultNetConfInfo's JoinSubnetV4 value +// call when ipv4mode=true +func (nInfo *DefaultNetInfo) JoinSubnetV4() *net.IPNet { + _, cidr, err := net.ParseCIDR(config.Gateway.V4JoinSubnet) + if err != nil { + // Join subnet should have been validated already by config + panic(fmt.Sprintf("Failed to parse join subnet %q: %v", config.Gateway.V4JoinSubnet, err)) + } + return cidr +} + +// JoinSubnetV6 returns the defaultNetConfInfo's JoinSubnetV6 value +// call when ipv6mode=true +func (nInfo *DefaultNetInfo) JoinSubnetV6() *net.IPNet { + _, cidr, err := net.ParseCIDR(config.Gateway.V6JoinSubnet) + if err != nil { + // Join subnet should have been validated already by config + panic(fmt.Sprintf("Failed to parse join subnet %q: %v", config.Gateway.V6JoinSubnet, err)) + } + return cidr +} + +// JoinSubnets returns the secondaryNetInfo's joinsubnet values (both v4&v6) +// used from Equals +func (nInfo *DefaultNetInfo) JoinSubnets() []*net.IPNet { + var defaultJoinSubnets []*net.IPNet + _, v4, err := net.ParseCIDR(config.Gateway.V4JoinSubnet) + if err != nil { + // Join subnet should have been validated already by config + panic(fmt.Sprintf("Failed to parse join subnet %q: %v", config.Gateway.V4JoinSubnet, err)) + } + defaultJoinSubnets = append(defaultJoinSubnets, v4) + _, v6, err := net.ParseCIDR(config.Gateway.V6JoinSubnet) + if err != nil { + // Join subnet should have been validated already by config + panic(fmt.Sprintf("Failed to parse join subnet %q: %v", config.Gateway.V6JoinSubnet, err)) + } + defaultJoinSubnets = append(defaultJoinSubnets, v6) + return defaultJoinSubnets +} + // Vlan returns the defaultNetConfInfo's Vlan value func (nInfo *DefaultNetInfo) Vlan() uint { return config.Gateway.VLANID @@ -208,6 +252,7 @@ type secondaryNetInfo struct { ipv4mode, ipv6mode bool subnets []config.CIDRNetworkEntry excludeSubnets []*net.IPNet + joinSubnets []*net.IPNet // all net-attach-def NAD names for this network, used to determine if a pod needs // to be plumbed for this network @@ -350,6 +395,31 @@ func (nInfo *secondaryNetInfo) ExcludeSubnets() []*net.IPNet { return nInfo.excludeSubnets } +// JoinSubnetV4 returns the defaultNetConfInfo's JoinSubnetV4 value +// call when ipv4mode=true +func (nInfo *secondaryNetInfo) JoinSubnetV4() *net.IPNet { + if len(nInfo.joinSubnets) == 0 { + return nil // localnet topology + } + return nInfo.joinSubnets[0] +} + +// JoinSubnetV6 returns the secondaryNetInfo's JoinSubnetV6 value +// call when ipv6mode=true +func (nInfo *secondaryNetInfo) JoinSubnetV6() *net.IPNet { + if len(nInfo.joinSubnets) <= 1 { + return nil // localnet topology + } + return nInfo.joinSubnets[1] +} + +// JoinSubnets returns the secondaryNetInfo's joinsubnet values (both v4&v6) +// used from Equals (since localnet doesn't have joinsubnets to compare nil v/s nil +// we need this util) +func (nInfo *secondaryNetInfo) JoinSubnets() []*net.IPNet { + return nInfo.joinSubnets +} + // Equals compares for equality this network information with the other func (nInfo *secondaryNetInfo) Equals(other BasicNetInfo) bool { if (nInfo == nil) != (other == nil) { @@ -383,7 +453,10 @@ func (nInfo *secondaryNetInfo) Equals(other BasicNetInfo) bool { } lessIPNet := func(a, b net.IPNet) bool { return a.String() < b.String() } - return cmp.Equal(nInfo.excludeSubnets, other.ExcludeSubnets(), cmpopts.SortSlices(lessIPNet)) + if !cmp.Equal(nInfo.excludeSubnets, other.ExcludeSubnets(), cmpopts.SortSlices(lessIPNet)) { + return false + } + return cmp.Equal(nInfo.joinSubnets, other.JoinSubnets(), cmpopts.SortSlices(lessIPNet)) } func (nInfo *secondaryNetInfo) copy() *secondaryNetInfo { @@ -402,6 +475,7 @@ func (nInfo *secondaryNetInfo) copy() *secondaryNetInfo { ipv6mode: nInfo.ipv6mode, subnets: nInfo.subnets, excludeSubnets: nInfo.excludeSubnets, + joinSubnets: nInfo.joinSubnets, nadNames: nInfo.nadNames.Clone(), } @@ -413,12 +487,16 @@ func newLayer3NetConfInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { if err != nil { return nil, err } - + joinSubnets, err := parseJoinSubnet(netconf.JoinSubnet) + if err != nil { + return nil, err + } ni := &secondaryNetInfo{ netName: netconf.Name, primaryNetwork: netconf.Role == types.NetworkRolePrimary, topology: types.Layer3Topology, subnets: subnets, + joinSubnets: joinSubnets, mtu: netconf.MTU, nadNames: sets.Set[string]{}, } @@ -431,12 +509,16 @@ func newLayer2NetConfInfo(netconf *ovncnitypes.NetConf) (NetInfo, error) { if err != nil { return nil, fmt.Errorf("invalid %s netconf %s: %v", netconf.Topology, netconf.Name, err) } - + joinSubnets, err := parseJoinSubnet(netconf.JoinSubnet) + if err != nil { + return nil, err + } ni := &secondaryNetInfo{ netName: netconf.Name, primaryNetwork: netconf.Role == types.NetworkRolePrimary, topology: types.Layer2Topology, subnets: subnets, + joinSubnets: joinSubnets, excludeSubnets: excludes, mtu: netconf.MTU, allowPersistentIPs: netconf.AllowPersistentIPs, @@ -517,6 +599,39 @@ func parseSubnets(subnetsString, excludeSubnetsString, topology string) ([]confi return subnets, excludeIPNets, nil } +func parseJoinSubnet(joinSubnet string) ([]*net.IPNet, error) { + // assign the default values first + // if user provided only 1 family; we still populate the default value + // of the other family from the get-go + _, v4cidr, err := net.ParseCIDR(types.UserDefinedPrimaryNetworkJoinSubnetV4) + if err != nil { + return nil, err + } + _, v6cidr, err := net.ParseCIDR(types.UserDefinedPrimaryNetworkJoinSubnetV6) + if err != nil { + return nil, err + } + joinSubnets := []*net.IPNet{v4cidr, v6cidr} + if strings.TrimSpace(joinSubnet) == "" { + // user has not specified a value; pick the default + return joinSubnets, nil + } + + // user has provided some value; so let's validate and ensure we can use them + joinSubnetCIDREntries, err := config.ParseClusterSubnetEntriesWithDefaults(joinSubnet, 0, 0) + if err != nil { + return nil, err + } + for _, joinSubnetCIDREntry := range joinSubnetCIDREntries { + if knet.IsIPv4CIDR(joinSubnetCIDREntry.CIDR) { + joinSubnets[0] = joinSubnetCIDREntry.CIDR + } else { + joinSubnets[1] = joinSubnetCIDREntry.CIDR + } + } + return joinSubnets, nil +} + func getIPMode(subnets []config.CIDRNetworkEntry) (bool, bool) { var ipv6Mode, ipv4Mode bool for _, subnet := range subnets { @@ -608,6 +723,10 @@ func ParseNetConf(netattachdef *nettypes.NetworkAttachmentDefinition) (*ovncnity return nil, fmt.Errorf("error parsing Network Attachment Definition %s/%s: %w", netattachdef.Namespace, netattachdef.Name, ErrorUnsupportedIPAMKey) } + if netconf.JoinSubnet != "" && netconf.Topology == types.LocalnetTopology { + return nil, fmt.Errorf("localnet topology does not allow specifying join-subnet as services are not supported") + } + return netconf, nil } diff --git a/go-controller/pkg/util/multi_network_test.go b/go-controller/pkg/util/multi_network_test.go index 92f724103f4..39f3246ccd2 100644 --- a/go-controller/pkg/util/multi_network_test.go +++ b/go-controller/pkg/util/multi_network_test.go @@ -345,16 +345,18 @@ func TestParseNetconf(t *testing.T) { "topology": "layer2", "subnets": "192.168.200.0/16", "role": "primary", - "netAttachDefName": "ns1/nad1" + "netAttachDefName": "ns1/nad1", + "joinSubnet": "100.66.0.0/16,fd99::/64" } `, expectedNetConf: &ovncnitypes.NetConf{ - Topology: "layer2", - NADName: "ns1/nad1", - MTU: 1400, - Role: "primary", - Subnets: "192.168.200.0/16", - NetConf: cnitypes.NetConf{Name: "tenant-red", Type: "ovn-k8s-cni-overlay"}, + Topology: "layer2", + NADName: "ns1/nad1", + MTU: 1400, + Role: "primary", + Subnets: "192.168.200.0/16", + NetConf: cnitypes.NetConf{Name: "tenant-red", Type: "ovn-k8s-cni-overlay"}, + JoinSubnet: "100.66.0.0/16,fd99::/64", }, }, { @@ -366,16 +368,18 @@ func TestParseNetconf(t *testing.T) { "topology": "layer3", "subnets": "192.168.200.0/16", "role": "primary", - "netAttachDefName": "ns1/nad1" + "netAttachDefName": "ns1/nad1", + "joinSubnet": "100.66.0.0/16" } `, expectedNetConf: &ovncnitypes.NetConf{ - Topology: "layer3", - NADName: "ns1/nad1", - MTU: 1400, - Role: "primary", - Subnets: "192.168.200.0/16", - NetConf: cnitypes.NetConf{Name: "tenant-red", Type: "ovn-k8s-cni-overlay"}, + Topology: "layer3", + NADName: "ns1/nad1", + MTU: 1400, + Role: "primary", + Subnets: "192.168.200.0/16", + NetConf: cnitypes.NetConf{Name: "tenant-red", Type: "ovn-k8s-cni-overlay"}, + JoinSubnet: "100.66.0.0/16", }, }, { @@ -428,6 +432,20 @@ func TestParseNetconf(t *testing.T) { expectedError: fmt.Errorf("unexpected network field \"role\" primary for \"localnet\" topology, " + "localnet topology does not allow network roles to be set since its always a secondary network"), }, + { + desc: "invalid attachment definition for a localnet topology with joinsubnet provided", + inputNetAttachDefConfigSpec: ` + { + "name": "tenantred", + "type": "ovn-k8s-cni-overlay", + "topology": "localnet", + "subnets": "192.168.200.0/16", + "joinSubnet": "100.66.0.0/16", + "netAttachDefName": "ns1/nad1" + } +`, + expectedError: fmt.Errorf("localnet topology does not allow specifying join-subnet as services are not supported"), + }, } for _, test := range tests { @@ -452,6 +470,119 @@ func TestParseNetconf(t *testing.T) { } } +func TestJoinSubnets(t *testing.T) { + type testConfig struct { + desc string + inputNetConf *ovncnitypes.NetConf + expectedSubnets []*net.IPNet + } + + tests := []testConfig{ + { + desc: "defaultNetInfo with default join subnet", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: ovntypes.DefaultNetworkName}, + Topology: ovntypes.Layer3Topology, + }, + expectedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet(config.Gateway.V4JoinSubnet), + ovntest.MustParseIPNet(config.Gateway.V6JoinSubnet), + }, + }, + { + desc: "secondaryL3NetInfo with default join subnet", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "blue-network"}, + Topology: ovntypes.Layer3Topology, + }, + expectedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV4), + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV6), + }, + }, + { + desc: "secondaryL2NetInfo with default join subnet", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "blue-network"}, + Topology: ovntypes.Layer2Topology, + }, + expectedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV4), + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV6), + }, + }, + { + desc: "secondaryLocalNetInfo with nil join subnet", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "blue-network"}, + Topology: ovntypes.LocalnetTopology, + }, + expectedSubnets: nil, + }, + { + desc: "secondaryL2NetInfo with user configured v4 join subnet", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "blue-network"}, + Topology: ovntypes.Layer2Topology, + JoinSubnet: "100.68.0.0/16", + }, + expectedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("100.68.0.0/16"), + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV6), // given user only provided v4, we set v6 to default value + }, + }, + { + desc: "secondaryL3NetInfo with user configured v6 join subnet", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "blue-network"}, + Topology: ovntypes.Layer3Topology, + JoinSubnet: "2001:db8:abcd:1234::/64", + }, + expectedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV4), + ovntest.MustParseIPNet("2001:db8:abcd:1234::/64"), // given user only provided v4, we set v6 to default value + }, + }, + { + desc: "secondaryL3NetInfo with user configured v4&&v6 join subnet", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "blue-network"}, + Topology: ovntypes.Layer3Topology, + JoinSubnet: "100.68.0.0/16,2001:db8:abcd:1234::/64", + }, + expectedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet("100.68.0.0/16"), + ovntest.MustParseIPNet("2001:db8:abcd:1234::/64"), // given user only provided v4, we set v6 to default value + }, + }, + { + desc: "secondaryL2NetInfo with user configured empty join subnet value takes default value", + inputNetConf: &ovncnitypes.NetConf{ + NetConf: cnitypes.NetConf{Name: "blue-network"}, + Topology: ovntypes.Layer2Topology, + JoinSubnet: "", + }, + expectedSubnets: []*net.IPNet{ + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV4), + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV6), // given user only provided v4, we set v6 to default value + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + g := gomega.NewWithT(t) + netInfo, err := NewNetInfo(test.inputNetConf) + g.Expect(err).To(gomega.BeNil()) + g.Expect(netInfo.JoinSubnets()).To(gomega.Equal(test.expectedSubnets)) + if netInfo.TopologyType() != ovntypes.LocalnetTopology { + g.Expect(netInfo.JoinSubnetV4()).To(gomega.Equal(test.expectedSubnets[0])) + g.Expect(netInfo.JoinSubnetV6()).To(gomega.Equal(test.expectedSubnets[1])) + } + }) + } +} + func TestIsPrimaryNetwork(t *testing.T) { type testConfig struct { desc string diff --git a/go-controller/pkg/util/node_annotations.go b/go-controller/pkg/util/node_annotations.go index fcf65a1a75a..a6262d18981 100644 --- a/go-controller/pkg/util/node_annotations.go +++ b/go-controller/pkg/util/node_annotations.go @@ -13,6 +13,7 @@ import ( kapi "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + utilnet "k8s.io/utils/net" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" @@ -69,8 +70,20 @@ const ( OvnNodeIfAddr = "k8s.ovn.org/node-primary-ifaddr" // ovnNodeGRLRPAddr is the CIDR form representation of Gate Router LRP IP address to join switch (i.e: 100.64.0.5/24) + // DEPRECATED; use ovnNodeGRLRPAddrs moving forward + // FIXME(tssurya): Remove this a few months from now; needed for backwards + // compatbility during upgrades while updating to use the new annotation "ovnNodeGRLRPAddrs" ovnNodeGRLRPAddr = "k8s.ovn.org/node-gateway-router-lrp-ifaddr" + // ovnNodeGRLRPAddrs is the CIDR form representation of Gate Router LRP IP address to join switch (i.e: 100.64.0.4/16) + // for all the networks keyed by the network-name and ipFamily. + // "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": "{ + // \"default\":{\"ipv4\":\"100.64.0.4/16\",\"ipv6\":\"fd98::4/64\"}, + // \"l2-network\":{\"ipv4\":\"100.65.0.4/16\",\"ipv6\":\"fd99::4/64\"}, + // \"l3-network\":{\"ipv4\":\"100.65.0.4/16\",\"ipv6\":\"fd99::4/64\"} + // }", + OVNNodeGRLRPAddrs = "k8s.ovn.org/node-gateway-router-lrp-ifaddrs" + // OvnNodeEgressLabel is a user assigned node label indicating to ovn-kubernetes that the node is to be used for egress IP assignment ovnNodeEgressLabel = "k8s.ovn.org/egress-assignable" @@ -450,14 +463,112 @@ func createPrimaryIfAddrAnnotation(annotationName string, nodeAnnotation map[str return nodeAnnotation, nil } -// CreateNodeGatewayRouterLRPAddrAnnotation sets the IPv4 / IPv6 values of the node's Gateway Router LRP to join switch. -func CreateNodeGatewayRouterLRPAddrAnnotation(nodeAnnotation map[string]interface{}, nodeIPNetv4, - nodeIPNetv6 *net.IPNet) (map[string]interface{}, error) { - return createPrimaryIfAddrAnnotation(ovnNodeGRLRPAddr, nodeAnnotation, nodeIPNetv4, nodeIPNetv6) +func NodeGatewayRouterLRPAddrsAnnotationChanged(oldNode, newNode *corev1.Node) bool { + return oldNode.Annotations[OVNNodeGRLRPAddrs] != newNode.Annotations[OVNNodeGRLRPAddrs] } -func NodeGatewayRouterLRPAddrAnnotationChanged(oldNode, newNode *corev1.Node) bool { - return oldNode.Annotations[ovnNodeGRLRPAddr] != newNode.Annotations[ovnNodeGRLRPAddr] +// UpdateNodeGatewayRouterLRPAddrsAnnotation updates a "k8s.ovn.org/node-gateway-router-lrp-ifaddrs" annotation for network "netName", +// with the specified network, suitable for passing to kube.SetAnnotationsOnNode. If joinSubnets is empty, +// it deletes the "k8s.ovn.org/node-gateway-router-lrp-ifaddrs" annotation for network "netName" +func UpdateNodeGatewayRouterLRPAddrsAnnotation(annotations map[string]string, joinSubnets []*net.IPNet, netName string) (map[string]string, error) { + if annotations == nil { + annotations = map[string]string{} + } + err := updateJoinSubnetAnnotation(annotations, OVNNodeGRLRPAddrs, netName, joinSubnets) + if err != nil { + return nil, err + } + return annotations, nil +} + +// updateJoinSubnetAnnotation add the joinSubnets of the given network to the input node annotations; +// input annotations is not nil +// if joinSubnets is empty, deletes the existing subnet annotation for given network from the input node annotations. +func updateJoinSubnetAnnotation(annotations map[string]string, annotationName, netName string, joinSubnets []*net.IPNet) error { + var bytes []byte + + // First get the all host subnets for all existing networks + subnetsMap, err := parseJoinSubnetAnnotation(annotations, annotationName) + if err != nil { + if !IsAnnotationNotSetError(err) { + return fmt.Errorf("failed to parse join subnet annotation %q: %w", + annotations, err) + } + // in the case that the annotation does not exist + subnetsMap = map[string]primaryIfAddrAnnotation{} + } + + // add or delete host subnet of the specified network + if len(joinSubnets) != 0 { + subnetVal := primaryIfAddrAnnotation{} + for _, net := range joinSubnets { + if utilnet.IsIPv4CIDR(net) { + subnetVal.IPv4 = net.String() + } else { + subnetVal.IPv6 = net.String() + } + } + subnetsMap[netName] = subnetVal + } else { + delete(subnetsMap, netName) + } + + // if no host subnet left, just delete the host subnet annotation from node annotations. + if len(subnetsMap) == 0 { + delete(annotations, annotationName) + return nil + } + + // Marshal all host subnets of all networks back to annotations. + bytes, err = json.Marshal(subnetsMap) + if err != nil { + return err + } + annotations[annotationName] = string(bytes) + return nil +} + +func parseJoinSubnetAnnotation(nodeAnnotations map[string]string, annotationName string) (map[string]primaryIfAddrAnnotation, error) { + annotation, ok := nodeAnnotations[annotationName] + if !ok { + return nil, newAnnotationNotSetError("could not find %q annotation", annotationName) + } + joinSubnetsNetworkMap := make(map[string]primaryIfAddrAnnotation) + if err := json.Unmarshal([]byte(annotation), &joinSubnetsNetworkMap); err != nil { + return nil, fmt.Errorf("failed to unmarshal annotation: %s, err: %w", annotationName, err) + } + + if len(joinSubnetsNetworkMap) == 0 { + return nil, fmt.Errorf("unexpected empty %s annotation", annotationName) + } + + joinsubnetMap := make(map[string]primaryIfAddrAnnotation) + for netName, subnetsStr := range joinSubnetsNetworkMap { + subnetVal := primaryIfAddrAnnotation{} + if subnetsStr.IPv4 == "" && subnetsStr.IPv6 == "" { + return nil, fmt.Errorf("annotation: %s does not have any IP information set", annotationName) + } + if subnetsStr.IPv4 != "" && config.IPv4Mode { + ip, ipNet, err := net.ParseCIDR(subnetsStr.IPv4) + if err != nil { + return nil, fmt.Errorf("failed to parse IPv4 address %s from annotation: %s, err: %w", + subnetsStr.IPv4, annotationName, err) + } + joinIP := &net.IPNet{IP: ip, Mask: ipNet.Mask} + subnetVal.IPv4 = joinIP.String() + } + if subnetsStr.IPv6 != "" && config.IPv6Mode { + ip, ipNet, err := net.ParseCIDR(subnetsStr.IPv6) + if err != nil { + return nil, fmt.Errorf("failed to parse IPv6 address %s from annotation: %s, err: %w", + subnetsStr.IPv4, annotationName, err) + } + joinIP := &net.IPNet{IP: ip, Mask: ipNet.Mask} + subnetVal.IPv6 = joinIP.String() + } + joinsubnetMap[netName] = subnetVal + } + return joinsubnetMap, nil } // CreateNodeTransitSwitchPortAddrAnnotation creates the node annotation for the node's Transit switch port addresses. @@ -537,6 +648,7 @@ func ParseNodePrimaryIfAddr(node *kapi.Node) (*ParsedNodeEgressIPConfiguration, } // ParseNodeGatewayRouterLRPAddr returns the IPv4 / IPv6 values for the node's gateway router +// DEPRECATED; kept for backwards compatibility func ParseNodeGatewayRouterLRPAddr(node *kapi.Node) (net.IP, error) { nodeIfAddrAnnotation, ok := node.Annotations[ovnNodeGRLRPAddr] if !ok { @@ -571,23 +683,30 @@ func parsePrimaryIfAddrAnnotation(node *kapi.Node, annotationName string) ([]*ne if nodeIfAddr.IPv4 == "" && nodeIfAddr.IPv6 == "" { return nil, fmt.Errorf("node: %q does not have any IP information set", node.Name) } + ipAddrs, err := convertPrimaryIfAddrAnnotationToIPNet(nodeIfAddr) + if err != nil { + return nil, fmt.Errorf("failed to parse annotation: %s for node %q, err: %w", annotationName, node.Name, err) + } + return ipAddrs, nil +} + +func convertPrimaryIfAddrAnnotationToIPNet(ifAddr primaryIfAddrAnnotation) ([]*net.IPNet, error) { var ipAddrs []*net.IPNet - if nodeIfAddr.IPv4 != "" { - ip, ipNet, err := net.ParseCIDR(nodeIfAddr.IPv4) + if ifAddr.IPv4 != "" { + ip, ipNet, err := net.ParseCIDR(ifAddr.IPv4) if err != nil { - return nil, fmt.Errorf("failed to parse IPv4 address %s from annotation: %s for node %q, err: %w", nodeIfAddr.IPv4, annotationName, node.Name, err) + return nil, fmt.Errorf("failed to parse IPv4 address %s, err: %w", ifAddr.IPv4, err) } ipAddrs = append(ipAddrs, &net.IPNet{IP: ip, Mask: ipNet.Mask}) } - if nodeIfAddr.IPv6 != "" { - ip, ipNet, err := net.ParseCIDR(nodeIfAddr.IPv6) + if ifAddr.IPv6 != "" { + ip, ipNet, err := net.ParseCIDR(ifAddr.IPv6) if err != nil { - return nil, fmt.Errorf("failed to parse IPv6 address %s from annotation: %s for node %q, err: %w", nodeIfAddr.IPv6, annotationName, node.Name, err) + return nil, fmt.Errorf("failed to parse IPv6 address %s, err: %w", ifAddr.IPv6, err) } ipAddrs = append(ipAddrs, &net.IPNet{IP: ip, Mask: ipNet.Mask}) } - return ipAddrs, nil } @@ -597,6 +716,22 @@ func ParseNodeGatewayRouterLRPAddrs(node *kapi.Node) ([]*net.IPNet, error) { return parsePrimaryIfAddrAnnotation(node, ovnNodeGRLRPAddr) } +// ParseNodeGatewayRouterJoinAddrs returns the IPv4 and/or IPv6 addresses for the node's gateway router port +// stored in the 'OVNNodeGRLRPAddrs' annotation +func ParseNodeGatewayRouterJoinAddrs(node *kapi.Node, netName string) ([]*net.IPNet, error) { + joinSubnetMap, err := parseJoinSubnetAnnotation(node.Annotations, OVNNodeGRLRPAddrs) + if err != nil { + return nil, fmt.Errorf("unable to parse annotation %s on node %s: err %w", + OVNNodeGRLRPAddrs, node.Name, err) + } + val, ok := joinSubnetMap[netName] + if !ok { + return nil, fmt.Errorf("unable to fetch annotation value on node %s for network %s", + node.Name, netName) + } + return convertPrimaryIfAddrAnnotationToIPNet(val) +} + // ParseNodeTransitSwitchPortAddrs returns the IPv4 and/or IPv6 addresses for the node's transit switch port // stored in the 'ovnTransitSwitchPortAddr' annotation func ParseNodeTransitSwitchPortAddrs(node *kapi.Node) ([]*net.IPNet, error) { diff --git a/go-controller/pkg/util/node_annotations_unit_test.go b/go-controller/pkg/util/node_annotations_unit_test.go index ae90b42d1d9..4c9aa708ef4 100644 --- a/go-controller/pkg/util/node_annotations_unit_test.go +++ b/go-controller/pkg/util/node_annotations_unit_test.go @@ -12,6 +12,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube" annotatorMock "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube/mocks" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -514,6 +515,126 @@ func TestParseNodeGatewayRouterLRPAddr(t *testing.T) { } } +func TestParseNodeGatewayRouterJoinAddrs(t *testing.T) { + tests := []struct { + desc string + inpNode v1.Node + netName string + errExpected bool + expOutput bool + }{ + { + desc: "Gateway router LPR IP address annotation not found for node, however, does not return error", + inpNode: v1.Node{}, + expOutput: false, + }, + { + desc: "success: Gateway router parse LPR IP address", + inpNode: v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/node-gateway-router-lrp-ifaddrs": `{"default":{"ipv4":"100.64.0.4/16"}}`}, + }, + }, + netName: types.DefaultNetworkName, + expOutput: true, + }, + { + desc: "success: Gateway router parse LPR IP address dual stack", + inpNode: v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/node-gateway-router-lrp-ifaddrs": `{"default":{"ipv4":"100.64.0.5/16","ipv6":"fd:98::/64"}}`}, + }, + }, + netName: types.DefaultNetworkName, + expOutput: true, + }, + { + desc: "success: Gateway router parse LPR IP address dual stack for the right network name", + inpNode: v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/node-gateway-router-lrp-ifaddrs": `{"default":{"ipv4":"100.64.0.5/16","ipv6":"fd:98::/64"},"l3-network":{"ipv4":"100.65.0.5/16","ipv6":"fd:99::/64"}}`}, + }, + }, + netName: "l3-network", + expOutput: true, + }, + { + desc: "error: Gateway router parse LPR IP address dual stack cannot find the requested network name", + inpNode: v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/node-gateway-router-lrp-ifaddrs": `{"default":{"ipv4":"100.64.0.5/16","ipv6":"fd:98::/64"},"l3-network":{"ipv4":"100.65.0.5/16","ipv6":"fd:99::/64"}}`}, + }, + }, + netName: "l2-network", + errExpected: true, + }, + { + desc: "error: Gateway router parse LPR IP address error", + inpNode: v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{"k8s.ovn.org/node-gateway-router-lrp-ifaddrs": `{"default":{"ipv4":"100.64.0.5"}}`}, + }, + }, + netName: types.DefaultNetworkName, + errExpected: true, + }, + } + config.IPv4Mode = true + config.IPv6Mode = true + for i, tc := range tests { + t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) { + cfg, e := ParseNodeGatewayRouterJoinAddrs(&tc.inpNode, tc.netName) + if tc.errExpected { + t.Log(e) + assert.Error(t, e) + assert.Nil(t, cfg) + } + if tc.expOutput { + assert.NotNil(t, cfg) + } + }) + } +} + +func TestCreateNodeGatewayRouterLRPAddrsAnnotation(t *testing.T) { + tests := []struct { + desc string + inpDefSubnetIps []*net.IPNet + outExp map[string]string + errExp bool + }{ + { + desc: "success path, valid default subnets", + inpDefSubnetIps: ovntest.MustParseIPNets("192.168.1.12/24"), + outExp: map[string]string{ + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": "{\"default\":{\"ipv4\":\"192.168.1.12/24\"}}", + }, + }, + { + desc: "success path, valid default dualstack subnets", + inpDefSubnetIps: ovntest.MustParseIPNets("192.168.1.12/24", "fd:98::5/64"), + outExp: map[string]string{ + "k8s.ovn.org/node-gateway-router-lrp-ifaddrs": "{\"default\":{\"ipv4\":\"192.168.1.12/24\",\"ipv6\":\"fd:98::5/64\"}}", + }, + }, + { + desc: "success path, inpDefSubnetIps is nil", + outExp: map[string]string{}, + }, + } + for i, tc := range tests { + t.Run(fmt.Sprintf("%d:%s", i, tc.desc), func(t *testing.T) { + res, err := UpdateNodeGatewayRouterLRPAddrsAnnotation(nil, tc.inpDefSubnetIps, types.DefaultNetworkName) + t.Log(res, err) + if tc.errExp { + assert.NotNil(t, err) + } else { + assert.True(t, reflect.DeepEqual(res, tc.outExp)) + } + }) + } +} + func TestSetGatewayMTUSupport(t *testing.T) { mockAnnotator := new(annotatorMock.Annotator) diff --git a/go-controller/pkg/util/pod_annotation.go b/go-controller/pkg/util/pod_annotation.go index fd84e029034..6d44a58fa33 100644 --- a/go-controller/pkg/util/pod_annotation.go +++ b/go-controller/pkg/util/pod_annotation.go @@ -443,19 +443,13 @@ func IsValidPodAnnotation(podAnnotation *PodAnnotation) bool { return podAnnotation != nil && len(podAnnotation.MAC) > 0 } -func joinSubnetToRoute(isIPv6 bool, gatewayIP net.IP) PodRoute { - joinSubnet := config.Gateway.V4JoinSubnet +func joinSubnetToRoute(netinfo NetInfo, isIPv6 bool, gatewayIP net.IP) PodRoute { + joinSubnet := netinfo.JoinSubnetV4() if isIPv6 { - joinSubnet = config.Gateway.V6JoinSubnet + joinSubnet = netinfo.JoinSubnetV6() } - _, subnet, err := net.ParseCIDR(joinSubnet) - if err != nil { - // Join subnet should have been validated already by config - panic(fmt.Sprintf("Failed to parse join subnet %q: %v", joinSubnet, err)) - } - return PodRoute{ - Dest: subnet, + Dest: joinSubnet, NextHop: gatewayIP, } } @@ -506,6 +500,8 @@ func AddRoutesGatewayIP( gatewayIPnet := GetNodeGatewayIfAddr(nodeSubnet) // Ensure default service network traffic always goes to OVN podAnnotation.Routes = append(podAnnotation.Routes, serviceCIDRToRoute(isIPv6, gatewayIPnet.IP)...) + // Ensure UDN join subnet traffic always goes to UDN LSP + podAnnotation.Routes = append(podAnnotation.Routes, joinSubnetToRoute(netinfo, isIPv6, gatewayIPnet.IP)) if network != nil && len(network.GatewayRequest) == 0 { // if specific default route for pod was not requested then add gatewayIP podAnnotation.Gateways = append(podAnnotation.Gateways, gatewayIPnet.IP) } @@ -532,6 +528,8 @@ func AddRoutesGatewayIP( } // Ensure default service network traffic always goes to OVN podAnnotation.Routes = append(podAnnotation.Routes, serviceCIDRToRoute(isIPv6, gatewayIPnet.IP)...) + // Ensure UDN join subnet traffic always goes to UDN LSP + podAnnotation.Routes = append(podAnnotation.Routes, joinSubnetToRoute(netinfo, isIPv6, gatewayIPnet.IP)) if network != nil && len(network.GatewayRequest) == 0 { // if specific default route for pod was not requested then add gatewayIP podAnnotation.Gateways = append(podAnnotation.Gateways, gatewayIPnet.IP) } @@ -594,7 +592,7 @@ func AddRoutesGatewayIP( } // Ensure default join subnet traffic always goes to OVN - podAnnotation.Routes = append(podAnnotation.Routes, joinSubnetToRoute(isIPv6, gatewayIPnet.IP)) + podAnnotation.Routes = append(podAnnotation.Routes, joinSubnetToRoute(netinfo, isIPv6, gatewayIPnet.IP)) } return nil diff --git a/go-controller/pkg/util/util_unit_test.go b/go-controller/pkg/util/util_unit_test.go index d0aab23711d..1bc19e33531 100644 --- a/go-controller/pkg/util/util_unit_test.go +++ b/go-controller/pkg/util/util_unit_test.go @@ -13,6 +13,7 @@ import ( "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" + ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" @@ -309,6 +310,10 @@ func TestGetActiveNetworkForNamespace(t *testing.T) { CIDR: ovntest.MustParseIPNet("100.128.0.0/16"), HostSubnetLength: 24, }}, + joinSubnets: []*net.IPNet{ + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV4), + ovntest.MustParseIPNet(ovntypes.UserDefinedPrimaryNetworkJoinSubnetV6), + }, }, }, {