kubevirt, routing: Use point to point routing

Live migrated pods will keep their IP and it will be from a subnet
different from the node switch, to continue routing to the proper node a
point to point route need to be added to ovn_cluster_router that will
change at live migration. This change add that route and implement the
live migration switch.

Signed-off-by: Enrique Llorente <ellorent@redhat.com>

go-controller/pkg/kubevirt/types.go

@@ -4,6 +4,8 @@ const (
 	AllowPodBridgeNetworkLiveMigrationAnnotation = "kubevirt.io/allow-pod-bridge-network-live-migration"
 	SubnetSwitchNameAnnotation                   = "kubevirt.io/subnet-switch-name"
 	VMLabel                                      = "kubevirt.io/vm"
+	MigrationTargetStartTimestampAnnotation      = "kubevirt.io/migration-target-start-timestamp"
+	NodeNameLabel                                = "kubevirt.io/nodeName"
 )
 
 type IPConfig struct {

go-controller/pkg/ovn/kubevirt.go

@@ -8,10 +8,12 @@ import (
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kubevirt"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
 	corev1 "k8s.io/api/core/v1"
 	kapi "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilnet "k8s.io/utils/net"
 )
 
 // getSwitchNames at some kubevirt scenarios the switch owner the IP is
@@ -84,6 +86,22 @@ func (oc *DefaultNetworkController) deleteDHCPOptions(pod *kapi.Pod) error {
 	return libovsdbops.DeleteDHCPOptionsWithPredicate(oc.nbClient, predicate)
 }
 
+func (oc *DefaultNetworkController) deletePodEnrouting(pod *kapi.Pod) error {
+	routePredicate := func(item *nbdb.LogicalRouterStaticRoute) bool {
+		return kubevirt.PodMatchesExternalIDs(pod, item.ExternalIDs)
+	}
+	if err := libovsdbops.DeleteLogicalRouterStaticRoutesWithPredicate(oc.nbClient, types.OVNClusterRouter, routePredicate); err != nil {
+		return err
+	}
+	policyPredicate := func(item *nbdb.LogicalRouterPolicy) bool {
+		return kubevirt.PodMatchesExternalIDs(pod, item.ExternalIDs)
+	}
+	if err := libovsdbops.DeleteLogicalRouterPoliciesWithPredicate(oc.nbClient, types.OVNClusterRouter, policyPredicate); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (oc *DefaultNetworkController) kubevirtCleanUp(pod *corev1.Pod) error {
 	if kubevirt.AllowPodBridgeNetworkLiveMigration(pod.Annotations) {
 		isLiveMigrationLefover, err := kubevirt.PodIsLiveMigrationLeftOver(oc.client, pod)
@@ -95,7 +113,114 @@ func (oc *DefaultNetworkController) kubevirtCleanUp(pod *corev1.Pod) error {
 			if err := oc.deleteDHCPOptions(pod); err != nil {
 				return err
 			}
+			if err := oc.deletePodEnrouting(pod); err != nil {
+				return err
+			}
 		}
 	}
 	return nil
 }
+
+func (oc *DefaultNetworkController) enroutePodAddressesToNode(pod *kapi.Pod) error {
+	podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, "default")
+	if err != nil {
+		return fmt.Errorf("failed reading ovn annotation: %v", err)
+	}
+
+	nodeGwAddressIPv4, nodeGwAddressIPv6, err := oc.lrpAddresses(types.GWRouterToJoinSwitchPrefix + types.GWRouterPrefix + pod.Spec.NodeName)
+	if err != nil {
+		return fmt.Errorf("failed reading LRP addresses: %v", err)
+	}
+	for _, podIP := range podAnnotation.IPs {
+		// Add a reroute policy to route VM n/s traffic to the node where the VM
+		// is running
+		ipVersion := "4"
+		nexthop := nodeGwAddressIPv4
+		if utilnet.IsIPv6CIDR(podIP) {
+			ipVersion = "6"
+			nexthop = nodeGwAddressIPv6
+		}
+		podAddress := podIP.IP.String()
+		match := fmt.Sprintf("ip%s.src == %s", ipVersion, podAddress)
+		egressPolicy := nbdb.LogicalRouterPolicy{
+			Match:    match,
+			Action:   nbdb.LogicalRouterPolicyActionReroute,
+			Nexthops: []string{nexthop},
+			Priority: 1,
+			ExternalIDs: map[string]string{
+				"namespace":      pod.Namespace,
+				kubevirt.VMLabel: pod.Labels[kubevirt.VMLabel],
+			},
+		}
+		if err := libovsdbops.CreateOrUpdateLogicalRouterPolicyWithPredicate(oc.nbClient, types.OVNClusterRouter, &egressPolicy, func(item *nbdb.LogicalRouterPolicy) bool {
+			return item.Priority == egressPolicy.Priority && item.Match == egressPolicy.Match && item.Action == egressPolicy.Action
+		}); err != nil {
+			return fmt.Errorf("failed adding point to point policy: %v", err)
+		}
+
+		// Add a policy to force send an ARP to discover VMs MAC and send
+		// directly to it since there is no more routers in the middle
+		outputPort := types.RouterToSwitchPrefix + pod.Spec.NodeName
+		ingressRoute := nbdb.LogicalRouterStaticRoute{
+			IPPrefix:   podAddress,
+			Nexthop:    podAddress,
+			Policy:     &nbdb.LogicalRouterStaticRoutePolicyDstIP,
+			OutputPort: &outputPort,
+			ExternalIDs: map[string]string{
+				"namespace":      pod.Namespace,
+				kubevirt.VMLabel: pod.Labels[kubevirt.VMLabel],
+			},
+		}
+		if err := libovsdbops.CreateOrReplaceLogicalRouterStaticRouteWithPredicate(oc.nbClient, types.OVNClusterRouter, &ingressRoute, func(item *nbdb.LogicalRouterStaticRoute) bool {
+			matches := item.IPPrefix == ingressRoute.IPPrefix && item.Nexthop == ingressRoute.Nexthop && item.Policy != nil && *item.Policy == *ingressRoute.Policy
+			return matches
+		}); err != nil {
+			return fmt.Errorf("failed adding static route: %v", err)
+		}
+	}
+	return nil
+}
+
+func (oc *DefaultNetworkController) enrouteVirtualMachine(pod *kapi.Pod) error {
+	targetNode := pod.Labels[kubevirt.NodeNameLabel]
+	targetStartTimestamp := pod.Annotations[kubevirt.MigrationTargetStartTimestampAnnotation]
+	// No live migration or target node was reached || qemu is already ready
+	if targetNode == pod.Spec.NodeName || targetStartTimestamp != "" {
+		if err := oc.enroutePodAddressesToNode(pod); err != nil {
+			return fmt.Errorf("failed enroutePodAddressesToNode for %s/%s: %w", pod.Namespace, pod.Name, err)
+		}
+	}
+	return nil
+}
+
+func (oc *DefaultNetworkController) lrpAddresses(lrpName string) (string, string, error) {
+	lrp := &nbdb.LogicalRouterPort{
+		Name: lrpName,
+	}
+
+	lrp, err := libovsdbops.GetLogicalRouterPort(oc.nbClient, lrp)
+	if err != nil {
+		return "", "", err
+	}
+	if len(lrp.Networks) == 0 {
+		return "", "", fmt.Errorf("missing LRP network")
+	}
+	var ipv4, ipv6 string
+	for _, network := range lrp.Networks {
+		lrpIP, _, err := net.ParseCIDR(network)
+		if err != nil {
+			return "", "", err
+		}
+		ip := lrpIP.String()
+		if ip == "" {
+			return "", "", fmt.Errorf("missing logical router port address")
+		}
+		if utilnet.IsIPv6(lrpIP) {
+			ipv6 = ip
+		} else {
+			ipv4 = ip
+		}
+
+	}
+	return ipv4, ipv6, nil
+}