add ovs internal-port for pod network interface

pkg/controller/config.go

@@ -45,6 +45,7 @@ type Configuration struct {
 
 	PodName      string
 	PodNamespace string
+	PodNicType   string
 
 	WorkerNum int
 	PprofPort int
@@ -90,6 +91,7 @@ func ParseFlags() (*Configuration, error) {
 		argsDefaultVlanName      = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name, default: ovn-vlan")
 		argsDefaultVlanID        = pflag.Int("default-vlan-id", 1, "The default vlan id, default: 1")
 		argsDefaultVlanRange     = pflag.String("default-vlan-range", "1,4095", "The default vlan range, default: 1-4095")
+		argsPodNicType           = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type, default: veth-pair")
 	)
 
 	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
@@ -137,6 +139,7 @@ func ParseFlags() (*Configuration, error) {
 		DefaultVlanRange:              *argsDefaultVlanRange,
 		PodName:                       os.Getenv("POD_NAME"),
 		PodNamespace:                  os.Getenv("KUBE_NAMESPACE"),
+		PodNicType:                    *argsPodNicType,
 	}
 
 	if util.IsNetworkVlan(config.NetworkType) && config.DefaultHostInterface == "" {

pkg/controller/pod.go

@@ -407,6 +407,7 @@ func (c *Controller) handleAddPod(key string) error {
 		pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.Gateway
 		pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)] = subnet.Name
 		pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true"
+		pod.Annotations[util.PodNicAnnotation] = c.config.PodNicType
 
 		if err := util.ValidatePodCidr(podNet.Subnet.Spec.CIDRBlock, ipStr); err != nil {
 			klog.Errorf("validate pod %s/%s failed, %v", namespace, name, err)

pkg/daemon/handler.go

@@ -3,11 +3,12 @@ package daemon
 import (
 	"context"
 	"fmt"
-	"k8s.io/apimachinery/pkg/labels"
 	"net/http"
 	"strings"
 	"time"
 
+	"k8s.io/apimachinery/pkg/labels"
+
 	"github.com/emicklei/go-restful"
 	"github.com/kubeovn/kube-ovn/pkg/ovs"
 	v1 "k8s.io/api/core/v1"
@@ -68,7 +69,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 	}
 
 	klog.Infof("add port request %v", podRequest)
-	var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, vlanID, ifName string
+	var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, vlanID, ifName, nicType string
 	var pod *v1.Pod
 	var err error
 	for i := 0; i < 15; i++ {
@@ -106,6 +107,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 		vlanID = pod.Annotations[fmt.Sprintf(util.VlanIdAnnotationTemplate, podRequest.Provider)]
 		ipAddr = util.GetIpAddrWithMask(ip, cidr)
 		ifName = podRequest.IfName
+		nicType = pod.Annotations[util.PodNicAnnotation]
 		break
 	}
 
@@ -130,7 +132,11 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 
 	if strings.HasSuffix(podRequest.Provider, util.OvnProvider) && subnet != "" {
 		klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s", ifName, macAddr, ipAddr, cidr, gw)
-		err := csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, ipAddr, gw, ingress, egress, vlanID, podRequest.DeviceID)
+		if nicType == util.VethType {
+			err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, ipAddr, gw, ingress, egress, vlanID, podRequest.DeviceID, nicType)
+		} else {
+			err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, ipAddr, gw, ingress, egress, vlanID, podRequest.DeviceID, nicType)
+		}
 		if err != nil {
 			errMsg := fmt.Errorf("configure nic failed %v", err)
 			klog.Error(errMsg)
@@ -257,7 +263,8 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon
 			}
 		}
 
-		err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.DeviceID, podRequest.IfName)
+		nicType := pod.Annotations[util.PodNicAnnotation]
+		err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.DeviceID, podRequest.IfName, nicType)
 		if err != nil {
 			errMsg := fmt.Errorf("del nic failed %v", err)
 			klog.Error(errMsg)

pkg/daemon/ovs.go

@@ -18,7 +18,7 @@ import (
 	"k8s.io/klog"
 )
 
-func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, ifName, mac, ip, gateway, ingress, egress, vlanID, DeviceID string) error {
+func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, ifName, mac, ip, gateway, ingress, egress, vlanID, DeviceID, nicType string) error {
 	var err error
 	var hostNicName, containerNicName string
 	if DeviceID == "" {
@@ -64,16 +64,23 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns,
 	if err != nil {
 		return fmt.Errorf("failed to open netns %q: %v", netns, err)
 	}
-	if err = configureContainerNic(containerNicName, ifName, ip, gateway, macAddr, podNS, csh.Config.MTU); err != nil {
+	if err = configureContainerNic(containerNicName, ifName, ip, gateway, macAddr, podNS, csh.Config.MTU, nicType); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, deviceID, ifName string) error {
-	hostNicName, _ := generateNicName(containerID, ifName)
+func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, deviceID, ifName, nicType string) error {
+	var nicName string
+	hostNicName, containerNicName := generateNicName(containerID, ifName)
+	if nicType == util.VethType {
+		nicName = hostNicName
+	} else {
+		nicName = containerNicName
+	}
+
 	// Remove ovs port
-	output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", hostNicName)
+	output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", nicName)
 	if err != nil {
 		return fmt.Errorf("failed to delete ovs port %v, %q", err, output)
 	}
@@ -83,13 +90,13 @@ func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, device
 	}
 
 	if deviceID == "" {
-		hostLink, err := netlink.LinkByName(hostNicName)
+		hostLink, err := netlink.LinkByName(nicName)
 		if err != nil {
 			// If link already not exists, return quietly
 			if _, ok := err.(netlink.LinkNotFoundError); ok {
 				return nil
 			}
-			return fmt.Errorf("find host link %s failed %v", hostNicName, err)
+			return fmt.Errorf("find host link %s failed %v", nicName, err)
 		}
 		if err = netlink.LinkDel(hostLink); err != nil {
 			return fmt.Errorf("delete host link %s failed %v", hostLink, err)
@@ -129,7 +136,7 @@ func configureHostNic(nicName, vlanID string) error {
 	return nil
 }
 
-func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAddr net.HardwareAddr, netns ns.NetNS, mtu int) error {
+func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string) error {
 	containerLink, err := netlink.LinkByName(nicName)
 	if err != nil {
 		return fmt.Errorf("can not find container nic %s %v", nicName, err)
@@ -140,9 +147,12 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAd
 	}
 
 	return ns.WithNetNSPath(netns.Path(), func(_ ns.NetNS) error {
-		if err = netlink.LinkSetName(containerLink, ifName); err != nil {
-			return err
+		if nicType == util.VethType {
+			if err = netlink.LinkSetName(containerLink, ifName); err != nil {
+				return err
+			}
 		}
+
 		if util.CheckProtocol(ipAddr) == kubeovnv1.ProtocolDual || util.CheckProtocol(ipAddr) == kubeovnv1.ProtocolIPv6 {
 			// For docker version >=17.x the "none" network will disable ipv6 by default.
 			// We have to enable ipv6 here to add v6 address and gateway.
@@ -158,8 +168,20 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAd
 			}
 		}
 
-		if err = configureNic(ifName, ipAddr, macAddr, mtu); err != nil {
-			return err
+		if nicType == util.VethType {
+			if err = configureNic(ifName, ipAddr, macAddr, mtu); err != nil {
+				return err
+			}
+		} else {
+			if err = configureNic(nicName, ipAddr, macAddr, mtu); err != nil {
+				return err
+			}
+			if err = addStaticEth0Nic(); err != nil {
+				return err
+			}
+			if err = configureAdditonalNic("eth0", ipAddr); err != nil {
+				return err
+			}
 		}
 
 		if ifName != "eth0" {
@@ -595,3 +617,109 @@ func renameLink(curName, newName string) error {
 
 	return nil
 }
+
+func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac, ip, gateway, ingress, egress, vlanID, DeviceID, nicType string) error {
+	var err error
+
+	_, containerNicName := generateNicName(containerID, ifName)
+	ipStr := util.GetIpWithoutMask(ip)
+	ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)
+	ovs.CleanDuplicatePort(ifaceID)
+
+	// Add container iface to ovs port as internal port
+	output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", containerNicName, "--",
+		"set", "interface", containerNicName, "type=internal", "--",
+		"set", "interface", containerNicName, fmt.Sprintf("external_ids:iface-id=%s", ifaceID),
+		fmt.Sprintf("external_ids:pod_name=%s", podName),
+		fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace),
+		fmt.Sprintf("external_ids:ip=%s", ipStr))
+	if err != nil {
+		return fmt.Errorf("add nic to ovs failed %v: %q", err, output)
+	}
+
+	// container nic must use same mac address from pod annotation, otherwise ovn will reject these packets by default
+	macAddr, err := net.ParseMAC(mac)
+	if err != nil {
+		return fmt.Errorf("failed to parse mac %s %v", macAddr, err)
+	}
+
+	if err = ovs.SetInterfaceBandwidth(fmt.Sprintf("%s.%s", podName, podNamespace), ingress, egress); err != nil {
+		return err
+	}
+
+	podNS, err := ns.GetNS(netns)
+	if err != nil {
+		return fmt.Errorf("failed to open netns %q: %v", netns, err)
+	}
+	if err = configureContainerNic(containerNicName, ifName, ip, gateway, macAddr, podNS, csh.Config.MTU, nicType); err != nil {
+		return err
+	}
+	return nil
+}
+
+// https://github.com/antrea-io/antrea/issues/1691
+func configureAdditonalNic(link, ip string) error {
+	nodeLink, err := netlink.LinkByName(link)
+	if err != nil {
+		return fmt.Errorf("can not find nic %s %v", link, err)
+	}
+
+	ipDelMap := make(map[string]netlink.Addr)
+	ipAddMap := make(map[string]netlink.Addr)
+	ipAddrs, err := netlink.AddrList(nodeLink, 0x0)
+	if err != nil {
+		return fmt.Errorf("can not get addr %s %v", nodeLink, err)
+	}
+	for _, ipAddr := range ipAddrs {
+		if strings.HasPrefix(ipAddr.IP.String(), "fe80::") {
+			continue
+		}
+		ipDelMap[ipAddr.IP.String()+"/"+ipAddr.Mask.String()] = ipAddr
+	}
+
+	for _, ipStr := range strings.Split(ip, ",") {
+		// Do not reassign same address for link
+		if _, ok := ipDelMap[ipStr]; ok {
+			delete(ipDelMap, ipStr)
+			continue
+		}
+
+		ipAddr, err := netlink.ParseAddr(ipStr)
+		if err != nil {
+			return fmt.Errorf("can not parse %s %v", ipStr, err)
+		}
+		ipAddMap[ipStr] = *ipAddr
+	}
+
+	for _, addr := range ipDelMap {
+		ipDel := addr
+		if err = netlink.AddrDel(nodeLink, &ipDel); err != nil {
+			return fmt.Errorf("delete address %s %v", addr, err)
+		}
+	}
+	for _, addr := range ipAddMap {
+		ipAdd := addr
+		if err = netlink.AddrAdd(nodeLink, &ipAdd); err != nil {
+			return fmt.Errorf("can not add address %v to nic %s, %v", addr, link, err)
+		}
+	}
+
+	return nil
+}
+
+func addStaticEth0Nic() error {
+	dummy := &netlink.Dummy{
+		LinkAttrs: netlink.LinkAttrs{
+			Name: "eth0",
+		},
+	}
+
+	if err := netlink.LinkAdd(dummy); err != nil {
+		if err := netlink.LinkDel(dummy); err != nil {
+			klog.Errorf("failed to delete eth0 %v", err)
+			return err
+		}
+		return fmt.Errorf("failed to crate static eth0 for %v", err)
+	}
+	return nil
+}

pkg/util/const.go

@@ -102,6 +102,10 @@ const (
 	EcmpRouteType   = "ecmp"
 	NormalRouteType = "normal"
 
+	PodNicAnnotation = "ovn.kubernetes.io/pod_nic_type"
+	VethType         = "veth-pair"
+	InternalType     = "internal-port"
+
 	ChassisLoc     = "/etc/openvswitch/system-id.conf"
 	HostnameEnv    = "KUBE_NODE_NAME"
 	ChasRetryTime  = 5