From 1b00190f3025fa66ef0ba1c3a4b8145eeb9a98d8 Mon Sep 17 00:00:00 2001 From: hzma Date: Fri, 21 May 2021 18:11:13 +0800 Subject: [PATCH] add ovs internal-port for pod network interface --- pkg/controller/config.go | 3 + pkg/controller/pod.go | 1 + pkg/daemon/handler.go | 15 ++-- pkg/daemon/ovs.go | 152 +++++++++++++++++++++++++++++++++++---- pkg/util/const.go | 4 ++ 5 files changed, 159 insertions(+), 16 deletions(-) diff --git a/pkg/controller/config.go b/pkg/controller/config.go index 59bf194c5fc..a3edf8d3952 100644 --- a/pkg/controller/config.go +++ b/pkg/controller/config.go @@ -45,6 +45,7 @@ type Configuration struct { PodName string PodNamespace string + PodNicType string WorkerNum int PprofPort int @@ -90,6 +91,7 @@ func ParseFlags() (*Configuration, error) { argsDefaultVlanName = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name, default: ovn-vlan") argsDefaultVlanID = pflag.Int("default-vlan-id", 1, "The default vlan id, default: 1") argsDefaultVlanRange = pflag.String("default-vlan-range", "1,4095", "The default vlan range, default: 1-4095") + argsPodNicType = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type, default: veth-pair") ) klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) @@ -137,6 +139,7 @@ func ParseFlags() (*Configuration, error) { DefaultVlanRange: *argsDefaultVlanRange, PodName: os.Getenv("POD_NAME"), PodNamespace: os.Getenv("KUBE_NAMESPACE"), + PodNicType: *argsPodNicType, } if util.IsNetworkVlan(config.NetworkType) && config.DefaultHostInterface == "" { diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index bbd85a5e468..d067d557c11 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -407,6 +407,7 @@ func (c *Controller) handleAddPod(key string) error { pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.Gateway pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podNet.ProviderName)] = subnet.Name pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)] = "true" + pod.Annotations[util.PodNicAnnotation] = c.config.PodNicType if err := util.ValidatePodCidr(podNet.Subnet.Spec.CIDRBlock, ipStr); err != nil { klog.Errorf("validate pod %s/%s failed, %v", namespace, name, err) diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go index 35bf7b52eac..d29c9cfaf35 100644 --- a/pkg/daemon/handler.go +++ b/pkg/daemon/handler.go @@ -3,11 +3,12 @@ package daemon import ( "context" "fmt" - "k8s.io/apimachinery/pkg/labels" "net/http" "strings" "time" + "k8s.io/apimachinery/pkg/labels" + "github.com/emicklei/go-restful" "github.com/kubeovn/kube-ovn/pkg/ovs" v1 "k8s.io/api/core/v1" @@ -68,7 +69,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon } klog.Infof("add port request %v", podRequest) - var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, vlanID, ifName string + var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, vlanID, ifName, nicType string var pod *v1.Pod var err error for i := 0; i < 15; i++ { @@ -106,6 +107,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon vlanID = pod.Annotations[fmt.Sprintf(util.VlanIdAnnotationTemplate, podRequest.Provider)] ipAddr = util.GetIpAddrWithMask(ip, cidr) ifName = podRequest.IfName + nicType = pod.Annotations[util.PodNicAnnotation] break } @@ -130,7 +132,11 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon if strings.HasSuffix(podRequest.Provider, util.OvnProvider) && subnet != "" { klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s", ifName, macAddr, ipAddr, cidr, gw) - err := csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, ipAddr, gw, ingress, egress, vlanID, podRequest.DeviceID) + if nicType == util.VethType { + err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, ipAddr, gw, ingress, egress, vlanID, podRequest.DeviceID, nicType) + } else { + err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, ipAddr, gw, ingress, egress, vlanID, podRequest.DeviceID, nicType) + } if err != nil { errMsg := fmt.Errorf("configure nic failed %v", err) klog.Error(errMsg) @@ -257,7 +263,8 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon } } - err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.DeviceID, podRequest.IfName) + nicType := pod.Annotations[util.PodNicAnnotation] + err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.DeviceID, podRequest.IfName, nicType) if err != nil { errMsg := fmt.Errorf("del nic failed %v", err) klog.Error(errMsg) diff --git a/pkg/daemon/ovs.go b/pkg/daemon/ovs.go index e090d93551c..3fe3a3744e9 100644 --- a/pkg/daemon/ovs.go +++ b/pkg/daemon/ovs.go @@ -18,7 +18,7 @@ import ( "k8s.io/klog" ) -func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, ifName, mac, ip, gateway, ingress, egress, vlanID, DeviceID string) error { +func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, ifName, mac, ip, gateway, ingress, egress, vlanID, DeviceID, nicType string) error { var err error var hostNicName, containerNicName string if DeviceID == "" { @@ -64,16 +64,23 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, if err != nil { return fmt.Errorf("failed to open netns %q: %v", netns, err) } - if err = configureContainerNic(containerNicName, ifName, ip, gateway, macAddr, podNS, csh.Config.MTU); err != nil { + if err = configureContainerNic(containerNicName, ifName, ip, gateway, macAddr, podNS, csh.Config.MTU, nicType); err != nil { return err } return nil } -func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, deviceID, ifName string) error { - hostNicName, _ := generateNicName(containerID, ifName) +func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, deviceID, ifName, nicType string) error { + var nicName string + hostNicName, containerNicName := generateNicName(containerID, ifName) + if nicType == util.VethType { + nicName = hostNicName + } else { + nicName = containerNicName + } + // Remove ovs port - output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", hostNicName) + output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", nicName) if err != nil { return fmt.Errorf("failed to delete ovs port %v, %q", err, output) } @@ -83,13 +90,13 @@ func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, device } if deviceID == "" { - hostLink, err := netlink.LinkByName(hostNicName) + hostLink, err := netlink.LinkByName(nicName) if err != nil { // If link already not exists, return quietly if _, ok := err.(netlink.LinkNotFoundError); ok { return nil } - return fmt.Errorf("find host link %s failed %v", hostNicName, err) + return fmt.Errorf("find host link %s failed %v", nicName, err) } if err = netlink.LinkDel(hostLink); err != nil { return fmt.Errorf("delete host link %s failed %v", hostLink, err) @@ -129,7 +136,7 @@ func configureHostNic(nicName, vlanID string) error { return nil } -func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAddr net.HardwareAddr, netns ns.NetNS, mtu int) error { +func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string) error { containerLink, err := netlink.LinkByName(nicName) if err != nil { return fmt.Errorf("can not find container nic %s %v", nicName, err) @@ -140,9 +147,12 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAd } return ns.WithNetNSPath(netns.Path(), func(_ ns.NetNS) error { - if err = netlink.LinkSetName(containerLink, ifName); err != nil { - return err + if nicType == util.VethType { + if err = netlink.LinkSetName(containerLink, ifName); err != nil { + return err + } } + if util.CheckProtocol(ipAddr) == kubeovnv1.ProtocolDual || util.CheckProtocol(ipAddr) == kubeovnv1.ProtocolIPv6 { // For docker version >=17.x the "none" network will disable ipv6 by default. // We have to enable ipv6 here to add v6 address and gateway. @@ -158,8 +168,20 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAd } } - if err = configureNic(ifName, ipAddr, macAddr, mtu); err != nil { - return err + if nicType == util.VethType { + if err = configureNic(ifName, ipAddr, macAddr, mtu); err != nil { + return err + } + } else { + if err = configureNic(nicName, ipAddr, macAddr, mtu); err != nil { + return err + } + if err = addStaticEth0Nic(); err != nil { + return err + } + if err = configureAdditonalNic("eth0", ipAddr); err != nil { + return err + } } if ifName != "eth0" { @@ -595,3 +617,109 @@ func renameLink(curName, newName string) error { return nil } + +func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac, ip, gateway, ingress, egress, vlanID, DeviceID, nicType string) error { + var err error + + _, containerNicName := generateNicName(containerID, ifName) + ipStr := util.GetIpWithoutMask(ip) + ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider) + ovs.CleanDuplicatePort(ifaceID) + + // Add container iface to ovs port as internal port + output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", containerNicName, "--", + "set", "interface", containerNicName, "type=internal", "--", + "set", "interface", containerNicName, fmt.Sprintf("external_ids:iface-id=%s", ifaceID), + fmt.Sprintf("external_ids:pod_name=%s", podName), + fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace), + fmt.Sprintf("external_ids:ip=%s", ipStr)) + if err != nil { + return fmt.Errorf("add nic to ovs failed %v: %q", err, output) + } + + // container nic must use same mac address from pod annotation, otherwise ovn will reject these packets by default + macAddr, err := net.ParseMAC(mac) + if err != nil { + return fmt.Errorf("failed to parse mac %s %v", macAddr, err) + } + + if err = ovs.SetInterfaceBandwidth(fmt.Sprintf("%s.%s", podName, podNamespace), ingress, egress); err != nil { + return err + } + + podNS, err := ns.GetNS(netns) + if err != nil { + return fmt.Errorf("failed to open netns %q: %v", netns, err) + } + if err = configureContainerNic(containerNicName, ifName, ip, gateway, macAddr, podNS, csh.Config.MTU, nicType); err != nil { + return err + } + return nil +} + +// https://github.com/antrea-io/antrea/issues/1691 +func configureAdditonalNic(link, ip string) error { + nodeLink, err := netlink.LinkByName(link) + if err != nil { + return fmt.Errorf("can not find nic %s %v", link, err) + } + + ipDelMap := make(map[string]netlink.Addr) + ipAddMap := make(map[string]netlink.Addr) + ipAddrs, err := netlink.AddrList(nodeLink, 0x0) + if err != nil { + return fmt.Errorf("can not get addr %s %v", nodeLink, err) + } + for _, ipAddr := range ipAddrs { + if strings.HasPrefix(ipAddr.IP.String(), "fe80::") { + continue + } + ipDelMap[ipAddr.IP.String()+"/"+ipAddr.Mask.String()] = ipAddr + } + + for _, ipStr := range strings.Split(ip, ",") { + // Do not reassign same address for link + if _, ok := ipDelMap[ipStr]; ok { + delete(ipDelMap, ipStr) + continue + } + + ipAddr, err := netlink.ParseAddr(ipStr) + if err != nil { + return fmt.Errorf("can not parse %s %v", ipStr, err) + } + ipAddMap[ipStr] = *ipAddr + } + + for _, addr := range ipDelMap { + ipDel := addr + if err = netlink.AddrDel(nodeLink, &ipDel); err != nil { + return fmt.Errorf("delete address %s %v", addr, err) + } + } + for _, addr := range ipAddMap { + ipAdd := addr + if err = netlink.AddrAdd(nodeLink, &ipAdd); err != nil { + return fmt.Errorf("can not add address %v to nic %s, %v", addr, link, err) + } + } + + return nil +} + +func addStaticEth0Nic() error { + dummy := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{ + Name: "eth0", + }, + } + + if err := netlink.LinkAdd(dummy); err != nil { + if err := netlink.LinkDel(dummy); err != nil { + klog.Errorf("failed to delete eth0 %v", err) + return err + } + return fmt.Errorf("failed to crate static eth0 for %v", err) + } + return nil +} diff --git a/pkg/util/const.go b/pkg/util/const.go index 22e4ad8e08e..dd73fe9c137 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -102,6 +102,10 @@ const ( EcmpRouteType = "ecmp" NormalRouteType = "normal" + PodNicAnnotation = "ovn.kubernetes.io/pod_nic_type" + VethType = "veth-pair" + InternalType = "internal-port" + ChassisLoc = "/etc/openvswitch/system-id.conf" HostnameEnv = "KUBE_NODE_NAME" ChasRetryTime = 5