Use Antrea IPAM for Antrea native secondary network support (#5427)

This commit changes Antrea managed secondary networks to use Antrea
IPAM for interface IP address allocation, and no longer use
Whereabouts.

Signed-off-by: Jianjun Shen <shenj@vmware.com>

hack/update-codegen-dockerized.sh

@@ -52,8 +52,7 @@ MOCKGEN_TARGETS=(
   "pkg/agent/querier AgentQuerier testing"
   "pkg/agent/route Interface testing"
   "pkg/agent/ipassigner IPAssigner testing"
-  "pkg/agent/secondarynetwork/podwatch InterfaceConfigurator testing"
-  "pkg/agent/secondarynetwork/ipam IPAMDelegator testing"
+  "pkg/agent/secondarynetwork/podwatch InterfaceConfigurator,IPAMAllocator testing"
   "pkg/agent/servicecidr Interface testing"
   "pkg/agent/util/ipset Interface testing"
   "pkg/agent/util/iptables Interface testing mock_iptables_linux.go" # Must specify linux.go suffix, otherwise compilation would fail on windows platform as source file has linux build tag.

pkg/agent/cniserver/interface_configuration_linux.go

@@ -274,7 +274,7 @@ func (ic *ifConfigurator) configureContainerLinkVeth(
 			}
 		}
 
-		klog.V(2).Infof("Configuring IP address for container %s", containerID)
+		klog.V(2).InfoS("Configuring IP address for container interface", "result", *result)
 		// result.Interfaces must be set before this.
 		if err := ipamConfigureIface(containerIface.Name, result); err != nil {
 			return fmt.Errorf("failed to configure IP address for container %s: %v", containerID, err)

pkg/agent/cniserver/ipam/antrea_ipam.go

@@ -58,8 +58,8 @@ const (
 
 // Resource needs to be unique since it is used as identifier in Del.
 // Therefore Container ID is used, while Pod/Namespace are shown for visibility.
-func getAllocationOwner(args *invoke.Args, k8sArgs *types.K8sArgs, reservedOwner *crdv1a2.IPAddressOwner, secondary bool) crdv1a2.IPAddressOwner {
-	podOwner := &crdv1a2.PodOwner{
+func getAllocationPodOwner(args *invoke.Args, k8sArgs *types.K8sArgs, reservedOwner *crdv1a2.IPAddressOwner, secondary bool) *crdv1a2.PodOwner {
+	podOwner := crdv1a2.PodOwner{
 		Name:        string(k8sArgs.K8S_POD_NAME),
 		Namespace:   string(k8sArgs.K8S_POD_NAMESPACE),
 		ContainerID: args.ContainerID,
@@ -69,14 +69,17 @@ func getAllocationOwner(args *invoke.Args, k8sArgs *types.K8sArgs, reservedOwner
 		// the secondary network interface.
 		podOwner.IFName = args.IfName
 	}
+	return &podOwner
+}
+
+func getAllocationOwner(args *invoke.Args, k8sArgs *types.K8sArgs, reservedOwner *crdv1a2.IPAddressOwner, secondary bool) *crdv1a2.IPAddressOwner {
+	podOwner := getAllocationPodOwner(args, k8sArgs, nil, secondary)
 	if reservedOwner != nil {
 		owner := *reservedOwner
 		owner.Pod = podOwner
-		return owner
-	}
-	return crdv1a2.IPAddressOwner{
-		Pod: podOwner,
+		return &owner
 	}
+	return &crdv1a2.IPAddressOwner{Pod: podOwner}
 }
 
 // Helper to generate IP config and default route, taking IP version into account
@@ -131,8 +134,8 @@ func (d *AntreaIPAM) setController(controller *AntreaIPAMController) {
 	d.controller = controller
 }
 
-// Add allocates next available IP address from associated IP Pool
-// Allocated IP and associated resource are stored in IP Pool status
+// Add allocates the next available IP address from the associated IP Pool. The
+// allocated IP and associated resource will be stored in the IP Pool status.
 func (d *AntreaIPAM) Add(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfig []byte) (bool, *IPAMResult, error) {
 	mine, allocator, ips, reservedOwner, err := d.owns(k8sArgs)
 	if err != nil {
@@ -143,7 +146,7 @@ func (d *AntreaIPAM) Add(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfi
 		return false, nil, nil
 	}
 
-	owner := getAllocationOwner(args, k8sArgs, reservedOwner, false)
+	owner := *getAllocationOwner(args, k8sArgs, reservedOwner, false)
 	var ip net.IP
 	var subnetInfo *crdv1a2.SubnetInfo
 	if reservedOwner != nil {
@@ -170,10 +173,10 @@ func (d *AntreaIPAM) Add(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfi
 	return true, &result, nil
 }
 
-// Del deletes IP associated with resource from IP Pool status
+// Del releases the IP associated with the resource from the IP Pool status.
 func (d *AntreaIPAM) Del(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfig []byte) (bool, error) {
-	owner := getAllocationOwner(args, k8sArgs, nil, false)
-	foundAllocation, err := d.del(owner.Pod)
+	podOwner := getAllocationPodOwner(args, k8sArgs, nil, false)
+	foundAllocation, err := d.del(podOwner)
 	if err != nil {
 		// Let the invoker retry at error.
 		return true, err
@@ -183,7 +186,7 @@ func (d *AntreaIPAM) Del(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfi
 	return foundAllocation, nil
 }
 
-// Check verifues IP associated with resource is tracked in IP Pool status
+// Check verifies the IP associated with the resource is tracked in the IP Pool status.
 func (d *AntreaIPAM) Check(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfig []byte) (bool, error) {
 	mine, allocator, _, _, err := d.owns(k8sArgs)
 	if err != nil {
@@ -205,7 +208,11 @@ func (d *AntreaIPAM) Check(args *invoke.Args, k8sArgs *types.K8sArgs, networkCon
 	return true, nil
 }
 
-func (d *AntreaIPAM) secondaryNetworkAdd(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfig *types.NetworkConfig) (*current.Result, error) {
+// SecondaryNetworkAllocate allocates IP addresses for a Pod secondary network interface, based on
+// the IPAM configuration of the passed CNI network configuration.
+// It supports IPAM for both Antrea-managed secondary networks and Multus-managed secondary
+// networks.
+func (d *AntreaIPAM) SecondaryNetworkAllocate(podOwner *crdv1a2.PodOwner, networkConfig *types.NetworkConfig) (*IPAMResult, error) {
 	ipamConf := networkConfig.IPAM
 	numPools := len(ipamConf.IPPools)
 
@@ -216,19 +223,18 @@ func (d *AntreaIPAM) secondaryNetworkAdd(args *invoke.Args, k8sArgs *types.K8sAr
 		return nil, fmt.Errorf("at least one Antrea IPPool or static address must be specified")
 	}
 
-	result := &current.Result{}
+	result := IPAMResult{}
 	if numPools > 0 {
 		if err := d.waitForControllerReady(); err != nil {
 			// Return error to let the invoker retry.
 			return nil, err
 		}
 
-		owner := getAllocationOwner(args, k8sArgs, nil, true)
 		var allocatorsToRelease []*poolallocator.IPPoolAllocator
 		defer func() {
 			for _, allocator := range allocatorsToRelease {
 				// Try to release the allocated IPs after an error.
-				allocator.ReleaseContainer(owner.Pod.ContainerID, owner.Pod.IFName)
+				allocator.ReleaseContainer(podOwner.ContainerID, podOwner.IFName)
 			}
 		}()
 
@@ -240,6 +246,7 @@ func (d *AntreaIPAM) secondaryNetworkAdd(args *invoke.Args, k8sArgs *types.K8sAr
 
 			var ip net.IP
 			var subnetInfo *crdv1a2.SubnetInfo
+			owner := crdv1a2.IPAddressOwner{Pod: podOwner}
 			ip, subnetInfo, err = allocator.AllocateNext(crdv1a2.IPAddressPhaseAllocated, owner)
 			if err != nil {
 				return nil, err
@@ -254,6 +261,10 @@ func (d *AntreaIPAM) secondaryNetworkAdd(args *invoke.Args, k8sArgs *types.K8sAr
 			// assume the CNI version >= 0.3.0, and so do not check the number of
 			// addresses.
 			result.IPs = append(result.IPs, ipConfig)
+			if result.VLANID == 0 {
+				// Return the first non-zero VLAN.
+				result.VLANID = subnetInfo.VLAN
+			}
 		}
 		// No failed allocation, so do not release allocated IPs.
 		allocatorsToRelease = nil
@@ -269,15 +280,23 @@ func (d *AntreaIPAM) secondaryNetworkAdd(args *invoke.Args, k8sArgs *types.K8sAr
 	// Copy routes and DNS from the input IPAM configuration.
 	result.Routes = ipamConf.Routes
 	result.DNS = ipamConf.DNS
-	return result, nil
+	return &result, nil
 }
 
-func (d *AntreaIPAM) secondaryNetworkDel(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfig *types.NetworkConfig) error {
-	owner := getAllocationOwner(args, k8sArgs, nil, true)
-	_, err := d.del(owner.Pod)
+// SecondaryNetworkRelease releases the IP addresses allocated for a Pod secondary network interface.
+func (d *AntreaIPAM) SecondaryNetworkRelease(owner *crdv1a2.PodOwner) error {
+	_, err := d.del(owner)
 	return err
 }
 
+func (d *AntreaIPAM) secondaryNetworkAdd(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfig *types.NetworkConfig) (*IPAMResult, error) {
+	return d.SecondaryNetworkAllocate(getAllocationPodOwner(args, k8sArgs, nil, true), networkConfig)
+}
+
+func (d *AntreaIPAM) secondaryNetworkDel(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfig *types.NetworkConfig) error {
+	return d.SecondaryNetworkRelease(getAllocationPodOwner(args, k8sArgs, nil, true))
+}
+
 func (d *AntreaIPAM) secondaryNetworkCheck(args *invoke.Args, k8sArgs *types.K8sArgs, networkConfig *types.NetworkConfig) error {
 	return fmt.Errorf("CNI CHECK is not implemented for secondary network")
 }
@@ -309,27 +328,24 @@ func (d *AntreaIPAM) del(podOwner *crdv1a2.PodOwner) (foundAllocation bool, err
 	return true, nil
 }
 
-// owns checks whether this driver owns coming IPAM request. This decision is based on
-// Antrea IPAM annotation for the resource (only Namespace annotation is supported as
-// of today). If annotation is not present, or annotated IP Pool not found, the driver
-// will not own the request and fall back to next IPAM driver.
-// return types:
+// owns checks whether this driver owns the coming IPAM request. This decision is based on Antrea
+// IPAM annotation for the resource (Pod or Namespace). If an annotation is not present, or the
+// annotated IP Pool not found, the driver should not own the request and will fall back to the next
+// IPAM driver.
+// return:
 // mineUnknown + PodNotFound error
 // mineUnknown + InvalidIPAnnotation error
 // mineFalse + nil error
 // mineTrue + timeout error
 // mineTrue + IPPoolNotFound error
 // mineTrue + nil error
 func (d *AntreaIPAM) owns(k8sArgs *types.K8sArgs) (mineType, *poolallocator.IPPoolAllocator, []net.IP, *crdv1a2.IPAddressOwner, error) {
-	// Wait controller ready to avoid inappropriate behavior on CNI request
+	// Wait controller ready to avoid inappropriate behaviors on the CNI request.
 	if err := d.waitForControllerReady(); err != nil {
-		// Return mineTrue to make this request failed and kubelet will retry.
+		// Return mineTrue to make this request fail and kubelet will retry.
 		return mineTrue, nil, nil, nil, err
 	}
 
-	// As of today, only Namespace annotation is supported
-	// In future, Deployment, Statefulset and Pod annotations will be
-	// supported as well
 	namespace := string(k8sArgs.K8S_POD_NAMESPACE)
 	podName := string(k8sArgs.K8S_POD_NAME)
 	klog.V(2).InfoS("Inspecting IPAM annotation", "Namespace", namespace, "Pod", podName)

pkg/agent/cniserver/ipam/antrea_ipam_test.go

@@ -541,8 +541,6 @@ func TestSecondaryNetworkAdd(t *testing.T) {
 	testCases := []struct {
 		name        string
 		networkConf *argtypes.NetworkConfig
-		args        *invoke.Args
-		k8sArgs     *argtypes.K8sArgs
 		initFunc    func(stopCh chan struct{}) *AntreaIPAM
 		expectedRes error
 	}{
@@ -589,14 +587,6 @@ func TestSecondaryNetworkAdd(t *testing.T) {
 					},
 				},
 			},
-			k8sArgs: &argtypes.K8sArgs{
-				CommonArgs:        cnitypes.CommonArgs{},
-				K8S_POD_NAME:      "test-pod",
-				K8S_POD_NAMESPACE: "test-ns",
-			},
-			args: &invoke.Args{
-				ContainerID: "container-id",
-			},
 			expectedRes: nil,
 			initFunc: func(stopCh chan struct{}) *AntreaIPAM {
 				k8sClient, crdClient := initTestClients()
@@ -646,7 +636,16 @@ func TestSecondaryNetworkAdd(t *testing.T) {
 				defer close(stopCh)
 			}
 
-			_, err := d.secondaryNetworkAdd(tt.args, tt.k8sArgs, tt.networkConf)
+			args := &invoke.Args{
+				ContainerID: "container-id",
+			}
+			k8sArgs := &argtypes.K8sArgs{
+				CommonArgs:        cnitypes.CommonArgs{},
+				K8S_POD_NAME:      "test-pod",
+				K8S_POD_NAMESPACE: "test-ns",
+			}
+
+			_, err := d.secondaryNetworkAdd(args, k8sArgs, tt.networkConf)
 			assert.Equal(t, tt.expectedRes, err)
 		})
 	}

pkg/agent/cniserver/ipam/hostlocal/gc.go

@@ -42,9 +42,12 @@ func GarbageCollectContainerIPs(network string, desiredIPs sets.Set[string]) err
 	dir := networkDir(network)
 
 	info, err := os.Stat(dir)
-	if os.IsNotExist(err) {
-		klog.V(2).InfoS("Host-local IPAM data directory does not exist, nothing to do", "dir", dir)
-		return nil
+	if err != nil {
+		if os.IsNotExist(err) {
+			klog.V(2).InfoS("Host-local IPAM data directory does not exist, nothing to do", "dir", dir)
+			return nil
+		}
+		return err
 	}
 	if !info.IsDir() {
 		return fmt.Errorf("path '%s' is not a directory: %w", dir, err)

pkg/agent/cniserver/ipam/ipam_service.go

@@ -25,12 +25,11 @@ import (
 	cnipb "antrea.io/antrea/pkg/apis/cni/v1beta1"
 )
 
-// List of ordered IPAM drivers
+// Ordered list of IPAM drivers.
 // The first driver in list that claims to own this request will proceed to
-// handle allocation/release
-// This model is useful for antrea IPAM feature that should trigger antrea
-// IPAM only if corresponding annotation is specified for Pod/Namespace
-// Otherwise IPAM should be handled by host-local plugin.
+// handle allocation/release. This model is useful for antrea IPAM feature that
+// should trigger Antrea IPAM only if a corresponding annotation is specified
+// for Pod/Namespace; otherwise IPAM should be handled by the host-local plugin.
 var ipamDrivers map[string][]IPAMDriver
 
 // A cache of IPAM results.
@@ -151,7 +150,11 @@ func IsIPAMTypeValid(ipamType string) bool {
 // Antrea IPAM for secondary network.
 func SecondaryNetworkAdd(cniArgs *cnipb.CniCmdArgs, k8sArgs *types.K8sArgs, networkConfig *types.NetworkConfig) (*current.Result, error) {
 	args := argsFromEnv(cniArgs)
-	return getAntreaIPAMDriver().secondaryNetworkAdd(args, k8sArgs, networkConfig)
+	ipamResult, err := getAntreaIPAMDriver().secondaryNetworkAdd(args, k8sArgs, networkConfig)
+	if err != nil {
+		return nil, err
+	}
+	return &ipamResult.Result, nil
 
 }
 
@@ -183,3 +186,9 @@ func ResetIPAMDriver(ipamType string, driver IPAMDriver) {
 func AddIPAMResult(key string, result *IPAMResult) {
 	ipamResults.Store(key, result)
 }
+
+// GetSecondaryNetworkAllocator returns the Antrea IPAM driver as the
+// SecondaryNetworkIPAMAllocator implementation.
+func GetSecondaryNetworkAllocator() *AntreaIPAM {
+	return getAntreaIPAMDriver()
+}

pkg/agent/cniserver/pod_configuration.go

@@ -260,7 +260,7 @@ func (pc *podConfigurator) configureInterfaces(
 	// be dropped by OVS.
 	if err = pc.ifConfigurator.advertiseContainerAddr(containerNetNS, containerIface.Name, &result.Result); err != nil {
 		// Do not return an error and fail the interface creation.
-		klog.ErrorS(err, "Failed to advertise IP address for container", "container ID", containerID)
+		klog.ErrorS(err, "Failed to advertise IP address for container", "container", containerID)
 	}
 	// Mark the manipulation as success to cancel deferred operations.
 	success = true
@@ -479,7 +479,7 @@ func (pc *podConfigurator) reconcile(pods []corev1.Pod, containerAccess *contain
 
 	// clean-up IPs that may still be allocated
 	klog.V(4).InfoS("Running IPAM garbage collection for unused Pod IPs")
-	if err := ipam.GarbageCollectContainerIPs(antreaCNIType, desiredPodIPs); err != nil {
+	if err := ipam.GarbageCollectContainerIPs(AntreaCNIType, desiredPodIPs); err != nil {
 		klog.ErrorS(err, "Error when garbage collecting previously-allocated IPs")
 	}
 

pkg/agent/cniserver/secondary.go

@@ -47,7 +47,7 @@ func (pc *podConfigurator) ConfigureSriovSecondaryInterface(
 	klog.InfoS("Configured SR-IOV interface", "Pod", klog.KRef(podNamespace, podName), "interface", containerInterfaceName, "hostInterface", hostIface)
 
 	if err = pc.ifConfigurator.advertiseContainerAddr(containerNetNS, containerIface.Name, result); err != nil {
-		klog.ErrorS(err, "Failed to advertise IP address for SR-IOV interface", "container ID", containerID, "interface", containerInterfaceName)
+		klog.ErrorS(err, "Failed to advertise IP address for SR-IOV interface", "container", containerID, "interface", containerInterfaceName)
 	}
 	return nil
 }
@@ -71,7 +71,7 @@ func (pc *podConfigurator) ConfigureVLANSecondaryInterface(
 	defer func() {
 		if !success {
 			if err := pc.ifConfigurator.removeContainerLink(containerID, hostIface.Name); err != nil {
-				klog.ErrorS(err, "failed to roll back veth creation", "container ID", containerID, "interface", containerInterfaceName)
+				klog.ErrorS(err, "failed to roll back veth creation", "container", containerID, "interface", containerInterfaceName)
 			}
 		}
 	}()
@@ -85,7 +85,7 @@ func (pc *podConfigurator) ConfigureVLANSecondaryInterface(
 	klog.InfoS("Configured VLAN interface", "Pod", klog.KRef(podNamespace, podName), "interface", containerInterfaceName, "hostInterface", hostIface)
 
 	if err := pc.ifConfigurator.advertiseContainerAddr(containerNetNS, containerIface.Name, result); err != nil {
-		klog.ErrorS(err, "Failed to advertise IP address for VLAN interface", "container ID", containerID, "interface", containerInterfaceName)
+		klog.ErrorS(err, "Failed to advertise IP address for VLAN interface", "container", containerID, "interface", containerInterfaceName)
 	}
 	success = true
 	return ovsPortUUID, nil