Run ipv6 lane in control plane jobs

This commit fixes things here and there
in e2e's to make control planes run on
v6 as well. Note that some tests are
skipped since they cannot be run on
github runners with v6 enabled.

NOTE: One of the multicast test is moved
from e2e.go to multicast.go where it belongs.

Signed-off-by: Surya Seetharaman <suryaseetharaman.9@gmail.com>

.github/workflows/test.yml

@@ -391,10 +391,12 @@ jobs:
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "local",  "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}
-          - {"target": "control-plane",     "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-disabled"}
+          - {"target": "control-plane",     "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-disabled"}
           - {"target": "control-plane",     "ha": "HA",   "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "snatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
           - {"target": "control-plane",     "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv4",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
+          - {"target": "control-plane",     "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv6",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
           - {"target": "control-plane",     "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "ic": "ic-single-node-zones"}
+          - {"target": "control-plane",     "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "ic": "ic-single-node-zones"}
           - {"target": "multi-homing",      "ha": "noHA", "gateway-mode": "local",  "ipfamily": "ipv4",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
           - {"target": "node-ip-mac-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic": "ic-disabled"}
           - {"target": "node-ip-mac-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4",      "disable-snat-multiple-gws": "SnatGW",   "second-bridge": "1br", "ic": "ic-single-node-zones"}

test/e2e/acl_logging.go

@@ -174,15 +174,7 @@ var _ = Describe("ACL Logging for EgressFirewall", func() {
 		allowACLVerdict          = "allow"
 		namespacePrefix          = "acl-log-egressfw"
 		secondaryNamespacePrefix = "acl-log-egressfw-sec"
-
-		// These targets must be off cluster - traffic to the cluster should always be
-		// allowed: https://docs.openshift.com/container-platform/4.10/networking/openshift_sdn/configuring-egress-firewall.html
-		// "As a cluster administrator, you can create an egress firewall for a project that restricts egress traffic leaving
-		// your OpenShift Container Platform cluster."
-		// Because the egress firewall feature only affects traffic leaving the cluster, we will not log for on-cluster targets.
-		allowedDstIP = "172.18.0.1"
-		deniedDstIP  = "172.19.0.10"
-		dstPort      = 8080
+		dstPort                  = 8080
 	)
 
 	fr := wrappedTestFramework(namespacePrefix)
@@ -192,15 +184,32 @@ var _ = Describe("ACL Logging for EgressFirewall", func() {
 		nsNameSecondary  string
 		pokePod          *v1.Pod
 		pokePodSecondary *v1.Pod
+		allowedDstIP     string
+		deniedDstIP      string
 	)
 
 	BeforeEach(func() {
+		// These targets must be off cluster - traffic to the cluster should always be
+		// allowed: https://docs.openshift.com/container-platform/4.10/networking/openshift_sdn/configuring-egress-firewall.html
+		// "As a cluster administrator, you can create an egress firewall for a project that restricts egress traffic leaving
+		// your OpenShift Container Platform cluster."
+		// Because the egress firewall feature only affects traffic leaving the cluster, we will not log for on-cluster targets.
+		allowedDstIP = "172.18.0.1"
+		deniedDstIP  = "172.19.0.10"
+		mask         := "32"
+		denyCIDR     := "0.0.0.0/0"
+		if IsIPv6Cluster(fr.ClientSet) {
+			allowedDstIP = "2001:4860:4860::8888"
+			deniedDstIP  = "2001:4860:4860::8844"
+			mask         = "128"
+			denyCIDR     = "::/0"
+		}
 		By("configuring the ACL logging level within the namespace")
 		nsName = fr.Namespace.Name
 		Expect(setNamespaceACLLogSeverity(fr, nsName, initialDenyACLSeverity, initialAllowACLSeverity, aclRemoveOptionDelete)).To(Succeed())
 
 		By("creating a \"default deny\" Egress Firewall")
-		err := makeEgressFirewall(nsName)
+		err := makeEgressFirewall(nsName, allowedDstIP, mask, denyCIDR)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("creating a pod running agnhost netexec")
@@ -221,7 +230,7 @@ var _ = Describe("ACL Logging for EgressFirewall", func() {
 		Expect(setNamespaceACLLogSeverity(fr, nsNameSecondary, initialDenyACLSeverity, initialAllowACLSeverity, aclRemoveOptionDelete)).To(Succeed())
 
 		By("creating a \"default deny\" Egress Firewall inside the secondary namespace")
-		err = makeEgressFirewall(nsNameSecondary)
+		err = makeEgressFirewall(nsNameSecondary, allowedDstIP, mask, denyCIDR)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("creating a pod running agnhost netexec inside the secondary namespace")
@@ -621,7 +630,7 @@ func makeDenyAllPolicy(f *framework.Framework, ns string, policyName string) (*k
 	return f.ClientSet.NetworkingV1().NetworkPolicies(ns).Create(context.TODO(), policy, metav1.CreateOptions{})
 }
 
-func makeEgressFirewall(ns string) error {
+func makeEgressFirewall(ns, allowedDstIP, mask, denyCIDR string) error {
 	egressFirewallYaml := "egressfirewall.yaml"
 	var egressFirewallConfig = fmt.Sprintf(`apiVersion: k8s.ovn.org/v1
 kind: EgressFirewall
@@ -632,11 +641,11 @@ spec:
   egress:
   - type: Allow
     to:
-      cidrSelector: 172.18.0.1/32
+      cidrSelector: %s/%s
   - type: Deny
     to:
-      cidrSelector: 0.0.0.0/0
-`)
+      cidrSelector: %s
+`, allowedDstIP, mask, denyCIDR)
 
 	if err := os.WriteFile(egressFirewallYaml, []byte(egressFirewallConfig), 0644); err != nil {
 		framework.Failf("Unable to write CRD config to disk: %v", err)

test/e2e/egressip.go

@@ -337,14 +337,14 @@ var _ = ginkgo.Describe("e2e egress IP validation", func() {
 		waitForStatus(node, setReady)
 	}
 
-	setNodeReachable := func(node string, setReachable bool) {
+	setNodeReachable := func(iptablesCmd, node string, setReachable bool) {
 		if !setReachable {
-			_, err := runCommand("docker", "exec", node, "iptables", "-I", "INPUT", "-p", "tcp", "--dport", "9107", "-j", "DROP")
+			_, err := runCommand("docker", "exec", node, iptablesCmd, "-I", "INPUT", "-p", "tcp", "--dport", "9107", "-j", "DROP")
 			if err != nil {
 				framework.Failf("failed to block port 9107 on node: %s, err: %v", node, err)
 			}
 		} else {
-			_, err := runCommand("docker", "exec", node, "iptables", "-I", "INPUT", "-p", "tcp", "--dport", "9107", "-j", "ACCEPT")
+			_, err := runCommand("docker", "exec", node, iptablesCmd, "-I", "INPUT", "-p", "tcp", "--dport", "9107", "-j", "ACCEPT")
 			if err != nil {
 				framework.Failf("failed to allow port 9107 on node: %s, err: %v", node, err)
 			}
@@ -423,7 +423,10 @@ var _ = ginkgo.Describe("e2e egress IP validation", func() {
 		// ensure all nodes are ready and reachable
 		for _, node := range nodes.Items {
 			setNodeReady(node.Name, true)
-			setNodeReachable(node.Name, true)
+			setNodeReachable("iptables", node.Name, true)
+			if IsIPv6Cluster(f.ClientSet) {
+				setNodeReachable("ip6tables", node.Name, true)
+			}
 		}
 	})
 
@@ -438,7 +441,10 @@ var _ = ginkgo.Describe("e2e egress IP validation", func() {
 		// ensure all nodes are ready and reachable
 		for _, node := range []string{egress1Node.name, egress2Node.name} {
 			setNodeReady(node, true)
-			setNodeReachable(node, true)
+			setNodeReachable("iptables", node, true)
+			if IsIPv6Cluster(f.ClientSet) {
+				setNodeReachable("ip6tables", node, true)
+			}
 		}
 	})
 
@@ -1117,6 +1123,9 @@ spec:
 			framework.Failf("Error: Check the OVN DB to ensure no SNATs are added for the standby egressIP, err: %v", err)
 		}
 		logicalIP := fmt.Sprintf("logical_ip=%s", srcPodIP.String())
+		if IsIPv6Cluster(f.ClientSet) {
+			logicalIP = fmt.Sprintf("logical_ip=\"%s\"", srcPodIP.String())
+		}
 		snats, err := e2ekubectl.RunKubectl("ovn-kubernetes", "exec", dbPod, "-c", dbContainerName, "--", "ovn-nbctl", "--no-leader-only", "--columns=external_ip", "find", "nat", logicalIP)
 		if err != nil {
 			framework.Failf("Error: Check the OVN DB to ensure no SNATs are added for the standby egressIP, err: %v", err)
@@ -1352,7 +1361,10 @@ spec:
 		createGenericPodWithLabel(f, pod1Name, pod1Node.name, f.Namespace.Name, command, podEgressLabel)
 
 		ginkgo.By(fmt.Sprintf("4. Make egress node: %s unreachable", node1))
-		setNodeReachable(node1, false)
+		setNodeReachable("iptables", node1, false)
+		if IsIPv6Cluster(f.ClientSet) {
+			setNodeReachable("ip6tables", node1, false)
+		}
 
 		otherNode := egress1Node.name
 		if node1 == egress1Node.name {
@@ -1374,7 +1386,10 @@ spec:
 		framework.ExpectNoError(err, "7. Check connectivity from pod to the api-server (running hostNetwork:true) and verifying that the connection is achieved, failed, err: %v", err)
 
 		ginkgo.By("8, Make node 2 unreachable")
-		setNodeReachable(node2, false)
+		setNodeReachable("iptables", node2, false)
+		if IsIPv6Cluster(f.ClientSet) {
+			setNodeReachable("ip6tables", node2, false)
+		}
 
 		ginkgo.By("9. Check that egress IP is un-assigned (empty status)")
 		verifyEgressIPStatusLengthEquals(0, nil)
@@ -1384,7 +1399,10 @@ spec:
 		framework.ExpectNoError(err, "10. Check connectivity from pod to an external \"node\" and verify that the IP is the node IP, failed, err: %v", err)
 
 		ginkgo.By("11. Make node 1 reachable again")
-		setNodeReachable(node1, true)
+		setNodeReachable("iptables", node1, true)
+		if IsIPv6Cluster(f.ClientSet) {
+			setNodeReachable("ip6tables", node1, true)
+		}
 
 		ginkgo.By("12. Check that egress IP is assigned to node 1 again")
 		statuses = verifyEgressIPStatusLengthEquals(1, func(statuses []egressIPStatus) bool {
@@ -1397,7 +1415,10 @@ spec:
 		framework.ExpectNoError(err, "13. Check connectivity from pod to an external \"node\" and verify that the IP is the egress IP, failed, err: %v", err)
 
 		ginkgo.By("14. Make node 2 reachable again")
-		setNodeReachable(node2, true)
+		setNodeReachable("iptables", node2, true)
+		if IsIPv6Cluster(f.ClientSet) {
+			setNodeReachable("ip6tables", node2, true)
+		}
 
 		ginkgo.By("15. Check that egress IP remains assigned to node 1. We should not be moving the egress IP to node 2 if the node 1 works fine, as to reduce cluster entropy - read: changes.")
 		statuses = verifyEgressIPStatusLengthEquals(1, func(statuses []egressIPStatus) bool {
@@ -1419,7 +1440,10 @@ spec:
 		framework.ExpectNoError(err, "19. Check connectivity from pod to an external \"node\" and verify that the IP is the egress IP, failed, err: %v", err)
 
 		ginkgo.By("20. Make node 1 not reachable")
-		setNodeReachable(node1, false)
+		setNodeReachable("iptables", node1, false)
+		if IsIPv6Cluster(f.ClientSet) {
+			setNodeReachable("ip6tables", node1, false)
+		}
 
 		ginkgo.By("21. Unlabel node 2")
 		e2enode.RemoveLabelOffNode(f.ClientSet, node2, "k8s.ovn.org/egress-assignable")
@@ -1434,7 +1458,10 @@ spec:
 		verifyEgressIPStatusLengthEquals(0, nil)
 
 		ginkgo.By("25. Make node 1 reachable again")
-		setNodeReachable(node1, true)
+		setNodeReachable("iptables", node1, true)
+		if IsIPv6Cluster(f.ClientSet) {
+			setNodeReachable("ip6tables", node1, true)
+		}
 
 		ginkgo.By("26. Check that egress IP is assigned to node 1 again")
 		statuses = verifyEgressIPStatusLengthEquals(1, func(statuses []egressIPStatus) bool {

test/e2e/multicast.go

@@ -3,12 +3,14 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
@@ -130,3 +132,83 @@ var _ = ginkgo.Describe("Multicast", func() {
 	})
 
 })
+
+var _ = ginkgo.Describe("e2e IGMP validation", func() {
+	const (
+		svcname              string = "igmp-test"
+		ovnNs                string = "ovn-kubernetes"
+		port                 string = "8080"
+		ovnWorkerNode        string = "ovn-worker"
+		ovnWorkerNode2       string = "ovn-worker2"
+		mcastGroup           string = "224.1.1.1"
+		mcastV6Group         string = "ff3e::4321:1234"
+		multicastListenerPod string = "multicast-listener-test-pod"
+		multicastSourcePod   string = "multicast-source-test-pod"
+		tcpdumpFileName      string = "tcpdump.txt"
+		retryTimeout                = 5 * time.Minute // polling timeout
+	)
+	var (
+		tcpDumpCommand = []string{"bash", "-c",
+			fmt.Sprintf("apk update; apk add tcpdump ; tcpdump multicast > %s", tcpdumpFileName)}
+		// Multicast group (-c 224.1.1.1), UDP (-u), TTL (-T 2), during (-t 3000) seconds, report every (-i 5) seconds
+		multicastSourceCommand = []string{"bash", "-c",
+			fmt.Sprintf("iperf -c %s -u -T 2 -t 3000 -i 5", mcastGroup)}
+	)
+	f := newPrivelegedTestFramework(svcname)
+	ginkgo.It("can retrieve multicast IGMP query", func() {
+		// Enable multicast of the test namespace annotation
+		ginkgo.By(fmt.Sprintf("annotating namespace: %s to enable multicast", f.Namespace.Name))
+		annotateArgs := []string{
+			"annotate",
+			"namespace",
+			f.Namespace.Name,
+			fmt.Sprintf("k8s.ovn.org/multicast-enabled=%s", "true"),
+		}
+		framework.RunKubectlOrDie(f.Namespace.Name, annotateArgs...)
+
+		// Create a multicast source pod
+		if IsIPv6Cluster(f.ClientSet) {
+			// Multicast group (-c ff3e::4321:1234), UDP (-u), TTL (-T 2), during (-t 3000) seconds, report every (-i 5) seconds, -V (Set the domain to IPv6)
+			multicastSourceCommand = []string{"bash", "-c",
+				fmt.Sprintf("iperf -c %s -u -T 2 -t 3000 -i 5 -V", mcastV6Group)}
+		}
+		ginkgo.By("creating a multicast source pod in node " + ovnWorkerNode)
+		createGenericPod(f, multicastSourcePod, ovnWorkerNode, f.Namespace.Name, multicastSourceCommand)
+
+		// Create a multicast listener pod
+		ginkgo.By("creating a multicast listener pod in node " + ovnWorkerNode2)
+		createGenericPod(f, multicastListenerPod, ovnWorkerNode2, f.Namespace.Name, tcpDumpCommand)
+		time.Sleep(5 * time.Hour)
+		// Wait for tcpdump on listener pod to be ready
+		err := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) {
+			kubectlOut, err := framework.RunKubectl(f.Namespace.Name, "exec", multicastListenerPod, "--", "/bin/bash", "-c", "ls")
+			if err != nil {
+				time.Sleep(5 * time.Hour)
+				framework.Failf("failed to retrieve multicast IGMP query: " + err.Error())
+			}
+			if !strings.Contains(kubectlOut, tcpdumpFileName) {
+				return false, nil
+			}
+			return true, nil
+		})
+		if err != nil {
+			framework.Failf("failed to retrieve multicast IGMP query: " + err.Error())
+		}
+
+		// The multicast listener pod join multicast group (-B 224.1.1.1), UDP (-u), during (-t 30) seconds, report every (-i 5) seconds
+		ginkgo.By("multicast listener pod join multicast group")
+		framework.RunKubectl(f.Namespace.Name, "exec", multicastListenerPod, "--", "/bin/bash", "-c", fmt.Sprintf("iperf -s -B %s -u -t 30 -i 5", mcastGroup))
+
+		ginkgo.By(fmt.Sprintf("verifying that the IGMP query has been received"))
+		kubectlOut, err := framework.RunKubectl(f.Namespace.Name, "exec", multicastListenerPod, "--", "/bin/bash", "-c", fmt.Sprintf("cat %s | grep igmp", tcpdumpFileName))
+		if err != nil {
+			framework.Failf("failed to retrieve multicast IGMP query: " + err.Error())
+		}
+		framework.Logf("output:")
+		framework.Logf(kubectlOut)
+		if kubectlOut == "" {
+			framework.Failf("failed to retrieve multicast IGMP query: igmp messages on the tcpdump logfile not found")
+		}
+	})
+
+})

test/e2e/service.go

@@ -313,10 +313,14 @@ var _ = ginkgo.Describe("Services", func() {
 				ginkgo.It("queries to the nodePort service shall work for TCP", func() {
 					for _, size := range []string{"small", "large"} {
 						for _, serviceNodeIP := range serviceNodeInternalIPs {
+							targetIP := serviceNodeIP
+							if IsIPv6Cluster(f.ClientSet) {
+								targetIP = fmt.Sprintf("[%s]", targetIP)
+							}
 							ginkgo.By(fmt.Sprintf("Sending TCP %s payload to service IP %s "+
 								"and expecting to receive the same payload", size, serviceNodeIP))
 							cmd := fmt.Sprintf("curl --max-time 10 -g -q -s http://%s:%d/echo?msg=%s",
-								serviceNodeIP,
+								targetIP,
 								servicePort,
 								echoPayloads[size],
 							)
@@ -407,6 +411,11 @@ var _ = ginkgo.Describe("Services", func() {
 								if stdout != echoPayloads[size] {
 									return fmt.Errorf("stdout does not match payloads[%s], %s != %s", size, stdout, echoPayloads[size])
 								}
+								// fc00:f853:ccd:e793::3 from :: dev breth0 src fc00:f853:ccd:e793::4 metric 256 expires 537sec mtu 1400 pref medium
+								// for IPV6 the regex changes a bit
+								if IsIPv6Cluster(f.ClientSet) {
+									echoMtuRegex = regexp.MustCompile(`expires.*mtu.*`)
+								}
 
 								if size == "large" {
 									cmd = fmt.Sprintf("ip route get %s", serviceNodeIP)

test/e2e/util.go

@@ -756,6 +756,9 @@ func pokeAllPodIPs(fr *framework.Framework, srcPodName string, dstPod *v1.Pod) e
 }
 
 func pokeExternalHostFromPod(fr *framework.Framework, namespace string, srcPodName, dstIp string, dstPort int) error {
+	if utilnet.IsIPv6String(dstIp) {
+		dstIp = fmt.Sprintf("[%s]", dstIp)
+	}
 	stdout, stderr, err := ExecShellInPodWithFullOutput(
 		fr,
 		namespace,

test/scripts/e2e-cp.sh

@@ -8,10 +8,29 @@ export KUBECONFIG=${HOME}/ovn.conf
 
 # Skip tests which are not IPv6 ready yet (see description of https://github.com/ovn-org/ovn-kubernetes/pull/2276)
 # (Note that netflow v5 is IPv4 only)
+# NOTE: Some of these tests that check connectivity to internet cannot be run.
+#       See https://github.com/actions/runner-images/issues/668#issuecomment-1480921915 for details
+# There were some past efforts to re-enable some of these skipped tests, but that never happened and they are
+# still failing v6 lane: https://github.com/ovn-org/ovn-kubernetes/pull/2505,
+# https://github.com/ovn-org/ovn-kubernetes/pull/2524, https://github.com/ovn-org/ovn-kubernetes/pull/2287; so
+# going to skip them again.
 IPV6_SKIPPED_TESTS="Should be allowed by externalip services|\
 should provide connection to external host by DNS name from a pod|\
+should provide Internet connection continuously when pod running master instance of ovnkube-master is killed|\
+should provide Internet connection continuously when all pods are killed on node running master instance of ovnkube-master|\
+should provide Internet connection continuously when all ovnkube-master pods are killed|\
+should provide Internet connection continuously when ovnkube-node pod is killed|\
+Should validate the egress firewall policy functionality against remote hosts|\
+Should validate the egress firewall policy functionality against cluster nodes by using node selector|\
+Should validate the egress IP functionality against remote hosts|\
+Should validate ICMP connectivity to multiple external gateways for an ECMP scenario|\
+Should validate ICMP connectivity to an external gateway\'s loopback address via a pod with external gateway annotations enabled|\
+Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario|\
+Should validate TCP/UDP connectivity to an external gateway\'s loopback address via a pod with external gateway annotations enabled|\
+Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes|\
 Should validate flow data of br-int is sent to an external gateway with netflow v5|\
-test tainting a node according to its defaults interface MTU size|\
+can retrieve multicast IGMP query|\
+test node readiness according to its defaults interface MTU size|\
 ipv4 pod"
 
 SKIPPED_TESTS=""