Skip to content

Commit

Permalink
Run ipv6 lane in control plane jobs
Browse files Browse the repository at this point in the history 
This commit fixes things here and there
in e2e's to make control planes run on
v6 as well. Note that some tests are
skipped since they cannot be run on
github runners with v6 enabled.

NOTE: One of the multicast test is moved
from e2e.go to multicast.go where it belongs.

Signed-off-by: Surya Seetharaman <suryaseetharaman.9@gmail.com>
  • Loading branch information
@tssurya
tssurya committed Feb 16, 2024
1 parent b776ad5 commit 53ccb16
Show file tree
Hide file tree
Showing 9 changed files with 227 additions and 110 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,10 +395,12 @@ jobs:
- {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "local", "ipfamily": "dualstack", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
- {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
- {"target": "shard-conformance", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
- {"target": "control-plane", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-disabled"}
- {"target": "control-plane", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-disabled"}
- {"target": "control-plane", "ha": "HA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "snatGW", "second-bridge": "1br", "ic": "ic-disabled"}
- {"target": "control-plane", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
- {"target": "control-plane", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
- {"target": "control-plane", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "ic": "ic-single-node-zones"}
- {"target": "control-plane", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "ic": "ic-single-node-zones"}
- {"target": "multi-homing", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-disabled"}
- {"target": "node-ip-mac-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv6", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-disabled"}
- {"target": "node-ip-mac-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
Expand Down
39 changes: 24 additions & 15 deletions test/e2e/acl_logging.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,15 +174,7 @@ var _ = Describe("ACL Logging for EgressFirewall", func() {
allowACLVerdict = "allow"
namespacePrefix = "acl-log-egressfw"
secondaryNamespacePrefix = "acl-log-egressfw-sec"

// These targets must be off cluster - traffic to the cluster should always be
// allowed: https://docs.openshift.com/container-platform/4.10/networking/openshift_sdn/configuring-egress-firewall.html
// "As a cluster administrator, you can create an egress firewall for a project that restricts egress traffic leaving
// your OpenShift Container Platform cluster."
// Because the egress firewall feature only affects traffic leaving the cluster, we will not log for on-cluster targets.
allowedDstIP = "172.18.0.1"
deniedDstIP = "172.19.0.10"
dstPort = 8080
dstPort = 8080
)

fr := wrappedTestFramework(namespacePrefix)
Expand All @@ -192,15 +184,32 @@ var _ = Describe("ACL Logging for EgressFirewall", func() {
nsNameSecondary string
pokePod *v1.Pod
pokePodSecondary *v1.Pod
allowedDstIP string
deniedDstIP string
)

BeforeEach(func() {
// These targets must be off cluster - traffic to the cluster should always be
// allowed: https://docs.openshift.com/container-platform/4.10/networking/openshift_sdn/configuring-egress-firewall.html
// "As a cluster administrator, you can create an egress firewall for a project that restricts egress traffic leaving
// your OpenShift Container Platform cluster."
// Because the egress firewall feature only affects traffic leaving the cluster, we will not log for on-cluster targets.
allowedDstIP = "172.18.0.1"
deniedDstIP = "172.19.0.10"
mask := "32"
denyCIDR := "0.0.0.0/0"
if IsIPv6Cluster(fr.ClientSet) {
allowedDstIP = "2001:4860:4860::8888"
deniedDstIP = "2001:4860:4860::8844"
mask = "128"
denyCIDR = "::/0"
}
By("configuring the ACL logging level within the namespace")
nsName = fr.Namespace.Name
Expect(setNamespaceACLLogSeverity(fr, nsName, initialDenyACLSeverity, initialAllowACLSeverity, aclRemoveOptionDelete)).To(Succeed())

By("creating a \"default deny\" Egress Firewall")
err := makeEgressFirewall(nsName)
err := makeEgressFirewall(nsName, allowedDstIP, mask, denyCIDR)
Expect(err).NotTo(HaveOccurred())

By("creating a pod running agnhost netexec")
Expand All @@ -221,7 +230,7 @@ var _ = Describe("ACL Logging for EgressFirewall", func() {
Expect(setNamespaceACLLogSeverity(fr, nsNameSecondary, initialDenyACLSeverity, initialAllowACLSeverity, aclRemoveOptionDelete)).To(Succeed())

By("creating a \"default deny\" Egress Firewall inside the secondary namespace")
err = makeEgressFirewall(nsNameSecondary)
err = makeEgressFirewall(nsNameSecondary, allowedDstIP, mask, denyCIDR)
Expect(err).NotTo(HaveOccurred())

By("creating a pod running agnhost netexec inside the secondary namespace")
Expand Down Expand Up @@ -621,7 +630,7 @@ func makeDenyAllPolicy(f *framework.Framework, ns string, policyName string) (*k
return f.ClientSet.NetworkingV1().NetworkPolicies(ns).Create(context.TODO(), policy, metav1.CreateOptions{})
}

func makeEgressFirewall(ns string) error {
func makeEgressFirewall(ns, allowedDstIP, mask, denyCIDR string) error {
egressFirewallYaml := "egressfirewall.yaml"
var egressFirewallConfig = fmt.Sprintf(`apiVersion: k8s.ovn.org/v1
kind: EgressFirewall
Expand All @@ -632,11 +641,11 @@ spec:
egress:
- type: Allow
to:
cidrSelector: 172.18.0.1/32
cidrSelector: %s/%s
- type: Deny
to:
cidrSelector: 0.0.0.0/0
`)
cidrSelector: %s
`, allowedDstIP, mask, denyCIDR)

if err := os.WriteFile(egressFirewallYaml, []byte(egressFirewallConfig), 0644); err != nil {
framework.Failf("Unable to write CRD config to disk: %v", err)
Expand Down
81 changes: 4 additions & 77 deletions test/e2e/e2e.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,11 +333,11 @@ func isolateIPv6Networks(networkA, networkB string) error {
if len(bridgeInfNames) != 2 {
return fmt.Errorf("expected two bridge names but found %d", len(bridgeInfNames))
}
_, err := runCommand("ip6tables", "-t", "filter", "-A", "FORWARD", "-i", bridgeInfNames[0], "-o", bridgeInfNames[1], "-j", "DROP")
_, err := runCommand("sudo", "ip6tables", "-t", "filter", "-A", "FORWARD", "-i", bridgeInfNames[0], "-o", bridgeInfNames[1], "-j", "DROP")
if err != nil {
return err
}
_, err = runCommand("ip6tables", "-t", "filter", "-A", "FORWARD", "-i", bridgeInfNames[1], "-o", bridgeInfNames[0], "-j", "DROP")
_, err = runCommand("sudo", "ip6tables", "-t", "filter", "-A", "FORWARD", "-i", bridgeInfNames[1], "-o", bridgeInfNames[0], "-j", "DROP")
return err
}

Expand Down Expand Up @@ -753,7 +753,7 @@ var _ = ginkgo.Describe("e2e control plane", func() {
framework.ExpectNoError(err, "one or more nodes failed to go back ready, schedulable, and untainted")
})

ginkgo.It("should provide Internet connection continuously when all pods are killed on node running master instance of ovnkube-control-plane.", func() {
ginkgo.It("should provide Internet connection continuously when all pods are killed on node running master instance of ovnkube-control-plane", func() {
ginkgo.By(fmt.Sprintf("Running container which tries to connect to %s in a loop", extDNSIP))

ovnKubeControlPlaneNode, err := findOvnKubeControlPlaneNode(controlPlanePodName, controlPlaneLeaseName)
Expand Down Expand Up @@ -794,7 +794,7 @@ var _ = ginkgo.Describe("e2e control plane", func() {
framework.ExpectNoError(<-errChan)
})

ginkgo.It("should provide Internet connection continuously when all ovnkube-control-plane pods are killed.", func() {
ginkgo.It("should provide Internet connection continuously when all ovnkube-control-plane pods are killed", func() {
ginkgo.By(fmt.Sprintf("Running container which tries to connect to %s in a loop", extDNSIP))

podChan, errChan := make(chan *v1.Pod), make(chan error)
Expand Down Expand Up @@ -2277,76 +2277,3 @@ var _ = ginkgo.Describe("e2e delete databases", func() {
singlePodConnectivityTest(f, "after-delete-db-pods")
})
})

var _ = ginkgo.Describe("e2e IGMP validation", func() {
const (
svcname string = "igmp-test"
ovnNs string = "ovn-kubernetes"
port string = "8080"
ovnWorkerNode string = "ovn-worker"
ovnWorkerNode2 string = "ovn-worker2"
mcastGroup string = "224.1.1.1"
multicastListenerPod string = "multicast-listener-test-pod"
multicastSourcePod string = "multicast-source-test-pod"
tcpdumpFileName string = "tcpdump.txt"
retryTimeout = 5 * time.Minute // polling timeout
)
var (
tcpDumpCommand = []string{"bash", "-c",
fmt.Sprintf("apk update; apk add tcpdump ; tcpdump multicast > %s", tcpdumpFileName)}
// Multicast group (-c 224.1.1.1), UDP (-u), TTL (-T 2), during (-t 3000) seconds, report every (-i 5) seconds
multicastSourceCommand = []string{"bash", "-c",
fmt.Sprintf("iperf -c %s -u -T 2 -t 3000 -i 5", mcastGroup)}
)
f := wrappedTestFramework(svcname)
ginkgo.It("can retrieve multicast IGMP query", func() {
// Enable multicast of the test namespace annotation
ginkgo.By(fmt.Sprintf("annotating namespace: %s to enable multicast", f.Namespace.Name))
annotateArgs := []string{
"annotate",
"namespace",
f.Namespace.Name,
fmt.Sprintf("k8s.ovn.org/multicast-enabled=%s", "true"),
}
e2ekubectl.RunKubectlOrDie(f.Namespace.Name, annotateArgs...)

// Create a multicast source pod
ginkgo.By("creating a multicast source pod in node " + ovnWorkerNode)
createGenericPod(f, multicastSourcePod, ovnWorkerNode, f.Namespace.Name, multicastSourceCommand)

// Create a multicast listener pod
ginkgo.By("creating a multicast listener pod in node " + ovnWorkerNode2)
createGenericPod(f, multicastListenerPod, ovnWorkerNode2, f.Namespace.Name, tcpDumpCommand)

// Wait for tcpdump on listener pod to be ready
err := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) {
kubectlOut, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", multicastListenerPod, "--", "/bin/bash", "-c", "ls")
if err != nil {
framework.Failf("failed to retrieve multicast IGMP query: " + err.Error())
}
if !strings.Contains(kubectlOut, tcpdumpFileName) {
return false, nil
}
return true, nil
})
if err != nil {
framework.Failf("failed to retrieve multicast IGMP query: " + err.Error())
}

// The multicast listener pod join multicast group (-B 224.1.1.1), UDP (-u), during (-t 30) seconds, report every (-i 5) seconds
ginkgo.By("multicast listener pod join multicast group")
e2ekubectl.RunKubectl(f.Namespace.Name, "exec", multicastListenerPod, "--", "/bin/bash", "-c", fmt.Sprintf("iperf -s -B %s -u -t 30 -i 5", mcastGroup))

ginkgo.By(fmt.Sprintf("verifying that the IGMP query has been received"))
kubectlOut, err := e2ekubectl.RunKubectl(f.Namespace.Name, "exec", multicastListenerPod, "--", "/bin/bash", "-c", fmt.Sprintf("cat %s | grep igmp", tcpdumpFileName))
if err != nil {
framework.Failf("failed to retrieve multicast IGMP query: " + err.Error())
}
framework.Logf("output:")
framework.Logf(kubectlOut)
if kubectlOut == "" {
framework.Failf("failed to retrieve multicast IGMP query: igmp messages on the tcpdump logfile not found")
}
})

})
16 changes: 12 additions & 4 deletions test/e2e/egress_firewall.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ spec:
ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest))
_, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "nc", "-vz", "-w", testTimeout, exFWPermitTcpDnsDest, "53")
if err != nil {
framework.Failf("Failed to connect to the remote host %s from container %s on node %s: %v", exFWPermitTcpDnsDest, ovnContainer, serverNodeInfo.name, err)
framework.Failf("Failed to connect to the remote host %s from container %s on node %s: %v", exFWPermitTcpDnsDest, srcPodName, serverNodeInfo.name, err)
}
// Verify the remote host/port as implicitly denied by the firewall policy is not reachable
ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined by the external firewall policy", exFWDenyTcpDnsDest))
Expand Down Expand Up @@ -524,6 +524,10 @@ spec:
frameworkNsFlag := fmt.Sprintf("--namespace=%s", f.Namespace.Name)
testContainer := fmt.Sprintf("%s-container", efPodName)
testContainerFlag := fmt.Sprintf("--container=%s", testContainer)
denyCIDR := "0.0.0.0/0"
if IsIPv6Cluster(f.ClientSet) {
denyCIDR = "::/0"
}
// egress firewall crd yaml configuration
var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall
apiVersion: k8s.ovn.org/v1
Expand All @@ -534,8 +538,8 @@ spec:
egress:
- type: Deny
to:
cidrSelector: 0.0.0.0/0
`, f.Namespace.Name)
cidrSelector: %s
`, f.Namespace.Name, denyCIDR)
// write the config to a file for application and defer the removal
if err := os.WriteFile(egressFirewallYamlFile, []byte(egressFirewallConfig), 0644); err != nil {
framework.Failf("Unable to write CRD config to disk: %v", err)
Expand Down Expand Up @@ -571,13 +575,17 @@ spec:
framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, f.Namespace.Name)

nodeIP := serverNodeInfo.nodeIP
externalContainerIP, _ := createClusterExternalContainer(externalContainerName, agnhostImage,
externalContainerIPV4, externalContainerIPV6 := createClusterExternalContainer(externalContainerName, agnhostImage,
[]string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort, externalContainerPort)},
[]string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort)})
defer deleteClusterExternalContainer(externalContainerName)

// 2. Check connectivity works both ways
// pod -> external container should work
externalContainerIP := externalContainerIPV4
if IsIPv6Cluster(f.ClientSet) {
externalContainerIP = externalContainerIPV6
}
ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d",
efPodName, externalContainerIP, externalContainerPort))
_, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", efPodName, testContainerFlag,
Expand Down
Loading

0 comments on commit 53ccb16

Please sign in to comment.