From 1456fe033f2e7294797b81f97a948b35b2cb6179 Mon Sep 17 00:00:00 2001 From: Hongliang Liu <75655411+hongliangl@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:00:40 +0800 Subject: [PATCH] Add e2e tests for BGPPolicy (#6523) * Add e2e tests for BGPPolicy Signed-off-by: Hongliang Liu * e2e: Address review comments on BGPPolicy Signed-off-by: Quan Tian --------- Signed-off-by: Hongliang Liu Signed-off-by: Quan Tian Co-authored-by: Quan Tian --- test/e2e/bgppolicy_test.go | 295 +++++++++++++++++++++++++++++++++++++ test/e2e/fixtures.go | 6 + test/e2e/framework.go | 14 ++ 3 files changed, 315 insertions(+) create mode 100644 test/e2e/bgppolicy_test.go diff --git a/test/e2e/bgppolicy_test.go b/test/e2e/bgppolicy_test.go new file mode 100644 index 00000000000..e7b4fcc68d8 --- /dev/null +++ b/test/e2e/bgppolicy_test.go @@ -0,0 +1,295 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "log" + "regexp" + "sort" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" + + "antrea.io/antrea/pkg/agent/types" + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + "antrea.io/antrea/test/e2e/providers/exec" +) + +const ( + externalASN = int32(65000) + + nodeASN = int32(64512) + updatedNodeASN = int32(64513) + + bgpPeerPassword = "password" + + bgpPolicyName = "test-policy" +) + +func getAllNodeIPs() []string { + ips := make([]string, 0, clusterInfo.numNodes) + for _, node := range clusterInfo.nodes { + ips = append(ips, node.ipv4Addr) + } + return ips +} + +type FRRRoute struct { + Prefix string + Nexthops []string +} + +func (f *FRRRoute) String() string { + sort.Strings(f.Nexthops) + return fmt.Sprintf("%s via %s", f.Prefix, strings.Join(f.Nexthops, ",")) +} + +func routesToStrings(routes []FRRRoute) []string { + s := make([]string, 0, len(routes)) + for _, route := range routes { + s = append(s, route.String()) + } + return s +} + +func TestBGPPolicy(t *testing.T) { + skipIfNotIPv4Cluster(t) + skipIfHasWindowsNodes(t) + skipIfExternalFRRNotSet(t) + + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + t.Log("Configure the remote FRR router with BGP") + configureExternalBGPRouter(t, externalASN, nodeASN, true) + + t.Log("Update the specific Secret storing the passwords of BGP peers") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: kubeNamespace, + Name: types.BGPPolicySecretName, + }, + Data: map[string][]byte{ + fmt.Sprintf("%s-%d", externalInfo.externalFRRIPv4, externalASN): []byte(bgpPeerPassword), + }, + } + _, err = data.clientset.CoreV1().Secrets(kubeNamespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + require.NoError(t, err) + defer data.clientset.CoreV1().Secrets(kubeNamespace).Delete(context.TODO(), types.BGPPolicySecretName, metav1.DeleteOptions{}) + + t.Log("Create a test agnhost Pod") + podName, podIPs, cleanupFunc := createAndWaitForPod(t, data, func(name string, ns string, nodeName string, hostNetwork bool) error { + args := []string{"netexec", "--http-port=8080"} + ports := []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 8080, + Protocol: corev1.ProtocolTCP, + }, + } + return NewPodBuilder(name, ns, agnhostImage). + OnNode(nodeName). + WithArgs(args). + WithPorts(ports). + WithHostNetwork(hostNetwork). + Create(data) + }, "agnhost-", nodeName(0), data.testNamespace, false) + defer cleanupFunc() + podIP := podIPs.IPv4.String() + + t.Log("Create a test Service") + svcClusterIP, err := data.createAgnhostClusterIPService("agnhost-svc", false, ptr.To[corev1.IPFamily](corev1.IPv4Protocol)) + defer data.deleteService(svcClusterIP.Namespace, svcClusterIP.Name) + require.NoError(t, err) + require.NotEqual(t, "", svcClusterIP.Spec.ClusterIP, "ClusterIP should not be empty") + clusterIP := svcClusterIP.Spec.ClusterIP + + t.Log("Create a test BGPPolicy selecting all Nodes as well as advertising ClusterIPs and Pod CIDRs") + bgpPolicy := &crdv1alpha1.BGPPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: bgpPolicyName, + }, + Spec: crdv1alpha1.BGPPolicySpec{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{}, + }, + LocalASN: nodeASN, + ListenPort: ptr.To[int32](179), + Advertisements: crdv1alpha1.Advertisements{ + Service: &crdv1alpha1.ServiceAdvertisement{ + IPTypes: []crdv1alpha1.ServiceIPType{crdv1alpha1.ServiceIPTypeClusterIP}, + }, + Pod: &crdv1alpha1.PodAdvertisement{}, + }, + BGPPeers: []crdv1alpha1.BGPPeer{ + {Address: externalInfo.externalFRRIPv4, ASN: externalASN}, + }, + }, + } + bgpPolicy, err = data.crdClient.CrdV1alpha1().BGPPolicies().Create(context.TODO(), bgpPolicy, metav1.CreateOptions{}) + defer data.crdClient.CrdV1alpha1().BGPPolicies().Delete(context.TODO(), bgpPolicyName, metav1.DeleteOptions{}) + require.NoError(t, err) + + t.Log("Get the routes installed on remote FRR router and verify them") + expectedRoutes := make([]FRRRoute, 0) + for _, node := range clusterInfo.nodes { + expectedRoutes = append(expectedRoutes, FRRRoute{Prefix: node.podV4NetworkCIDR, Nexthops: []string{node.ipv4Addr}}) + } + expectedRoutes = append(expectedRoutes, FRRRoute{Prefix: clusterIP + "/32", Nexthops: getAllNodeIPs()}) + checkFRRRouterBGPRoutes(t, expectedRoutes, nil) + + t.Log("Verify the connectivity of the installed routes on remote FRR route") + ipsToConnect := []string{podIP, clusterIP} + for _, ip := range ipsToConnect { + cmd := fmt.Sprintf("/usr/bin/wget -O - http://%s:8080/hostname -T 5", ip) + rc, stdout, _, err := exec.RunDockerExecCommand(externalInfo.externalFRRCID, cmd, "/", nil, "") + require.NoError(t, err) + require.Equal(t, 0, rc) + require.Equal(t, podName, stdout) + } + + t.Log("Update the BGP configuration on the remote FRR router") + configureExternalBGPRouter(t, externalASN, updatedNodeASN, false) + + _, err = data.updateServiceInternalTrafficPolicy("agnhost-svc", true) + require.NoError(t, err) + + t.Log("Update the test BGPPolicy with a new local ASN") + updatedBGPPolicy := bgpPolicy.DeepCopy() + updatedBGPPolicy.Spec.LocalASN = updatedNodeASN + updatedBGPPolicy.Spec.Advertisements.Pod = nil + _, err = data.crdClient.CrdV1alpha1().BGPPolicies().Update(context.TODO(), updatedBGPPolicy, metav1.UpdateOptions{}) + require.NoError(t, err) + + t.Log("Get routes installed on remote FRR router and verify them") + expectedRoutes = []FRRRoute{{Prefix: clusterIP + "/32", Nexthops: []string{nodeIPv4(0)}}} + notExpectedRoutes := []FRRRoute{{Prefix: clusterIP + "/32", Nexthops: getAllNodeIPs()}} + checkFRRRouterBGPRoutes(t, expectedRoutes, notExpectedRoutes) + + t.Log("verify the connectivity of the installed routes on remote FRR route") + ipsToConnect = []string{clusterIP} + for _, ip := range ipsToConnect { + cmd := fmt.Sprintf("/usr/bin/wget -O - http://%s:8080/hostname -T 5", ip) + rc, stdout, _, err := exec.RunDockerExecCommand(externalInfo.externalFRRCID, cmd, "/", nil, "") + require.NoError(t, err) + require.Equal(t, 0, rc) + require.Equal(t, podName, stdout) + } +} + +func checkFRRRouterBGPRoutes(t *testing.T, expectedRoutes, notExpectedRoutes []FRRRoute) { + t.Helper() + expectedRouteStrings := routesToStrings(expectedRoutes) + notExpectedRouteStrings := routesToStrings(notExpectedRoutes) + var gotRoutes []FRRRoute + err := wait.PollUntilContextTimeout(context.Background(), time.Second, 30*time.Second, true, func(context.Context) (bool, error) { + var err error + gotRoutes, err = dumpFRRRouterBGPRoutes() + if err != nil { + return false, err + } + gotRoutesSet := sets.NewString(routesToStrings(gotRoutes)...) + if !gotRoutesSet.HasAll(expectedRouteStrings...) { + return false, nil + } + if gotRoutesSet.HasAny(notExpectedRouteStrings...) { + return false, nil + } + return true, nil + }) + + require.NoError(t, err, "Failed to get the expected BGP routes, expected: %v, unexpected: %v, got: %v", expectedRoutes, notExpectedRoutes, gotRoutes) +} + +func runVtyshCommands(commands []string) (int, string, string, error) { + return exec.RunDockerExecCommand(externalInfo.externalFRRCID, "/usr/bin/vtysh", "/", nil, strings.Join(commands, "\n")) +} + +func configureExternalBGPRouter(t *testing.T, externalASN, nodeASN int32, deferCleanup bool) { + commands := []string{ + "configure terminal", + fmt.Sprintf("router bgp %d", externalASN), + "no bgp ebgp-requires-policy", + "no bgp network import-check", + } + for _, node := range clusterInfo.nodes { + commands = append(commands, fmt.Sprintf("neighbor %s remote-as %d", node.ipv4Addr, nodeASN)) + commands = append(commands, fmt.Sprintf("neighbor %s password %s", node.ipv4Addr, bgpPeerPassword)) + } + commands = append(commands, + "exit", + "exit", + "write memory") + rc, stdout, stderr, err := runVtyshCommands(commands) + require.NoError(t, err, "Configuring external BGP router failed, rc: %v, stdout: %s, stderr: %s", rc, stdout, stderr) + require.Equal(t, 0, rc, "Configuring external BGP router returned non-zero code, stdout: %s, stderr: %s", stdout, stderr) + + if deferCleanup { + t.Cleanup(func() { + rc, stdout, stderr, err := runVtyshCommands([]string{ + "configure terminal", + fmt.Sprintf("no router bgp %d", externalASN), + "exit", + "write memory", + }) + require.NoError(t, err, "Restoring external BGP router failed, rc: %v, stdout: %s, stderr: %s", rc, stdout, stderr) + require.Equal(t, 0, rc, "Restoring external BGP router returned non-zero code, stdout: %s, stderr: %s", stdout, stderr) + }) + } +} + +func dumpFRRRouterBGPRoutes() ([]FRRRoute, error) { + rc, stdout, stderr, err := runVtyshCommands([]string{"show ip route bgp"}) + log.Println(stdout) + log.Println(stderr) + if err != nil || rc != 0 { + return nil, fmt.Errorf("error when running command to show BGP route") + } + + routePattern := regexp.MustCompile(`B>\* ([\d\.\/]+) \[.*?\] via ([\d\.]+),`) + nexthopPattern := regexp.MustCompile(`\* +via ([\d\.]+),`) + var routes []FRRRoute + lines := strings.Split(stdout, "\n") + for _, line := range lines { + routeMatches := routePattern.FindStringSubmatch(line) + if routeMatches != nil { + route := FRRRoute{ + Prefix: routeMatches[1], + Nexthops: []string{routeMatches[2]}, + } + routes = append(routes, route) + continue + } + + nexthopMatches := nexthopPattern.FindStringSubmatch(line) + if nexthopMatches != nil && len(routes) > 0 { + last := len(routes) - 1 + routes[last].Nexthops = append(routes[last].Nexthops, nexthopMatches[1]) + } + } + return routes, nil +} diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index b8aa8674648..e948b988d82 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -68,6 +68,12 @@ func skipIfProviderIs(tb testing.TB, name string, reason string) { } } +func skipIfExternalFRRNotSet(tb testing.TB) { + if testOptions.externalFRRIPs == "" { + tb.Skipf("Skipping test since the external FRR IPs are not set ") + } +} + func skipIfNotRequired(tb testing.TB, keys ...string) { for _, v := range keys { if strings.Contains(testOptions.skipCases, v) { diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 39d43adcb75..218aead4859 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -2068,6 +2068,20 @@ func (data *TestData) updateServiceExternalTrafficPolicy(serviceName string, nod return data.clientset.CoreV1().Services(data.testNamespace).Update(context.TODO(), svc, metav1.UpdateOptions{}) } +func (data *TestData) updateServiceInternalTrafficPolicy(serviceName string, nodeLocalInternal bool) (*corev1.Service, error) { + svc, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) + if err != nil { + return svc, err + } + if nodeLocalInternal { + svc.Spec.InternalTrafficPolicy = ptr.To(corev1.ServiceInternalTrafficPolicyLocal) + } else { + svc.Spec.InternalTrafficPolicy = ptr.To(corev1.ServiceInternalTrafficPolicyCluster) + } + + return data.clientset.CoreV1().Services(data.testNamespace).Update(context.TODO(), svc, metav1.UpdateOptions{}) +} + func (data *TestData) updateService(serviceName string, mutateFunc func(service *corev1.Service)) (*corev1.Service, error) { svc, err := data.clientset.CoreV1().Services(data.testNamespace).Get(context.TODO(), serviceName, metav1.GetOptions{}) if err != nil {