From 588cad9c4e33a9694a39b332b852afb2ad3348b6 Mon Sep 17 00:00:00 2001 From: Jianjun Shen Date: Tue, 21 Nov 2023 00:03:08 -0500 Subject: [PATCH] Add e2e test for VLAN secondary network Extend secondary network e2e tests to support testing secondary VLAN networks. Add a script to run the test on a Kind cluster, and enable the test on Github CI. Signed-off-by: Jianjun Shen --- .github/workflows/kind.yml | 45 +++ build/charts/antrea/conf/antrea-agent.conf | 2 +- ci/kind/kind-setup.sh | 2 +- ci/kind/test-secondary-network-kind.sh | 125 +++++++ hack/generate-manifest.sh | 82 +++-- test/e2e-secondary-network/framework.go | 50 --- .../infra/prepare_cluster.sh | 52 +-- .../infra/prepare_sriov.sh | 15 +- .../infra/secondary-network-configuration.yml | 14 - .../infra/secondary-networks.yml | 147 +++++++++ .../infra/virtual-network-instance-crd.yml | 63 ---- test/e2e-secondary-network/main_test.go | 69 +--- .../secondary_network_test.go | 305 +++++++++--------- test/e2e/antctl_test.go | 2 +- test/e2e/bandwidth_test.go | 12 +- test/e2e/connectivity_test.go | 2 +- test/e2e/egress_test.go | 4 +- test/e2e/entry.go | 192 +++++++++++ test/e2e/fixtures.go | 2 +- test/e2e/flowaggregator_test.go | 4 +- test/e2e/framework.go | 24 +- test/e2e/main_test.go | 150 --------- test/e2e/performance_test.go | 2 +- test/e2e/proxy_test.go | 4 +- test/e2e/wireguard_test.go | 2 +- 25 files changed, 768 insertions(+), 603 deletions(-) create mode 100755 ci/kind/test-secondary-network-kind.sh delete mode 100644 test/e2e-secondary-network/framework.go delete mode 100755 test/e2e-secondary-network/infra/secondary-network-configuration.yml create mode 100644 test/e2e-secondary-network/infra/secondary-networks.yml delete mode 100644 test/e2e-secondary-network/infra/virtual-network-instance-crd.yml create mode 100644 test/e2e/entry.go diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 2b4c7532fc5..0c7730c1acb 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -511,6 +511,51 @@ jobs: path: log.tar.gz retention-days: 30 + test-secondary-network: + name: Antrea-native (VLAN) secondary network tests on a Kind cluster on Linux + needs: [build-antrea-coverage-image] + runs-on: [ubuntu-latest] + steps: + - name: Free disk space + # https://github.com/actions/virtual-environments/issues/709 + run: | + sudo apt-get clean + df -h + - uses: actions/checkout@v4 + with: + show-progress: false + - uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + - name: Download Antrea image from previous job + uses: actions/download-artifact@v3 + with: + name: antrea-ubuntu-cov + - name: Load Antrea image + run: | + docker load -i antrea-ubuntu.tar + docker tag antrea/antrea-ubuntu-coverage:latest antrea/antrea-ubuntu:latest + - name: Install Kind + run: | + KIND_VERSION=$(head -n1 ./ci/kind/version) + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run secondary network tests + run: | + mkdir log + ANTREA_LOG_DIR=$PWD/log ./ci/kind/test-secondary-network-kind.sh + - name: Tar log files + if: ${{ failure() }} + run: tar -czf log.tar.gz log + - name: Upload test log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: secondary-network.tar.gz + path: log.tar.gz + retention-days: 30 + test-upgrade-from-N-1: name: Upgrade from Antrea version N-1 needs: build-antrea-coverage-image diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index b565034f28f..11ee1f54b0e 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -436,7 +436,7 @@ secondaryNetwork: {{- with .Values.secondaryNetwork }} # Configuration of OVS bridges for secondary network. ovsBridges: - {{- toYaml .ovsBridges | trim | nindent 6 }} + {{- toYaml .ovsBridges | trim | nindent 4 }} {{- end }} {{- end }} diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh index 868e0ef75e9..27dd529f418 100755 --- a/ci/kind/kind-setup.sh +++ b/ci/kind/kind-setup.sh @@ -233,7 +233,7 @@ function configure_extra_networks { i=$((i+1)) done - nodes="$(kind get nodes --name $CLUSTER_NAME | grep worker)" + nodes="$(kind get nodes --name $CLUSTER_NAME)" for node in $nodes; do for network in $networks; do docker network connect $network $node >/dev/null 2>&1 diff --git a/ci/kind/test-secondary-network-kind.sh b/ci/kind/test-secondary-network-kind.sh new file mode 100755 index 00000000000..6da860ee5d2 --- /dev/null +++ b/ci/kind/test-secondary-network-kind.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +# Copyright 2024 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function echoerr { + >&2 echo "$@" +} + +_usage="Usage: $0 [--setup-only|--test-only|--cleanup-only] [--help|-h] + --setup-only Only perform setting up the cluster and run test. + --test-only Only run test on current cluster. Not set up/clean up the cluster. + --cleanup-only Only perform cleaning up the cluster. + --help, -h Print this message and exit. +" + +function print_usage { + echoerr -n "$_usage" +} + +THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +TESTBED_CMD=$THIS_DIR"/kind-setup.sh" +YML_CMD=$THIS_DIR"/../../hack/generate-manifest.sh" +ATTACHMENT_DEFINITION_YAML=$THIS_DIR"/../../test/e2e-secondary-network/infra/network-attachment-definition-crd.yml" +SECONDARY_NETWORKS_YAML=$THIS_DIR"/../../test/e2e-secondary-network/infra/secondary-networks.yml" + +TIMEOUT="5m" +# Create a secondary OVS bridge with the Node's physical interface on the extra +# network. +MANIFEST_ARGS="--secondary-bridge br-secondary --physical-interface eth1" +# Antrea is deployed by this script. Do not deploy it again in the test. +TEST_OPTIONS="--logs-export-dir=$ANTREA_LOG_DIR --deploy-antrea=false" +EXTRA_NETWORK="20.20.20.0/24" + +setup_only=false +cleanup_only=false +test_only=false + +function quit { + result=$? + if [[ $setup_only || $test_only ]]; then + exit $result + fi + echoerr "Cleaning testbed" + $TESTBED_CMD destroy kind +} + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + --setup-only) + setup_only=true + shift + ;; + --cleanup-only) + cleanup_only=true + shift + ;; + --test-only) + test_only=true + shift + ;; + -h|--help) + print_usage + exit 0 + ;; + *) # unknown option + echoerr "Unknown option $1" + exit 1 + ;; +esac +done + +if [[ $cleanup_only == "true" ]];then + $TESTBED_CMD destroy kind + exit 0 +fi + +trap "quit" INT EXIT + +IMAGE_LIST=("projects.registry.vmware.com/antrea/toolbox:1.3-0" \ + "antrea/antrea-agent-ubuntu:latest" \ + "antrea/antrea-controller-ubuntu:latest") + +printf -v IMAGES "%s " "${IMAGE_LIST[@]}" + +function setup_cluster { + args=$1 + echo "creating test bed with args $args" + eval "timeout 600 $TESTBED_CMD create kind $args" +} + +function run_test { + echo "Generating Antrea manifest with args $MANIFEST_ARGS" + $YML_CMD $MANIFEST_ARGS | kubectl apply -f - + + # Wait for antrea-controller start to make sure the IPPool validation webhook is ready. + kubectl rollout status --timeout=1m deployment.apps/antrea-controller -n kube-system + kubectl apply -f $ATTACHMENT_DEFINITION_YAML + kubectl apply -f $SECONDARY_NETWORKS_YAML + + go test -v -timeout=$TIMEOUT antrea.io/antrea/test/e2e-secondary-network -run=TestVLANNetwork -provider=kind $TEST_OPTIONS +} + +echo "======== Testing Antrea-native secondary network support ==========" +if [[ $test_only == "false" ]];then + setup_cluster "--extra-networks \"$EXTRA_NETWORK\" --images \"$IMAGES\"" +fi +run_test +exit 0 diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index d62df65a456..4fdeb2eede9 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -20,30 +20,32 @@ function echoerr { >&2 echo "$@" } -_usage="Usage: $0 [--mode (dev|release)] [--encap-mode] [--ipsec] [--tun (geneve|vxlan|gre|stt)] [--verbose-log] [--help|-h] +_usage="Usage: $0 [--mode (dev|release)] [--encap-mode (mode)] [--ipsec] [--tun (geneve|vxlan|gre|stt)] [--verbose-log] Generate a YAML manifest for Antrea using Helm and print it to stdout. - --mode (dev|release) Choose the configuration variant that you need (default is 'dev') - --encap-mode Traffic encapsulation mode. (default is 'encap') - --cloud Generate a manifest appropriate for running Antrea in Public Cloud - --ipsec Generate a manifest with IPsec encryption of tunnel traffic enabled + --mode (dev|release) Choose the configuration variant that you need (default is 'dev'). + --encap-mode (mode) Traffic encapsulation mode (default is 'encap'). + --cloud Generate a manifest appropriate for running Antrea in Public Cloud. + --ipsec Generate a manifest with IPsec encryption of tunnel traffic enabled. --feature-gates A comma-separated list of key=value pairs that describe feature gates, e.g. TrafficControl=true,Egress=false. - --proxy-all Generate a manifest with Antrea proxy with all Service support enabled - --tun (geneve|vxlan|gre|stt) Choose encap tunnel type from geneve, gre, stt and vxlan (default is geneve) - --verbose-log Generate a manifest with increased log-level (level 4) for Antrea agent and controller. - This option will work only in 'dev' mode. + --proxy-all Generate a manifest with Antrea proxy with all Service support enabled. + --tun (geneve|vxlan|gre|stt) Choose encap tunnel type from geneve, gre, stt and vxlan (default is geneve). --on-delete Generate a manifest with antrea-agent's update strategy set to OnDelete. This option will work only in 'dev' mode. - --coverage Generates a manifest which supports measuring code coverage of Antrea binaries. - --simulator Generates a manifest with antrea-agent simulator included - --custom-adm-controller Generates a manifest with custom Antrea admission controller to validate/mutate resources. - --hw-offload Generates a manifest with hw-offload enabled in the antrea-ovs container. - --sriov Generates a manifest which enables use of Kubelet API for SR-IOV device info. - --flexible-ipam Generates a manifest with flexible IPAM enabled. - --help, -h Print this message and exit - --multicast Generates a manifest for multicast. - --multicast-interfaces Multicast interface names (default is empty) - --extra-helm-values-file Optional extra helm values file to override the default config values - --extra-helm-values Optional extra helm values to override the default config values + --coverage Generate a manifest which supports measuring code coverage of Antrea binaries. + --simulator Generate a manifest with antrea-agent simulator included. + --custom-adm-controller Generate a manifest with custom Antrea admission controller to validate/mutate resources. + --flexible-ipam Generate a manifest with flexible IPAM enabled. + --hw-offload Generate a manifest with hw-offload enabled in the antrea-ovs container. + --sriov Generate a manifest which enables use of kubelet API for SR-IOV device info. + --secondary-bridge (bridge) Generate a manifest which enables secondary network and creates a secondary OVS bridge. + --physical-interface (device) Specify the physical interface of the secondary OVS bridge. + --multicast Generate a manifest for multicast. + --multicast-interfaces Multicast interface names (default is empty). + --extra-helm-values-file Optional extra helm values file to override the default config values. + --extra-helm-values Optional extra helm values to override the default config values. + --verbose-log Generate a manifest with increased log-level (level 4) for Antrea agent and controller. + This option will work only in 'dev' mode. + --help, -h Print this message and exit. In 'release' mode, environment variables AGENT_IMG_NAME, CONTROLLER_IMG_NAME, and IMG_TAG must be set. @@ -75,9 +77,11 @@ COVERAGE=false K8S_115=false SIMULATOR=false CUSTOM_ADM_CONTROLLER=false +FLEXIBLE_IPAM=false HW_OFFLOAD=false SRIOV=false -FLEXIBLE_IPAM=false +SECONDARY_BRIDGE="" +PHYSICAL_INTERFACE="" MULTICAST=false MULTICAST_INTERFACES="" HELM_VALUES_FILES=() @@ -126,10 +130,6 @@ case $key in TUN_TYPE="$2" shift 2 ;; - --verbose-log) - VERBOSE_LOG=true - shift - ;; --on-delete) ON_DELETE=true shift @@ -146,6 +146,10 @@ case $key in CUSTOM_ADM_CONTROLLER=true shift ;; + --flexible-ipam) + FLEXIBLE_IPAM=true + shift + ;; --hw-offload) HW_OFFLOAD=true shift @@ -154,9 +158,13 @@ case $key in SRIOV=true shift ;; - --flexible-ipam) - FLEXIBLE_IPAM=true - shift + --secondary-bridge) + SECONDARY_BRIDGE="$2" + shift 2 + ;; + --physical-interface) + PHYSICAL_INTERFACE="$2" + shift 2 ;; --multicast) MULTICAST=true @@ -178,6 +186,10 @@ case $key in HELM_VALUES+=("$2") shift 2 ;; + --verbose-log) + VERBOSE_LOG=true + shift + ;; -h|--help) print_usage exit 0 @@ -322,6 +334,20 @@ EOF HELM_VALUES_FILES+=("$TMP_DIR/sriov.yml") fi +if [[ $SECONDARY_BRIDGE != "" ]]; then + if [[ $PHYSICAL_INTERFACE != "" ]]; then + ovs_bridges="[{bridgeName: $SECONDARY_BRIDGE, physicalInterfaces: [$PHYSICAL_INTERFACE]}]" + else + ovs_bridges="[{bridgeName: \"$SECONDARY_BRIDGE\"}]" + fi + cat << EOF > $TMP_DIR/secondary-network.yml +secondaryNetwork: + ovsBridges: $ovs_bridges +EOF + HELM_VALUES+=("featureGates.SecondaryNetwork=true" "featureGates.AntreaIPAM=true") + HELM_VALUES_FILES+=("$TMP_DIR/secondary-network.yml") +fi + if [ "$MODE" == "dev" ]; then if [[ -z "$AGENT_IMG_NAME" ]]; then if $COVERAGE; then diff --git a/test/e2e-secondary-network/framework.go b/test/e2e-secondary-network/framework.go deleted file mode 100644 index f87e789f575..00000000000 --- a/test/e2e-secondary-network/framework.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2022 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "log" - "time" - - antreae2e "antrea.io/antrea/test/e2e" -) - -type TestData struct { - e2eTestData *antreae2e.TestData - logsDirForTestCase string -} - -const ( - busyboxImage = "projects.registry.vmware.com/antrea/busybox" - defaultInterval = 1 * time.Second -) - -var testData *TestData - -type ClusterInfo struct { - controlPlaneNodeName string -} - -var clusterInfo ClusterInfo - -func (data *TestData) createClient(kubeconfigPath string) error { - e2edata = &antreae2e.TestData{} - if err := e2edata.CreateClient(kubeconfigPath); err != nil { - log.Fatalf("Error when creating K8s ClientSet: %v", err) - return err - } - data.e2eTestData = e2edata - return nil -} diff --git a/test/e2e-secondary-network/infra/prepare_cluster.sh b/test/e2e-secondary-network/infra/prepare_cluster.sh index 0c5d335b719..a5bc82dc8e3 100755 --- a/test/e2e-secondary-network/infra/prepare_cluster.sh +++ b/test/e2e-secondary-network/infra/prepare_cluster.sh @@ -18,14 +18,13 @@ THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" pushd $THIS_DIR _usage="Usage: $0 [--setup] [--cleanup] - [--network ] [--host-name ] [--help] + [--network ] [--host-name ] [--help] Deploy Antrea CNI and its native secondary network prerequisites on an existing/running K8s cluster control node. --setup Deploy all the pre-requisite plugins and CRDs for secondary network configuration. --cleanup Remove all the pre-requisite plugins and CRDs deployed for secondary network configuration. - --network sriov or veth (provide --network=sriov for sriov based secondary network configuration. - --network=veth for veth based secondary network configuration). + --network Secondary network type, can be sriov or vlan. --host-name K8s cluster control node's host name or IP address." _ssh_config_info="Prerequisite: Kubernetes(K8s) cluster is up and running at the local or remote server. @@ -64,12 +63,9 @@ function print_help { NAMESPACE="kube-system" ANTREA_DS="ds/antrea-agent" -WHEREABOUTS_CNI_REL_TAG=v0.6.1 -export URL_WHEREABOUTS_IP_POOLS="https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/$WHEREABOUTS_CNI_REL_TAG/doc/crds/whereabouts.cni.cncf.io_ippools.yaml" -export URL_WHEREABOUTS_OVERLAPPING_IP_RANGE_RES="https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/$WHEREABOUTS_CNI_REL_TAG/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml" export YAML_ANTREA='antrea.yml' export YAML_NET_ATTACH_DEF_CRD='network-attachment-definition-crd.yml' -export YAML_VIRTUAL_NET_INSTANCE='virtual-network-instance-crd.yml' +export YAML_SECONDARY_NETWORKS='secondary-networks.yml' action='' configure='' @@ -78,12 +74,12 @@ network='' function generateAntreaConfig() { genManifest=$THIS_DIR/../../../hack/generate-manifest.sh - echo $genManifest + echo $genManifest if [ "$1" == "sriov" ]; then - $genManifest --sriov --whereabouts --extra-helm-values "featureGates.SecondaryNetwork=true" > $YAML_ANTREA - elif [ "$1" == "veth" ]; then - $genManifest --whereabouts --extra-helm-values "featureGates.SecondaryNetwork=true" > $YAML_ANTREA - else + $genManifest --sriov --extra-helm-values "featureGates.SecondaryNetwork=true" > $YAML_ANTREA + elif [ "$1" == "vlan" ]; then + $genManifest --extra-helm-values "featureGates.SecondaryNetwork=true" > $YAML_ANTREAc + else echoerr "Incorrect network option $1. Failed to generate antrea.yml" exit 1 fi @@ -102,30 +98,13 @@ function AntreaCNI() { echoerr "Failed to $action Antrea CNI" exit 1 fi - # Setting timeout to report lack of progress after set time is elapsed + # Setting timeout to report lack of progress after set time is elapsed kubectl patch $ANTREA_DS -p '{"spec":{"progressDeadlineSeconds":30}}' -n $NAMESPACE kubectl -n $NAMESPACE rollout status $ANTREA_DS } -function WhereaboutsCNI() { - kubectl $action -f $URL_WHEREABOUTS_IP_POOLS - if [ $? -ne 0 ]; then - echoerr "Failed to $action Whereabouts CNI" - exit 1 - fi - echo "$action WhereaboutsCNI Done!" - - kubectl $action -f $URL_WHEREABOUTS_OVERLAPPING_IP_RANGE_RES - if [ $? -ne 0 ]; then - echoerr "Failed to $action Whereabouts overlapping IP Range reservation" - exit 1 - fi - echo "$action WhereaboutsOverlappingIpRangeRes Done!" -} - - function VirtualNetworks() { - kubectl $action -f $YAML_VIRTUAL_NET_INSTANCE + kubectl $action -f $YAML_SECONDARY_NETWORKS if [ $? -ne 0 ]; then echoerr "Failed to $action Virtual Network Instance: $instance" exit 1 @@ -147,7 +126,6 @@ function NetworkAttachmentDefinition() { function configureSecondaryNetworkPrerequisite() { NetworkAttachmentDefinition - WhereaboutsCNI AntreaCNI echo "Setup is up and running..." } @@ -181,10 +159,10 @@ while [[ $# -gt 0 ]]; do hostName="$2" shift 2 ;; - --network) - network=$2 - shift 2 - ;; + --network) + network=$2 + shift 2 + ;; --help) print_usage exit 0 @@ -198,7 +176,7 @@ done # Download all the prerequisite files. if [[ $configure == true ]]; then - # Generate antrea.yml with --sriov and --whereabouts config options. + # Generate antrea.yml with --sriov config option. generateAntreaConfig $network fi configureSecondaryNetworkPrerequisite diff --git a/test/e2e-secondary-network/infra/prepare_sriov.sh b/test/e2e-secondary-network/infra/prepare_sriov.sh index 18b8787faa5..df930780463 100755 --- a/test/e2e-secondary-network/infra/prepare_sriov.sh +++ b/test/e2e-secondary-network/infra/prepare_sriov.sh @@ -22,8 +22,8 @@ _usage="Usage: $0 [--setup] [--cleanup] Configure/cleanup SR-IOV VFs and deploy the SR-IOV device plugin at the remote single node cluster. --setup Configures the SR-IOV VFs and deploy the SR-IOV device plugin at the remote cluster (single node cluster). - --cleanup cleanup SR-IOV device plugin and VFs at the remote cluster (single node cluster). - --sriov-interface Base network interface (remote node scope) to create VFs. + --cleanup cleanup SR-IOV device plugin and VFs at the remote cluster (single node cluster). + --sriov-interface Base network interface (remote node scope) to create VFs. --sriov-vf-count Number of virtual functions to be created on the base network interface. --host-name K8s cluster's host name or IP address." @@ -63,7 +63,6 @@ function print_help { SRIOV_PLUGIN_REL_TAG=v3.5.1 -WHEREABOUTS_CNI_REL_TAG=v0.6.1 export URL_SRIOV_DP_CONFIG_MAP="https://raw.githubusercontent.com/k8snetworkplumbingwg/sriov-network-device-plugin/$SRIOV_PLUGIN_REL_TAG/deployments/configMap.yaml" export URL_SRIOV_DP_DAEMONSET="https://raw.githubusercontent.com/k8snetworkplumbingwg/sriov-network-device-plugin/$SRIOV_PLUGIN_REL_TAG/deployments/k8s-v1.16/sriovdp-daemonset.yaml" export YAML_ANTREA='antrea.yml' @@ -171,7 +170,7 @@ function updateKubeConfigForTargetCluster() { echoerr "Failed to copy the downloaded configuration files!" exit 1 fi - export KUBECONFIG=/tmp/kubernetes/$hostName/admin.conf + export KUBECONFIG=/tmp/kubernetes/$hostName/admin.conf } @@ -191,11 +190,11 @@ EOF fi SriovDevicePlugin fi - if [[ $configure == "true" ]]; then + if [[ $configure == "true" ]]; then echo "SR-IOV successfully configured at the remote server." - else - echo "SR-IOV successfully deconfigured at the remote server." - fi + else + echo "SR-IOV successfully deconfigured at the remote server." + fi } if [[ $# -eq 0 ]]; then diff --git a/test/e2e-secondary-network/infra/secondary-network-configuration.yml b/test/e2e-secondary-network/infra/secondary-network-configuration.yml deleted file mode 100755 index a0d9d43e3c7..00000000000 --- a/test/e2e-secondary-network/infra/secondary-network-configuration.yml +++ /dev/null @@ -1,14 +0,0 @@ -interface_type: - interfacetype: sriov -sriov_conf: - networkinterface: ens785f0 - - numberofvfs: 24 -vir_net: - totalnumberofvirtualnetworks: 3 - virtualnetworknames: ["virtual-network-instance1.yaml","virtual-network-instance2.yaml", "virtual-network-instance3.yaml"] -create_pod: - numberofpods: 3 - describe: [["testpodsec1",2,"virtual-net1,virtual-net2","eth1,eth2","sanity1.yaml"], - ["testpodsec2",2,"virtual-net1,virtual-net2","eth2,eth3","sanity2.yaml"], - ["testpodsec3",1,"virtual-net1","eth4","sanity3.yaml"]] diff --git a/test/e2e-secondary-network/infra/secondary-networks.yml b/test/e2e-secondary-network/infra/secondary-networks.yml new file mode 100644 index 00000000000..5495dd18394 --- /dev/null +++ b/test/e2e-secondary-network/infra/secondary-networks.yml @@ -0,0 +1,147 @@ +--- +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: secnet-ipv4-1 +spec: + ipVersion: 4 + ipRanges: + - cidr: "148.14.24.0/24" + gateway: "148.14.24.1" + prefixLength: 24 + +--- +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: secnet-ipv4-2 +spec: + ipVersion: 4 + ipRanges: + - start: "148.14.25.111" + end: "148.14.25.123" + gateway: "148.14.25.1" + prefixLength: 24 + +--- +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: secnet-ipv4-3 +spec: + ipVersion: 4 + ipRanges: + - cidr: "148.14.26.0/24" + gateway: "148.14.26.1" + prefixLength: 24 + vlan: 300 + +--- +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: secnet-ipv6-3 +spec: + ipVersion: 6 + ipRanges: + - cidr: "10:2400::0/96" + gateway: "10:2400::1" + prefixLength: 64 + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net1 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "sriov", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-1"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net2 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "sriov", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-2"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net3 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "sriov", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-3"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: vlan-net1 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "vlan", + "mtu": 1200, + "vlan": 100, + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-1"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: vlan-net2 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "vlan", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-2"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: vlan-net3 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "vlan", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-3", "secnet-ipv6-3"] + } + }' diff --git a/test/e2e-secondary-network/infra/virtual-network-instance-crd.yml b/test/e2e-secondary-network/infra/virtual-network-instance-crd.yml deleted file mode 100644 index f83affc9572..00000000000 --- a/test/e2e-secondary-network/infra/virtual-network-instance-crd.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: virtual-net1 -spec: - config: '{ - "cniVersion": "0.3.0", - "type": "antrea", - "networkType": "sriov", - "ipam": { - "type": "whereabouts", - "datastore": "kubernetes", - "kubernetes": { - "kubeconfig": "/host/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" - }, - "range": "148.14.24.0/24" - } - }' - ---- - -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: virtual-net2 -spec: - config: '{ - "cniVersion": "0.3.0", - "type": "antrea", - "networkType": "sriov", - "ipam": { - "type": "whereabouts", - "datastore": "kubernetes", - "kubernetes": { - "kubeconfig": "/host/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" - }, - "range": "148.14.25.0/24" - } - }' - ---- - -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: virtual-net3 -spec: - config: '{ - "cniVersion": "0.3.0", - "type": "antrea", - "networkType": "sriov", - "ipam": { - "type": "whereabouts", - "datastore": "kubernetes", - "kubernetes": { - "kubeconfig": "/host/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" - }, - "range": "148.14.26.0/24" - } - }' - ---- diff --git a/test/e2e-secondary-network/main_test.go b/test/e2e-secondary-network/main_test.go index 968513812b1..28b7b06697d 100644 --- a/test/e2e-secondary-network/main_test.go +++ b/test/e2e-secondary-network/main_test.go @@ -16,80 +16,15 @@ // instantiates and initializes objects imported from pkg, and runs // the process -package e2e +package e2esecondary import ( - "flag" - "log" "os" - "path" "testing" antreae2e "antrea.io/antrea/test/e2e" ) -type TestOptions struct { - logsExportDir string - enableAntreaIPAM bool - skipCases string - linuxVMs string -} - -var e2edata *antreae2e.TestData -var testOptions TestOptions -var homeDir, _ = os.UserHomeDir() - -// setupLogging creates a temporary directory to export the test logs if necessary. If a directory -// was provided by the user, it checks that the directory exists. -func (tOptions *TestOptions) setupLogging() func() { - if tOptions.logsExportDir == "" { - name, err := os.MkdirTemp("", "antrea-e2e-secondary-test-") - if err != nil { - log.Fatalf("Error when creating temporary directory to export logs: %v", err) - } - log.Printf("Test logs (if any) will be exported under the '%s' directory", name) - tOptions.logsExportDir = name - // we will delete the temporary directory if no logs are exported - return func() { - if empty, _ := antreae2e.IsDirEmpty(name); empty { - log.Printf("Removing empty logs directory '%s'", name) - _ = os.Remove(name) - } else { - log.Printf("Logs exported under '%s', it is your responsibility to delete the directory when you no longer need it", name) - } - } - } - fInfo, err := os.Stat(tOptions.logsExportDir) - if err != nil { - log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.logsExportDir, err) - } - if !fInfo.Mode().IsDir() { - log.Fatalf("'%s' is not a valid directory", tOptions.logsExportDir) - } - // no-op cleanup function - return func() {} -} - -func testMain(m *testing.M) int { - flag.StringVar(&testOptions.logsExportDir, "logs-export-dir", "", "Export directory for test logs") - flag.BoolVar(&testOptions.enableAntreaIPAM, "antrea-ipam", false, "Run tests with AntreaIPAM") - flag.StringVar(&testOptions.skipCases, "skip", "", "Key words to skip cases") - flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") - flag.Parse() - - cleanupLogging := testOptions.setupLogging() - defer cleanupLogging() - - testData = &TestData{} - log.Println("Creating K8s ClientSet") - kubeconfigPath := path.Join(homeDir, ".kube", "secondary_network_cluster", "config") - if err := testData.createClient(kubeconfigPath); err != nil { - log.Fatalf("Error when creating K8s ClientSet: %v", err) - } - ret := m.Run() - return ret -} - func TestMain(m *testing.M) { - os.Exit(testMain(m)) + os.Exit(antreae2e.RunTests(m)) } diff --git a/test/e2e-secondary-network/secondary_network_test.go b/test/e2e-secondary-network/secondary_network_test.go index 97c61152088..ac6ebe66f13 100644 --- a/test/e2e-secondary-network/secondary_network_test.go +++ b/test/e2e-secondary-network/secondary_network_test.go @@ -12,123 +12,62 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package e2esecondary import ( "fmt" - "log" "net" - "os" "strings" "testing" "time" logs "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" antreae2e "antrea.io/antrea/test/e2e" ) -// Structure to extract and store the secondary network configuration information parsed from secondary-network-configuration.yml. -type PodConfig struct { - InterfaceType struct { - interfaceType string `yaml:"interfacetype"` - } `yaml:"interface_type"` - SriovConf struct { - networkInterface string `yaml:"networkinterface"` - numberOfVfs int `yaml:"numberofvfs"` - } `yaml:"sriov_conf"` - VirNet struct { - totalNumberOfVirtualNetworks int `yaml:"totalnumberofvirtualnetworks"` - virtualNetworknames []string `yaml:"virtualnetworknames"` - } `yaml:"vir_net"` - CreatePod struct { - numberOfPods int `yaml:"numberofpods"` - describe [][]interface{} `yaml:"describe"` - } `yaml:"create_pod"` +type testPodInfo struct { + podName string + nodeName string + // map from interface name to secondary network name. + interfaceNetworks map[string]string } -var service PodConfig - -// Structure for extracting the variables for describing the Pod from secondary-network-configuration.yml file. -type describePodInfo struct { - nameOfPods string - countOfVirtualNetworksPerPod int - nameOfVirtualNetworkPerPod []string - nameOfInterfacePerPod []string +type testData struct { + e2eTestData *antreae2e.TestData + networkType string + pods []*testPodInfo } -var podData []describePodInfo -var totalNumberOfPods int -var interfaceType string - const ( - secondaryNetworkConfigYAML = "./infra/secondary-network-configuration.yml" - nameSpace = "kube-system" - ctrName = "busyboxpod" - testPodName = "testsecpod" - osType = "linux" - count = 5 - size = 40 - defaultTimeout = 10 * time.Second - reqName = "intel.com/intel_sriov_netdevice" - resNum = 3 + networkTypeSriov = "sriov" + networkTypeVLAN = "vlan" + + // Namespace of NetworkAttachmentDefinition CRs. + attachDefNamespace = "default" + + containerName = "busybox" + podApp = "secondaryTest" + osType = "linux" + pingCount = 5 + pingSize = 40 + defaultTimeout = 10 * time.Second + sriovReqName = "intel.com/intel_sriov_netdevice" + sriovResNum = 3 ) -const ( - podName = iota - podVNsCount - podVirtualNetwork - podInterfaceName -) - -// setupTestWithSecondaryNetworkConfig sets up all the prerequisites for running the test including the antrea enabled and running, extracting Pod and secondary network interface information and setting log directory for the test -func (data *TestData) setupTestWithSecondaryNetworkConfig(tb testing.TB) (*TestData, error) { - // Extracting the Pods information from the secondary_network_configuration.yml file. - if err := data.extractPodsInfo(); err != nil { - tb.Errorf("Error in extracting Pods info from secondary-network-configuration.yml : %v", err) - return nil, err - } - // Set log directory for test execution. - if err := data.e2eTestData.SetupLogDirectoryForTest(tb.Name()); err != nil { - tb.Errorf("Error creating logs directory '%s': %v", data.logsDirForTestCase, err) - return nil, err - } - return data, nil -} - -// extractPodsInfo extracts the Pod and secondary network interface information for the creation of Podsfrom secondary-network-configuration.yaml file -func (data *TestData) extractPodsInfo() error { - var errYamlUnmarshal error - _, err := os.Stat(secondaryNetworkConfigYAML) - if err != nil { - return fmt.Errorf("Parsing of the Pod configuration file failed") - - } - secondaryNetworkConfigYAML, _ := os.ReadFile(secondaryNetworkConfigYAML) - errYamlUnmarshal = yaml.Unmarshal(secondaryNetworkConfigYAML, &service) - if errYamlUnmarshal != nil { - return fmt.Errorf("Parsing %s failed", secondaryNetworkConfigYAML) - } - interfaceType = service.InterfaceType.interfaceType - totalNumberOfPods = service.CreatePod.numberOfPods - for _, s := range service.CreatePod.describe { - output := describePodInfo{nameOfPods: s[podName].(string), countOfVirtualNetworksPerPod: s[podVNsCount].(int), nameOfVirtualNetworkPerPod: strings.Split(s[podVirtualNetwork].(string), ","), nameOfInterfacePerPod: strings.Split(s[podInterfaceName].(string), ",")} - podData = append(podData, output) - } - return nil -} // formAnnotationStringOfPod forms the annotation string, used in the generation of each Pod YAML file. -func (data *TestData) formAnnotationStringOfPod(pod int) string { +func (data *testData) formAnnotationStringOfPod(pod *testPodInfo) string { var annotationString = "" - for xPodVN := 0; xPodVN < podData[pod].countOfVirtualNetworksPerPod; xPodVN++ { - var podNetworkSpec = "{\"name\": \"" + podData[pod].nameOfVirtualNetworkPerPod[xPodVN] + "\" ,\"interface\": \"" + podData[pod].nameOfInterfacePerPod[xPodVN] + "\" , \"type\": \"" + interfaceType + "\"}" + for i, n := range pod.interfaceNetworks { + podNetworkSpec := fmt.Sprintf("{\"name\": \"%s\", \"namespace\": \"%s\", \"interface\": \"%s\"}", + n, attachDefNamespace, i) if annotationString == "" { annotationString = "[" + podNetworkSpec } else { - annotationString = annotationString + "," + podNetworkSpec + annotationString = annotationString + ", " + podNetworkSpec } } annotationString = annotationString + "]" @@ -136,77 +75,100 @@ func (data *TestData) formAnnotationStringOfPod(pod int) string { } // createPodOnNode creates the Pod for the specific annotations as per the parsed Pod information using the NewPodBuilder API -func (data *TestData) createPodOnNode(t *testing.T, ns string, nodeName string) error { +func (data *testData) createPods(t *testing.T, ns string) error { var err error - for xPod := 0; xPod < totalNumberOfPods; xPod++ { - err := data.createPodForSecondaryNetwork(ns, nodeName, xPod, testPodName, resNum) + for _, pod := range data.pods { + err := data.createPodForSecondaryNetwork(ns, pod) if err != nil { - return fmt.Errorf("Error in creating pods.., err: %v", err) + return fmt.Errorf("error in creating pods.., err: %v", err) } } return err } -// getSecondaryInterface shows up the secondary interfaces created for the specific Pod and extracts the IP address for the same. -func (data *TestData) getSecondaryInterface(targetPod int, targetInterface int) (string, error) { - cmd := []string{"/bin/sh", "-c", fmt.Sprintf("ip addr show %s | grep \"inet\" | awk '{print $2}' | cut -d/ -f1", podData[targetPod].nameOfInterfacePerPod[targetInterface])} - stdout, _, err := data.e2eTestData.RunCommandFromPod(nameSpace, podData[targetPod].nameOfPods, ctrName, cmd) +// The Wrapper function createPodForSecondaryNetwork creates the Pod adding the annotation, arguments, commands, Node, container name, +// resource requests and limits as arguments with the NewPodBuilder API +func (data *testData) createPodForSecondaryNetwork(ns string, pod *testPodInfo) error { + podBuilder := antreae2e.NewPodBuilder(pod.podName, ns, antreae2e.ToolboxImage). + OnNode(pod.nodeName).WithContainerName(containerName). + WithAnnotations(map[string]string{ + "k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%s", data.formAnnotationStringOfPod(pod)), + }). + WithLabels(map[string]string{ + "App": fmt.Sprintf("%s", podApp), + }) + + if data.networkType == networkTypeSriov { + computeResources := resource.NewQuantity(sriovResNum, resource.DecimalSI) + podBuilder = podBuilder.WithResources(corev1.ResourceList{sriovReqName: *computeResources}, corev1.ResourceList{sriovReqName: *computeResources}) + } + return podBuilder.Create(data.e2eTestData) +} + +// getSecondaryInterface checks the secondary interfaces created for the specific Pod and returns its IPv4 address. +func (data *testData) getSecondaryInterface(targetPod *testPodInfo, interfaceName string) (string, error) { + cmd := []string{"/bin/sh", "-c", fmt.Sprintf("ip addr show %s | grep 'inet ' | awk '{print $2}' | cut -d/ -f1", interfaceName)} + stdout, _, err := data.e2eTestData.RunCommandFromPod(data.e2eTestData.GetTestNamespace(), targetPod.podName, containerName, cmd) stdout = strings.TrimSuffix(stdout, "\n") - if stdout == "" { - log.Fatalf("Error: Interface %s not found on %s. err: %v", podData[targetPod].nameOfInterfacePerPod[targetInterface], podData[targetPod].nameOfPods, err) + if err != nil || stdout == "" { + return "", fmt.Errorf("interface %s not found on %s. err: %v", interfaceName, targetPod.podName, err) } return stdout, nil } // checkSubnet checks if the IP address to be pinged has the same subnet as the Pod from which the IP Address is pinged. -func (data *TestData) checkSubnet(t *testing.T, sourcePod int, targetPod int, targetInterface int) (bool, error) { - for podCheckForSubnet := 0; podCheckForSubnet < podData[sourcePod].countOfVirtualNetworksPerPod; podCheckForSubnet++ { - if podData[sourcePod].nameOfVirtualNetworkPerPod[podCheckForSubnet] == podData[targetPod].nameOfVirtualNetworkPerPod[targetInterface] { - _, err := data.getSecondaryInterface(sourcePod, podCheckForSubnet) +func (data *testData) checkSubnet(t *testing.T, sourcePod, targetPod *testPodInfo, targetNetwork string) (bool, error) { + for i, n := range sourcePod.interfaceNetworks { + if n == targetNetwork { + _, err := data.getSecondaryInterface(sourcePod, i) if err != nil { - t.Logf("Error in ping: Interface %s for the source test Pod %s not created", podData[sourcePod].nameOfInterfacePerPod[podCheckForSubnet], podData[sourcePod].nameOfPods) return false, err } + return true, nil } } - return true, nil + return false, nil } -// pingBetweenInterfaces parses through all the created Podsand pings the other Pod if the IP Address of the secondary network interface of the Pod is in the same subnet. Sleep time of 3 seconds is ensured for the successful ping between the pods. -func (data *TestData) pingBetweenInterfaces(t *testing.T) error { - for sourcePod := 0; sourcePod < totalNumberOfPods; sourcePod++ { - for targetPod := 0; targetPod < totalNumberOfPods; targetPod++ { - for targetInterface := 0; targetInterface < podData[targetPod].countOfVirtualNetworksPerPod; targetInterface++ { - if podData[targetPod].nameOfPods == podData[sourcePod].nameOfPods { - continue - } - _, err := data.e2eTestData.PodWaitFor(defaultTimeout, podData[targetPod].nameOfPods, nameSpace, func(pod *corev1.Pod) (bool, error) { +// pingBetweenInterfaces parses through all the created Pods and pings the other Pod if the two Pods +// both have a secondary network interface on the same network. +func (data *testData) pingBetweenInterfaces(t *testing.T) error { + e2eTestData := data.e2eTestData + namespace := e2eTestData.GetTestNamespace() + for _, sourcePod := range data.pods { + for _, targetPod := range data.pods { + if targetPod.podName == sourcePod.podName { + continue + } + for i, n := range targetPod.interfaceNetworks { + _, err := e2eTestData.PodWaitFor(defaultTimeout, targetPod.podName, namespace, func(pod *corev1.Pod) (bool, error) { return pod.Status.Phase == corev1.PodRunning, nil }) if err != nil { - t.Logf("Error when waiting for the perftest client Pod: %s", podData[targetPod].nameOfPods) + return fmt.Errorf("error when waiting for Pod %s: %v", targetPod.podName, err) } - flag, _ := data.checkSubnet(t, sourcePod, targetPod, targetInterface) - if flag != false { - secondaryIpAddress, _ := data.getSecondaryInterface(targetPod, targetInterface) - ip := net.ParseIP(secondaryIpAddress) - if ip != nil { - var IPToPing antreae2e.PodIPs - if ip.To4() != nil { - IPToPing = antreae2e.PodIPs{IPv4: &ip} - } else { - IPToPing = antreae2e.PodIPs{IPv6: &ip} - } - err := data.e2eTestData.RunPingCommandFromTestPod(antreae2e.PodInfo{Name: podData[sourcePod].nameOfPods, OS: osType, NodeName: clusterInfo.controlPlaneNodeName, Namespace: nameSpace}, nameSpace, &IPToPing, ctrName, count, size, false) - if err == nil { - logs.Infof("Ping '%s' -> '%s'( Interface: %s, IP Address: %s): OK", podData[sourcePod].nameOfPods, podData[targetPod].nameOfPods, podData[targetPod].nameOfInterfacePerPod[targetInterface], secondaryIpAddress) - } else { - t.Logf("Ping '%s' -> '%s'( Interface: %s, IP Address: %s): ERROR (%v)", podData[sourcePod].nameOfPods, podData[targetPod].nameOfPods, podData[targetPod].nameOfInterfacePerPod[targetInterface], secondaryIpAddress, err) - } + matched, _ := data.checkSubnet(t, sourcePod, targetPod, n) + if matched { + secondaryIPAddress, err := data.getSecondaryInterface(targetPod, i) + if err != nil { + return err + } + ip := net.ParseIP(secondaryIPAddress) + if ip == nil { + return fmt.Errorf("failed to parse IP (%s) for interface %s of Pod %s", secondaryIPAddress, i, targetPod.podName) + } + var IPToPing antreae2e.PodIPs + if ip.To4() != nil { + IPToPing = antreae2e.PodIPs{IPv4: &ip} } else { - t.Logf("Error in Ping: Target interface %v of %v Pod not created", podData[targetPod].nameOfInterfacePerPod[targetInterface], podData[targetPod].nameOfPods) + IPToPing = antreae2e.PodIPs{IPv6: &ip} + } + if err := e2eTestData.RunPingCommandFromTestPod(antreae2e.PodInfo{Name: sourcePod.podName, OS: osType, NodeName: sourcePod.nodeName, Namespace: namespace}, + namespace, &IPToPing, containerName, pingCount, pingSize, false); err != nil { + return fmt.Errorf("ping '%s' -> '%s'(Interface: %s, IP Address: %s) failed: %v", sourcePod.podName, targetPod.podName, i, secondaryIPAddress, err) } + logs.Infof("ping '%s' -> '%s'( Interface: %s, IP Address: %s): OK", sourcePod.podName, targetPod.podName, i, secondaryIPAddress) } } } @@ -214,32 +176,69 @@ func (data *TestData) pingBetweenInterfaces(t *testing.T) error { return nil } -// The Wrapper function createPodForSecondaryNetwork creates the Pod adding the annotation, arguments, commands, Node, container name, -// resource requests and limits as arguments with the NewPodBuilder API -func (data *TestData) createPodForSecondaryNetwork(ns string, nodeName string, podNum int, testPodName string, resNum int64) error { - computeResources := resource.NewQuantity(resNum, resource.DecimalSI) - return antreae2e.NewPodBuilder(podData[podNum].nameOfPods, ns, busyboxImage).OnNode(nodeName).WithContainerName(ctrName).WithCommand([]string{"sleep", "infinity"}).WithAnnotations( - map[string]string{ - "k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%s", data.formAnnotationStringOfPod(podNum)), - }).WithLabels( - map[string]string{ - "App": fmt.Sprintf("%s", testPodName), - }).WithResources(corev1.ResourceList{reqName: *computeResources}, corev1.ResourceList{reqName: *computeResources}).Create(data.e2eTestData) -} - -func TestNativeSecondaryNetwork(t *testing.T) { - // once the setupTestWithSecondaryNetworkConfig is successful, we have all the prerequisites enabled and running. - _, err := testData.setupTestWithSecondaryNetworkConfig(t) +func testSecondaryNetwork(t *testing.T, networkType string, pods []*testPodInfo) { + e2eTestData, err := antreae2e.SetupTest(t) if err != nil { - t.Logf("Error when setupTestWithSecondaryNetworkConfig: %v", err) + t.Fatalf("Error when setting up test: %v", err) } + defer antreae2e.TeardownTest(t, e2eTestData) + + testData := &testData{e2eTestData: e2eTestData, networkType: networkType, pods: pods} + t.Run("testCreateTestPodOnNode", func(t *testing.T) { - testData.createPodOnNode(t, nameSpace, clusterInfo.controlPlaneNodeName) + testData.createPods(t, e2eTestData.GetTestNamespace()) }) t.Run("testpingBetweenInterfaces", func(t *testing.T) { err := testData.pingBetweenInterfaces(t) if err != nil { - t.Logf("Error when pinging between interfaces: %v", err) + t.Fatalf("Error when pinging between interfaces: %v", err) } }) } + +func TestSriovNetwork(t *testing.T) { + // Create Pods on the control plane Node, assuming a single Node cluster for the SR-IOV + // test. + controlPlaneNodeName := antreae2e.NodeName(0) + pods := []*testPodInfo{ + { + podName: "sriov-pod1", + nodeName: controlPlaneNodeName, + interfaceNetworks: map[string]string{"eth1": "sriov-net1", "eth2": "sriov-net2"}, + }, + { + podName: "sriov-pod2", + nodeName: controlPlaneNodeName, + interfaceNetworks: map[string]string{"eth2": "sriov-net1", "eth3": "sriov-net3"}, + }, + { + podName: "sriov-pod3", + nodeName: controlPlaneNodeName, + interfaceNetworks: map[string]string{"eth4": "sriov-net1"}, + }, + } + testSecondaryNetwork(t, networkTypeSriov, pods) +} + +func TestVLANNetwork(t *testing.T) { + node1 := antreae2e.NodeName(1) + node2 := antreae2e.NodeName(2) + pods := []*testPodInfo{ + { + podName: "vlan-pod1", + nodeName: node1, + interfaceNetworks: map[string]string{"eth1": "vlan-net1", "eth2": "vlan-net2"}, + }, + { + podName: "vlan-pod2", + nodeName: node1, + interfaceNetworks: map[string]string{"eth1": "vlan-net1", "eth2": "vlan-net3"}, + }, + { + podName: "vlan-pod3", + nodeName: node2, + interfaceNetworks: map[string]string{"eth1": "vlan-net2"}, + }, + } + testSecondaryNetwork(t, networkTypeVLAN, pods) +} diff --git a/test/e2e/antctl_test.go b/test/e2e/antctl_test.go index ae7936c0f10..11d059797d2 100644 --- a/test/e2e/antctl_test.go +++ b/test/e2e/antctl_test.go @@ -313,7 +313,7 @@ func testAntctlProxy(t *testing.T, data *TestData, antctlServiceAccountName stri const proxyContainerName = "proxy" const proxyPort = 8001 - require.NoError(t, NewPodBuilder(testPodName, data.testNamespace, toolboxImage).WithContainerName(testContainerName).OnNode(controlPlaneNodeName()).InHostNetwork().Create(data)) + require.NoError(t, NewPodBuilder(testPodName, data.testNamespace, ToolboxImage).WithContainerName(testContainerName).OnNode(controlPlaneNodeName()).InHostNetwork().Create(data)) defer data.DeletePodAndWait(defaultTimeout, testPodName, data.testNamespace) require.NoError(t, data.podWaitForRunning(defaultTimeout, testPodName, data.testNamespace), "test Pod not in the Running state") diff --git a/test/e2e/bandwidth_test.go b/test/e2e/bandwidth_test.go index 5e3f1b738d4..cfe70a2d9ac 100644 --- a/test/e2e/bandwidth_test.go +++ b/test/e2e/bandwidth_test.go @@ -66,14 +66,14 @@ func TestBenchmarkBandwidth(t *testing.T) { // testBenchmarkBandwidthIntraNode runs the bandwidth benchmark between Pods on same node. func testBenchmarkBandwidthIntraNode(t *testing.T, data *TestData) { - if err := NewPodBuilder("perftest-a", data.testNamespace, toolboxImage).OnNode(controlPlaneNodeName()).Create(data); err != nil { + if err := NewPodBuilder("perftest-a", data.testNamespace, ToolboxImage).OnNode(controlPlaneNodeName()).Create(data); err != nil { t.Fatalf("Error when creating the perftest client Pod: %v", err) } if err := data.podWaitForRunning(defaultTimeout, "perftest-a", data.testNamespace); err != nil { t.Fatalf("Error when waiting for the perftest client Pod: %v", err) } cmd := []string{"iperf3", "-s"} - if err := NewPodBuilder("perftest-b", data.testNamespace, toolboxImage).WithCommand(cmd).OnNode(controlPlaneNodeName()).WithPorts([]v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}).Create(data); err != nil { + if err := NewPodBuilder("perftest-b", data.testNamespace, ToolboxImage).WithCommand(cmd).OnNode(controlPlaneNodeName()).WithPorts([]v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}).Create(data); err != nil { t.Fatalf("Error when creating the perftest server Pod: %v", err) } podBIPs, err := data.podWaitForIPs(defaultTimeout, "perftest-b", data.testNamespace) @@ -94,14 +94,14 @@ func benchmarkBandwidthService(t *testing.T, endpointNode, clientNode string, da if err != nil { t.Fatalf("Error when creating perftest service: %v", err) } - if err := NewPodBuilder("perftest-a", data.testNamespace, toolboxImage).OnNode(clientNode).Create(data); err != nil { + if err := NewPodBuilder("perftest-a", data.testNamespace, ToolboxImage).OnNode(clientNode).Create(data); err != nil { t.Fatalf("Error when creating the perftest client Pod: %v", err) } if err := data.podWaitForRunning(defaultTimeout, "perftest-a", data.testNamespace); err != nil { t.Fatalf("Error when waiting for the perftest client Pod: %v", err) } cmd := []string{"iperf3", "-s"} - if err := NewPodBuilder("perftest-b", data.testNamespace, toolboxImage).WithCommand(cmd).OnNode(clientNode).WithPorts([]v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}).Create(data); err != nil { + if err := NewPodBuilder("perftest-b", data.testNamespace, ToolboxImage).WithCommand(cmd).OnNode(clientNode).WithPorts([]v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}).Create(data); err != nil { t.Fatalf("Error when creating the perftest server Pod: %v", err) } if err := data.podWaitForRunning(defaultTimeout, "perftest-b", data.testNamespace); err != nil { @@ -158,7 +158,7 @@ func testPodTrafficShaping(t *testing.T, data *TestData) { t.Run(tt.name, func(t *testing.T) { clientPodName := fmt.Sprintf("client-a-%d", i) serverPodName := fmt.Sprintf("server-a-%d", i) - if err := NewPodBuilder(clientPodName, data.testNamespace, toolboxImage).OnNode(nodeName).WithAnnotations( + if err := NewPodBuilder(clientPodName, data.testNamespace, ToolboxImage).OnNode(nodeName).WithAnnotations( map[string]string{ "kubernetes.io/egress-bandwidth": fmt.Sprintf("%dM", tt.clientEgressBandwidth), }, @@ -170,7 +170,7 @@ func testPodTrafficShaping(t *testing.T, data *TestData) { t.Fatalf("Error when waiting for the perftest client Pod: %v", err) } cmd := []string{"iperf3", "-s"} - if err := NewPodBuilder(serverPodName, data.testNamespace, toolboxImage).WithCommand(cmd).OnNode(nodeName).WithPorts([]v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}).WithAnnotations( + if err := NewPodBuilder(serverPodName, data.testNamespace, ToolboxImage).WithCommand(cmd).OnNode(nodeName).WithPorts([]v1.ContainerPort{{Protocol: v1.ProtocolTCP, ContainerPort: iperfPort}}).WithAnnotations( map[string]string{ "kubernetes.io/ingress-bandwidth": fmt.Sprintf("%dM", tt.serverIngressBandwidth), }, diff --git a/test/e2e/connectivity_test.go b/test/e2e/connectivity_test.go index f6e64ae7013..a9a7d0ddd9f 100644 --- a/test/e2e/connectivity_test.go +++ b/test/e2e/connectivity_test.go @@ -193,7 +193,7 @@ func testHostPortPodConnectivity(t *testing.T, data *TestData) { // alternating in this podInfo slice so that the test can cover different connectivity cases between different OSes. func createPodsOnDifferentNodes(t *testing.T, data *TestData, namespace, tag string) (podInfos []PodInfo, cleanup func() error) { dsName := "connectivity-test" + tag - _, deleteDaemonSet, err := data.createDaemonSet(dsName, namespace, toolboxContainerName, toolboxImage, []string{"sleep", "3600"}, nil) + _, deleteDaemonSet, err := data.createDaemonSet(dsName, namespace, toolboxContainerName, ToolboxImage, []string{"sleep", "3600"}, nil) if err != nil { t.Fatalf("Error when creating DaemonSet '%s': %v", dsName, err) } diff --git a/test/e2e/egress_test.go b/test/e2e/egress_test.go index 07171833c79..4a3145615b3 100644 --- a/test/e2e/egress_test.go +++ b/test/e2e/egress_test.go @@ -797,14 +797,14 @@ func testEgressUpdateBandwidth(t *testing.T, data *TestData) { fakeExternalCmd := "iperf3 -s" cmd, _ := getCommandInFakeExternalNetwork(fakeExternalCmd, 24, "1.1.1.1", "1.1.1.254") - err := NewPodBuilder(fakeExternalName, data.testNamespace, toolboxImage).OnNode(egressNode).WithCommand([]string{"bash", "-c", cmd}).InHostNetwork().Privileged().Create(data) + err := NewPodBuilder(fakeExternalName, data.testNamespace, ToolboxImage).OnNode(egressNode).WithCommand([]string{"bash", "-c", cmd}).InHostNetwork().Privileged().Create(data) require.NoError(t, err, "Failed to create fake external Pod") defer deletePodWrapper(t, data, data.testNamespace, fakeExternalName) err = data.podWaitForRunning(defaultTimeout, fakeExternalName, data.testNamespace) require.NoError(t, err, "Error when waiting for fake external Pod to be in the Running state") clientPodName := "client-pod" - err = NewPodBuilder(clientPodName, data.testNamespace, toolboxImage).OnNode(egressNode).Create(data) + err = NewPodBuilder(clientPodName, data.testNamespace, ToolboxImage).OnNode(egressNode).Create(data) require.NoError(t, err, "Failed to create client Pod") defer deletePodWrapper(t, data, data.testNamespace, clientPodName) err = data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace) diff --git a/test/e2e/entry.go b/test/e2e/entry.go new file mode 100644 index 00000000000..c7fa2531495 --- /dev/null +++ b/test/e2e/entry.go @@ -0,0 +1,192 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "flag" + "fmt" + "log" + "os" + "testing" +) + +// setupLogging creates a temporary directory to export the test logs if necessary. If a directory +// was provided by the user, it checks that the directory exists. +func (tOptions *TestOptions) setupLogging() func() { + if tOptions.logsExportDir == "" { + name, err := os.MkdirTemp("", "antrea-test-") + if err != nil { + log.Fatalf("Error when creating temporary directory to export logs: %v", err) + } + log.Printf("Test logs (if any) will be exported under the '%s' directory", name) + tOptions.logsExportDir = name + // we will delete the temporary directory if no logs are exported + return func() { + if empty, _ := IsDirEmpty(name); empty { + log.Printf("Removing empty logs directory '%s'", name) + _ = os.Remove(name) + } else { + log.Printf("Logs exported under '%s', it is your responsibility to delete the directory when you no longer need it", name) + } + } + } + fInfo, err := os.Stat(tOptions.logsExportDir) + if err != nil { + log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.logsExportDir, err) + } + if !fInfo.Mode().IsDir() { + log.Fatalf("'%s' is not a valid directory", tOptions.logsExportDir) + } + // no-op cleanup function + return func() {} +} + +// setupCoverage checks if the directory provided by the user exists. +func (tOptions *TestOptions) setupCoverage(data *TestData) func() { + if tOptions.coverageDir != "" { + fInfo, err := os.Stat(tOptions.coverageDir) + if err != nil { + log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.coverageDir, err) + } + if !fInfo.Mode().IsDir() { + log.Fatalf("'%s' is not a valid directory", tOptions.coverageDir) + } + + } + // cpNodeCoverageDir is a directory on the control-plane Node, where tests can deposit test + // coverage data. + log.Printf("Creating directory '%s' on Node '%s'\n", cpNodeCoverageDir, controlPlaneNodeName()) + rc, _, _, err := data.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("mkdir -p %s", cpNodeCoverageDir)) + if err != nil || rc != 0 { + log.Fatalf("Failed to create directory '%s' on control-plane Node", cpNodeCoverageDir) + } + return func() { + log.Printf("Removing directory '%s' on Node '%s'\n", cpNodeCoverageDir, controlPlaneNodeName()) + // best effort + data.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("rm -rf %s", cpNodeCoverageDir)) + } + +} + +// testMain is meant to be called by TestMain and enables the use of defer statements. +func testMain(m *testing.M) int { + flag.StringVar(&testOptions.providerName, "provider", "vagrant", "K8s test cluster provider") + flag.StringVar(&testOptions.providerConfigPath, "provider-cfg-path", "", "Optional config file for provider") + flag.StringVar(&testOptions.logsExportDir, "logs-export-dir", "", "Export directory for test logs") + flag.BoolVar(&testOptions.logsExportOnSuccess, "logs-export-on-success", false, "Export logs even when a test is successful") + flag.BoolVar(&testOptions.withBench, "benchtest", false, "Run tests include benchmark tests") + flag.BoolVar(&testOptions.enableCoverage, "coverage", false, "Run tests and measure coverage") + flag.BoolVar(&testOptions.enableAntreaIPAM, "antrea-ipam", false, "Run tests with AntreaIPAM") + flag.BoolVar(&testOptions.flowVisibility, "flow-visibility", false, "Run flow visibility tests") + flag.BoolVar(&testOptions.deployAntrea, "deploy-antrea", true, "Deploy Antrea before running tests") + flag.StringVar(&testOptions.coverageDir, "coverage-dir", "", "Directory for coverage data files") + flag.StringVar(&testOptions.skipCases, "skip-cases", "", "Key words to skip cases") + flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") + flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs") + flag.StringVar(&testOptions.externalServerIPs, "external-server-ips", "", "IP addresses of external server, at most one IP per IP family") + flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family") + flag.IntVar(&testOptions.vlanID, "vlan-id", 0, "ID of the VLAN network the Nodes reside in") + flag.Parse() + + cleanupLogging := testOptions.setupLogging() + defer cleanupLogging() + + testData = &TestData{} + if err := testData.InitProvider(testOptions.providerName, testOptions.providerConfigPath); err != nil { + log.Fatalf("Error when initializing provider: %v", err) + } + log.Println("Creating K8s ClientSet") + kubeconfigPath, err := testData.provider.GetKubeconfigPath() + if err != nil { + log.Fatalf("Error when getting Kubeconfig path: %v", err) + } + if err := testData.CreateClient(kubeconfigPath); err != nil { + log.Fatalf("Error when creating K8s ClientSet: %v", err) + } + log.Println("Collecting information about K8s cluster") + if err := testData.collectClusterInfo(); err != nil { + log.Fatalf("Error when collecting information about K8s cluster: %v", err) + } + if clusterInfo.podV4NetworkCIDR != "" { + log.Printf("Pod IPv4 network: '%s'", clusterInfo.podV4NetworkCIDR) + } + if clusterInfo.podV6NetworkCIDR != "" { + log.Printf("Pod IPv6 network: '%s'", clusterInfo.podV6NetworkCIDR) + } + if clusterInfo.svcV4NetworkCIDR != "" { + log.Printf("Service IPv4 network: '%s'", clusterInfo.svcV4NetworkCIDR) + } + if clusterInfo.svcV6NetworkCIDR != "" { + log.Printf("Service IPv6 network: '%s'", clusterInfo.svcV6NetworkCIDR) + } + log.Printf("Num nodes: %d", clusterInfo.numNodes) + if err := testData.collectExternalInfo(); err != nil { + log.Fatalf("Error when collecting external information: %v", err) + } + err = ensureAntreaRunning(testData) + if err != nil { + log.Fatalf("Error when deploying Antrea: %v", err) + } + // Collect PodCIDRs after Antrea is running as Antrea is responsible for allocating PodCIDRs in some cases. + // Polling is not needed here because antrea-agents won't be up and running if PodCIDRs of their Nodes are not set. + if err = testData.collectPodCIDRs(); err != nil { + log.Fatalf("Error collecting PodCIDRs: %v", err) + } + AntreaConfigMap, err = testData.GetAntreaConfigMap(antreaNamespace) + if err != nil { + log.Fatalf("Error when getting antrea-config configmap: %v", err) + } + if testOptions.enableCoverage { + cleanupCoverage := testOptions.setupCoverage(testData) + defer cleanupCoverage() + defer gracefulExitAntrea(testData) + } + ret := m.Run() + return ret +} + +func gracefulExitAntrea(testData *TestData) { + if err := testData.gracefulExitAntreaController(testOptions.coverageDir); err != nil { + log.Fatalf("Error when gracefully exit antrea controller: %v", err) + } + if err := testData.gracefulExitAntreaAgent(testOptions.coverageDir, "all"); err != nil { + log.Fatalf("Error when gracefully exit antrea agent: %v", err) + } + if err := testData.collectAntctlCovFilesFromControlPlaneNode(testOptions.coverageDir); err != nil { + log.Fatalf("Error when collecting antctl coverage files from control-plane Node: %v", err) + } +} + +// The following funcs are used in e2e-secondary-network. + +func RunTests(m *testing.M) int { + return testMain(m) +} + +func SetupTest(tb testing.TB) (*TestData, error) { + return setupTest(tb) +} + +func TeardownTest(tb testing.TB, data *TestData) { + teardownTest(tb, data) +} + +func NodeName(idx int) string { + return nodeName(idx) +} + +func (data *TestData) GetTestNamespace() string { + return data.testNamespace +} diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 45af883bccd..9a699e1cd32 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -253,10 +253,10 @@ func setupTest(tb testing.TB) (*TestData, error) { name = replacer.Replace(name) name = strings.ToLower(name) testData.testNamespace = randName(name + "-") - tb.Logf("Creating '%s' K8s Namespace", testData.testNamespace) if err := ensureAntreaRunning(testData); err != nil { return nil, err } + tb.Logf("Creating '%s' K8s Namespace", testData.testNamespace) if err := testData.CreateNamespace(testData.testNamespace, nil); err != nil { return nil, err } diff --git a/test/e2e/flowaggregator_test.go b/test/e2e/flowaggregator_test.go index e51ae4f27b4..e4c4e6e5a59 100644 --- a/test/e2e/flowaggregator_test.go +++ b/test/e2e/flowaggregator_test.go @@ -1732,7 +1732,7 @@ func deployDenyNetworkPolicies(t *testing.T, data *TestData, pod1, pod2 string, func createPerftestPods(data *TestData) (*PodIPs, *PodIPs, *PodIPs, *PodIPs, *PodIPs, error) { cmd := []string{"iperf3", "-s"} create := func(name string, nodeName string, ports []corev1.ContainerPort) error { - return NewPodBuilder(name, data.testNamespace, toolboxImage).WithContainerName("iperf").WithCommand(cmd).OnNode(nodeName).WithPorts(ports).Create(data) + return NewPodBuilder(name, data.testNamespace, ToolboxImage).WithContainerName("iperf").WithCommand(cmd).OnNode(nodeName).WithPorts(ports).Create(data) } var err error var podIPsArray [5]*PodIPs @@ -1888,7 +1888,7 @@ func testL7FlowExporterController(t *testing.T, data *TestData, isIPv6 bool) { clientPodName := "l7flowexportertestpodclient" clientPodLabels := map[string]string{"flowexportertest": "l7"} clientPodAnnotations := map[string]string{antreaagenttypes.L7FlowExporterAnnotationKey: "both"} - require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, toolboxImage).OnNode(nodeName).WithContainerName("l7flowexporter").WithLabels(clientPodLabels).WithAnnotations(clientPodAnnotations).Create(data)) + require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, ToolboxImage).OnNode(nodeName).WithContainerName("l7flowexporter").WithLabels(clientPodLabels).WithAnnotations(clientPodAnnotations).Create(data)) clientPodIPs, err := data.podWaitForIPs(defaultTimeout, clientPodName, data.testNamespace) require.NoErrorf(t, err, "Error when waiting for IP for Pod '%s': %v", clientPodName, err) defer deletePodWrapper(t, data, data.testNamespace, clientPodName) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 82374921712..42a59e5c3b1 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -126,32 +126,31 @@ const ( antreaControllerConfName = "antrea-controller.conf" flowAggregatorConfName = "flow-aggregator.conf" - nameSuffixLength = 8 + BusyboxImage = "projects.registry.vmware.com/antrea/busybox" agnhostImage = "registry.k8s.io/e2e-test-images/agnhost:2.29" - busyboxImage = "projects.registry.vmware.com/antrea/busybox" mcjoinImage = "projects.registry.vmware.com/antrea/mcjoin:v2.9" nginxImage = "projects.registry.vmware.com/antrea/nginx:1.21.6-alpine" iisImage = "mcr.microsoft.com/windows/servercore/iis" - toolboxImage = "projects.registry.vmware.com/antrea/toolbox:1.2-1" + ToolboxImage = "projects.registry.vmware.com/antrea/toolbox:1.2-1" ipfixCollectorImage = "projects.registry.vmware.com/antrea/ipfix-collector:v0.8.2" - ipfixCollectorPort = "4739" - clickHouseHTTPPort = "8123" nginxLBService = "nginx-loadbalancer" + ipfixCollectorPort = "4739" exporterFlowPollInterval = 1 * time.Second exporterActiveFlowExportTimeout = 2 * time.Second exporterIdleFlowExportTimeout = 1 * time.Second aggregatorActiveFlowRecordTimeout = 3500 * time.Millisecond aggregatorInactiveFlowRecordTimeout = 6 * time.Second aggregatorClickHouseCommitInterval = 1 * time.Second + clickHouseHTTPPort = "8123" + defaultCHDatabaseURL = "tcp://clickhouse-clickhouse.flow-visibility.svc:9000" statefulSetRestartAnnotationKey = "antrea-e2e/restartedAt" - defaultCHDatabaseURL = "tcp://clickhouse-clickhouse.flow-visibility.svc:9000" - iperfPort = 5201 - iperfSvcPort = 9999 + iperfPort = 5201 + iperfSvcPort = 9999 ) type ClusterNode struct { @@ -1506,7 +1505,7 @@ func (data *TestData) UpdatePod(namespace, name string, mutateFunc func(*corev1. // createBusyboxPodOnNode creates a Pod in the test namespace with a single busybox container. The // Pod will be scheduled on the specified Node (if nodeName is not empty). func (data *TestData) createBusyboxPodOnNode(name string, ns string, nodeName string, hostNetwork bool) error { - return NewPodBuilder(name, ns, busyboxImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).Create(data) + return NewPodBuilder(name, ns, BusyboxImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).Create(data) } // createMcJoinPodOnNode creates a Pod in the test namespace with a single mcjoin container. The @@ -1518,7 +1517,7 @@ func (data *TestData) createMcJoinPodOnNode(name string, ns string, nodeName str // createToolboxPodOnNode creates a Pod in the test namespace with a single toolbox container. The // Pod will be scheduled on the specified Node (if nodeName is not empty). func (data *TestData) createToolboxPodOnNode(name string, ns string, nodeName string, hostNetwork bool) error { - return NewPodBuilder(name, ns, toolboxImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).Create(data) + return NewPodBuilder(name, ns, ToolboxImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).Create(data) } // createNginxPodOnNode creates a Pod in the test namespace with a single nginx container. The @@ -2123,10 +2122,6 @@ func (data *TestData) deleteNetworkpolicy(policy *networkingv1.NetworkPolicy) er return nil } -func RandName(prefix string) string { - return prefix + randSeq(nameSuffixLength) -} - // A DNS-1123 subdomain must consist of lower case alphanumeric characters var lettersAndDigits = []rune("abcdefghijklmnopqrstuvwxyz0123456789") @@ -2142,6 +2137,7 @@ func randSeq(n int) string { // randName generates a DNS-1123 subdomain name func randName(prefix string) string { + nameSuffixLength := 8 return prefix + randSeq(nameSuffixLength) } diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index d9ae5cd04aa..2d590e333f3 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -15,160 +15,10 @@ package e2e import ( - "flag" - "fmt" - "log" "os" "testing" ) -// setupLogging creates a temporary directory to export the test logs if necessary. If a directory -// was provided by the user, it checks that the directory exists. -func (tOptions *TestOptions) setupLogging() func() { - if tOptions.logsExportDir == "" { - name, err := os.MkdirTemp("", "antrea-test-") - if err != nil { - log.Fatalf("Error when creating temporary directory to export logs: %v", err) - } - log.Printf("Test logs (if any) will be exported under the '%s' directory", name) - tOptions.logsExportDir = name - // we will delete the temporary directory if no logs are exported - return func() { - if empty, _ := IsDirEmpty(name); empty { - log.Printf("Removing empty logs directory '%s'", name) - _ = os.Remove(name) - } else { - log.Printf("Logs exported under '%s', it is your responsibility to delete the directory when you no longer need it", name) - } - } - } - fInfo, err := os.Stat(tOptions.logsExportDir) - if err != nil { - log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.logsExportDir, err) - } - if !fInfo.Mode().IsDir() { - log.Fatalf("'%s' is not a valid directory", tOptions.logsExportDir) - } - // no-op cleanup function - return func() {} -} - -// setupCoverage checks if the directory provided by the user exists. -func (tOptions *TestOptions) setupCoverage(data *TestData) func() { - if tOptions.coverageDir != "" { - fInfo, err := os.Stat(tOptions.coverageDir) - if err != nil { - log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.coverageDir, err) - } - if !fInfo.Mode().IsDir() { - log.Fatalf("'%s' is not a valid directory", tOptions.coverageDir) - } - - } - // cpNodeCoverageDir is a directory on the control-plane Node, where tests can deposit test - // coverage data. - log.Printf("Creating directory '%s' on Node '%s'\n", cpNodeCoverageDir, controlPlaneNodeName()) - rc, _, _, err := data.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("mkdir -p %s", cpNodeCoverageDir)) - if err != nil || rc != 0 { - log.Fatalf("Failed to create directory '%s' on control-plane Node", cpNodeCoverageDir) - } - return func() { - log.Printf("Removing directory '%s' on Node '%s'\n", cpNodeCoverageDir, controlPlaneNodeName()) - // best effort - data.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("rm -rf %s", cpNodeCoverageDir)) - } - -} - -// testMain is meant to be called by TestMain and enables the use of defer statements. -func testMain(m *testing.M) int { - flag.StringVar(&testOptions.providerName, "provider", "vagrant", "K8s test cluster provider") - flag.StringVar(&testOptions.providerConfigPath, "provider-cfg-path", "", "Optional config file for provider") - flag.StringVar(&testOptions.logsExportDir, "logs-export-dir", "", "Export directory for test logs") - flag.BoolVar(&testOptions.logsExportOnSuccess, "logs-export-on-success", false, "Export logs even when a test is successful") - flag.BoolVar(&testOptions.withBench, "benchtest", false, "Run tests include benchmark tests") - flag.BoolVar(&testOptions.enableCoverage, "coverage", false, "Run tests and measure coverage") - flag.BoolVar(&testOptions.enableAntreaIPAM, "antrea-ipam", false, "Run tests with AntreaIPAM") - flag.BoolVar(&testOptions.flowVisibility, "flow-visibility", false, "Run flow visibility tests") - flag.BoolVar(&testOptions.deployAntrea, "deploy-antrea", true, "Deploy Antrea before running tests") - flag.StringVar(&testOptions.coverageDir, "coverage-dir", "", "Directory for coverage data files") - flag.StringVar(&testOptions.skipCases, "skip-cases", "", "Key words to skip cases") - flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") - flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs") - flag.StringVar(&testOptions.externalServerIPs, "external-server-ips", "", "IP addresses of external server, at most one IP per IP family") - flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family") - flag.IntVar(&testOptions.vlanID, "vlan-id", 0, "ID of the VLAN network the Nodes reside in") - flag.Parse() - - cleanupLogging := testOptions.setupLogging() - defer cleanupLogging() - - testData = &TestData{} - if err := testData.InitProvider(testOptions.providerName, testOptions.providerConfigPath); err != nil { - log.Fatalf("Error when initializing provider: %v", err) - } - log.Println("Creating K8s ClientSet") - kubeconfigPath, err := testData.provider.GetKubeconfigPath() - if err != nil { - log.Fatalf("Error when getting Kubeconfig path: %v", err) - } - if err := testData.CreateClient(kubeconfigPath); err != nil { - log.Fatalf("Error when creating K8s ClientSet: %v", err) - } - log.Println("Collecting information about K8s cluster") - if err := testData.collectClusterInfo(); err != nil { - log.Fatalf("Error when collecting information about K8s cluster: %v", err) - } - if clusterInfo.podV4NetworkCIDR != "" { - log.Printf("Pod IPv4 network: '%s'", clusterInfo.podV4NetworkCIDR) - } - if clusterInfo.podV6NetworkCIDR != "" { - log.Printf("Pod IPv6 network: '%s'", clusterInfo.podV6NetworkCIDR) - } - if clusterInfo.svcV4NetworkCIDR != "" { - log.Printf("Service IPv4 network: '%s'", clusterInfo.svcV4NetworkCIDR) - } - if clusterInfo.svcV6NetworkCIDR != "" { - log.Printf("Service IPv6 network: '%s'", clusterInfo.svcV6NetworkCIDR) - } - log.Printf("Num nodes: %d", clusterInfo.numNodes) - if err := testData.collectExternalInfo(); err != nil { - log.Fatalf("Error when collecting external information: %v", err) - } - err = ensureAntreaRunning(testData) - if err != nil { - log.Fatalf("Error when deploying Antrea: %v", err) - } - // Collect PodCIDRs after Antrea is running as Antrea is responsible for allocating PodCIDRs in some cases. - // Polling is not needed here because antrea-agents won't be up and running if PodCIDRs of their Nodes are not set. - if err = testData.collectPodCIDRs(); err != nil { - log.Fatalf("Error collecting PodCIDRs: %v", err) - } - AntreaConfigMap, err = testData.GetAntreaConfigMap(antreaNamespace) - if err != nil { - log.Fatalf("Error when getting antrea-config configmap: %v", err) - } - if testOptions.enableCoverage { - cleanupCoverage := testOptions.setupCoverage(testData) - defer cleanupCoverage() - defer gracefulExitAntrea(testData) - } - ret := m.Run() - return ret -} - -func gracefulExitAntrea(testData *TestData) { - if err := testData.gracefulExitAntreaController(testOptions.coverageDir); err != nil { - log.Fatalf("Error when gracefully exit antrea controller: %v", err) - } - if err := testData.gracefulExitAntreaAgent(testOptions.coverageDir, "all"); err != nil { - log.Fatalf("Error when gracefully exit antrea agent: %v", err) - } - if err := testData.collectAntctlCovFilesFromControlPlaneNode(testOptions.coverageDir); err != nil { - log.Fatalf("Error when collecting antctl coverage files from control-plane Node: %v", err) - } -} - func TestMain(m *testing.M) { os.Exit(testMain(m)) } diff --git a/test/e2e/performance_test.go b/test/e2e/performance_test.go index 1b73b9ec88e..947fe2395ac 100644 --- a/test/e2e/performance_test.go +++ b/test/e2e/performance_test.go @@ -190,7 +190,7 @@ func setupTestPods(data *TestData, b *testing.B) (nginxPodIP, perfPodIP *PodIPs) } b.Logf("Creating a toolbox test Pod") - perfPod := createPerfTestPodDefinition(toolboxPodName, toolboxContainerName, toolboxImage) + perfPod := createPerfTestPodDefinition(toolboxPodName, toolboxContainerName, ToolboxImage) _, err = data.clientset.CoreV1().Pods(data.testNamespace).Create(context.TODO(), perfPod, metav1.CreateOptions{}) if err != nil { b.Fatalf("Error when creating toolbox test Pod: %v", err) diff --git a/test/e2e/proxy_test.go b/test/e2e/proxy_test.go index ff75a762208..ed9307f459d 100644 --- a/test/e2e/proxy_test.go +++ b/test/e2e/proxy_test.go @@ -1101,7 +1101,7 @@ func TestProxyLoadBalancerModeDSR(t *testing.T) { backendNode2 := workerNodeName(2) internalClient := "internal-client" - err = NewPodBuilder(internalClient, data.testNamespace, toolboxImage).OnNode(ingressNode).Create(data) + err = NewPodBuilder(internalClient, data.testNamespace, ToolboxImage).OnNode(ingressNode).Create(data) require.NoError(t, err, "Failed to create internal client") defer deletePodWrapper(t, data, data.testNamespace, internalClient) internalClientIPs, err := data.podWaitForIPs(defaultTimeout, internalClient, data.testNamespace) @@ -1145,7 +1145,7 @@ func TestProxyLoadBalancerModeDSR(t *testing.T) { // Create another netns to fake an external network on the host network Pod. externalClient := randName("external-client-") cmd, externalNetns := getCommandInFakeExternalNetwork("sleep infinity", externalIPPrefix, externalClientIP, externalClientGateway) - err := NewPodBuilder(externalClient, data.testNamespace, toolboxImage).OnNode(ingressNode).WithCommand([]string{"sh", "-c", cmd}).InHostNetwork().Privileged().Create(data) + err := NewPodBuilder(externalClient, data.testNamespace, ToolboxImage).OnNode(ingressNode).WithCommand([]string{"sh", "-c", cmd}).InHostNetwork().Privileged().Create(data) require.NoError(t, err, "Failed to create external client") defer deletePodWrapper(t, data, data.testNamespace, externalClient) err = data.podWaitForRunning(defaultTimeout, externalClient, data.testNamespace) diff --git a/test/e2e/wireguard_test.go b/test/e2e/wireguard_test.go index 349bcf57ecd..7205ea5b4d8 100644 --- a/test/e2e/wireguard_test.go +++ b/test/e2e/wireguard_test.go @@ -145,7 +145,7 @@ func testServiceConnectivity(t *testing.T, data *TestData) { defer cleanup() // Create the a hostNetwork Pod on a Node different from the service's backend Pod, so the service traffic will be transferred across the tunnel. - require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, busyboxImage).OnNode(clientPodNode).WithCommand([]string{"sleep", "3600"}).InHostNetwork().Create(data)) + require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, BusyboxImage).OnNode(clientPodNode).WithCommand([]string{"sleep", "3600"}).InHostNetwork().Create(data)) defer data.DeletePodAndWait(defaultTimeout, clientPodName, data.testNamespace) require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace))