diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index ce5c196eafe..fad85f21909 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -509,6 +509,51 @@ jobs: path: log.tar.gz retention-days: 30 + test-secondary-network: + name: Antrea-native (VLAN) secondary network tests on a Kind cluster on Linux + needs: [build-antrea-coverage-image] + runs-on: [ubuntu-latest] + steps: + - name: Free disk space + # https://github.com/actions/virtual-environments/issues/709 + run: | + sudo apt-get clean + df -h + - uses: actions/checkout@v4 + with: + show-progress: false + - uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + - name: Download Antrea image from previous job + uses: actions/download-artifact@v3 + with: + name: antrea-ubuntu-cov + - name: Load Antrea image + run: | + docker load -i antrea-ubuntu.tar + docker tag antrea/antrea-ubuntu-coverage:latest antrea/antrea-ubuntu:latest + - name: Install Kind + run: | + KIND_VERSION=$(head -n1 ./ci/kind/version) + curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv kind /usr/local/bin + - name: Run secondary network tests + run: | + mkdir log + ANTREA_LOG_DIR=$PWD/log ./ci/kind/test-secondary-network-kind.sh + - name: Tar log files + if: ${{ failure() }} + run: tar -czf log.tar.gz log + - name: Upload test log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: secondary-network.tar.gz + path: log.tar.gz + retention-days: 30 + test-upgrade-from-N-1: name: Upgrade from Antrea version N-1 needs: build-antrea-coverage-image diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index e93fe46c4fb..40609465d41 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -427,7 +427,7 @@ secondaryNetwork: {{- with .Values.secondaryNetwork }} # Configuration of OVS bridges for secondary network. ovsBridges: - {{- toYaml .ovsBridges | trim | nindent 6 }} + {{- toYaml .ovsBridges | trim | nindent 4 }} {{- end }} {{- end }} diff --git a/ci/kind/kind-setup.sh b/ci/kind/kind-setup.sh index 11296ce1fa9..89515b5faef 100755 --- a/ci/kind/kind-setup.sh +++ b/ci/kind/kind-setup.sh @@ -220,7 +220,7 @@ function configure_extra_networks { i=$((i+1)) done - nodes="$(kind get nodes --name $CLUSTER_NAME | grep worker)" + nodes="$(kind get nodes --name $CLUSTER_NAME)" for node in $nodes; do for network in $networks; do docker network connect $network $node >/dev/null 2>&1 diff --git a/ci/kind/test-secondary-network-kind.sh b/ci/kind/test-secondary-network-kind.sh new file mode 100755 index 00000000000..1e9899ecd7b --- /dev/null +++ b/ci/kind/test-secondary-network-kind.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +# Copyright 2023 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The script runs NetworkPolicy conformance tests on Kind with different traffic encapsulation modes. + +set -eo pipefail + +function echoerr { + >&2 echo "$@" +} + +_usage="Usage: $0 [--setup-only|--test-only|--cleanup-only] [--help|-h] + --setup-only Only perform setting up the cluster and run test. + --test-only Only run test on current cluster. Not set up/clean up the cluster. + --cleanup-only Only perform cleaning up the cluster. + --help, -h Print this message and exit. +" + +function print_usage { + echoerr -n "$_usage" +} + +TESTBED_CMD=$(dirname $0)"/kind-setup.sh" +YML_CMD=$(dirname $0)"/../../hack/generate-manifest.sh" +ATTACHMENT_DEFINITION_YAML=$(dirname $0)"/../../test/e2e-secondary-network/infra/network-attachment-definition-crd.yml" +SECONDARY_NETWORKS_YAML=$(dirname $0)"/../../test/e2e-secondary-network/infra/secondary-networks.yml" + +TIMEOUT="5m" +# Create a secondary OVS bridge with the Node's physical interface on the extra +# network. +MANIFEST_ARGS="--secondary-bridge br-secondary --physical-interface eth1" +# Antrea is deployed by this script. Do not deploy it again in the test. +TEST_OPTIONS="--logs-export-dir=$ANTREA_LOG_DIR --deploy-antrea=false" +EXTRA_NETWORK="20.20.20.0/24" + +setup_only=false +cleanup_only=false +test_only=false + +function quit { + result=$? + if [[ $setup_only || $test_only ]]; then + exit $result + fi + echoerr "Cleaning testbed" + $TESTBED_CMD destroy kind +} + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + --setup-only) + setup_only=true + shift + ;; + --cleanup-only) + cleanup_only=true + shift + ;; + --test-only) + test_only=true + shift + ;; + -h|--help) + print_usage + exit 0 + ;; + *) # unknown option + echoerr "Unknown option $1" + exit 1 + ;; +esac +done + +if [[ $cleanup_only == "true" ]];then + $TESTBED_CMD destroy kind + exit 0 +fi + +trap "quit" INT EXIT + +IMAGE_LIST=("projects.registry.vmware.com/antrea/busybox" \ + "antrea/antrea-ubuntu:latest") + +printf -v IMAGES "%s " "${IMAGE_LIST[@]}" + +function setup_cluster { + args=$1 + echo "creating test bed with args $args" + eval "timeout 600 $TESTBED_CMD create kind $args" +} + +function run_test { + echo "Generating Antrea manifest with args $MANIFEST_ARGS" + $YML_CMD $MANIFEST_ARGS | kubectl apply -f - + + # Wait for antrea-controller start to make sure the IPPool validation webhook is ready. + kubectl rollout status --timeout=1m deployment.apps/antrea-controller -n kube-system + kubectl apply -f $ATTACHMENT_DEFINITION_YAML + kubectl apply -f $SECONDARY_NETWORKS_YAML + + go test -v -timeout=$TIMEOUT antrea.io/antrea/test/e2e-secondary-network -run=TestVLANNetwork -provider=kind $TEST_OPTIONS +} + +echo "======== Testing Antrea-native secondary network support ==========" +if [[ $test_only == "false" ]];then + setup_cluster "--extra-networks \"$EXTRA_NETWORK\" --images \"$IMAGES\"" +fi +run_test +exit 0 diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index 8a35cfc8ed0..4a41e700e92 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -20,30 +20,32 @@ function echoerr { >&2 echo "$@" } -_usage="Usage: $0 [--mode (dev|release)] [--encap-mode] [--ipsec] [--tun (geneve|vxlan|gre|stt)] [--verbose-log] [--help|-h] +_usage="Usage: $0 [--mode (dev|release)] [--encap-mode (mode)] [--ipsec] [--tun (geneve|vxlan|gre|stt)] [--verbose-log] Generate a YAML manifest for Antrea using Helm and print it to stdout. - --mode (dev|release) Choose the configuration variant that you need (default is 'dev') - --encap-mode Traffic encapsulation mode. (default is 'encap') - --cloud Generate a manifest appropriate for running Antrea in Public Cloud - --ipsec Generate a manifest with IPsec encryption of tunnel traffic enabled + --mode (dev|release) Choose the configuration variant that you need (default is 'dev'). + --encap-mode (mode) Traffic encapsulation mode (default is 'encap'). + --cloud Generate a manifest appropriate for running Antrea in Public Cloud. + --ipsec Generate a manifest with IPsec encryption of tunnel traffic enabled. --feature-gates A comma-separated list of key=value pairs that describe feature gates, e.g. TrafficControl=true,Egress=false. - --proxy-all Generate a manifest with Antrea proxy with all Service support enabled - --tun (geneve|vxlan|gre|stt) Choose encap tunnel type from geneve, gre, stt and vxlan (default is geneve) - --verbose-log Generate a manifest with increased log-level (level 4) for Antrea agent and controller. - This option will work only in 'dev' mode. + --proxy-all Generate a manifest with Antrea proxy with all Service support enabled. + --tun (geneve|vxlan|gre|stt) Choose encap tunnel type from geneve, gre, stt and vxlan (default is geneve). --on-delete Generate a manifest with antrea-agent's update strategy set to OnDelete. This option will work only in 'dev' mode. - --coverage Generates a manifest which supports measuring code coverage of Antrea binaries. - --simulator Generates a manifest with antrea-agent simulator included - --custom-adm-controller Generates a manifest with custom Antrea admission controller to validate/mutate resources. - --hw-offload Generates a manifest with hw-offload enabled in the antrea-ovs container. - --sriov Generates a manifest which enables use of Kubelet API for SR-IOV device info. - --flexible-ipam Generates a manifest with flexible IPAM enabled. - --help, -h Print this message and exit - --multicast Generates a manifest for multicast. - --multicast-interfaces Multicast interface names (default is empty) - --extra-helm-values-file Optional extra helm values file to override the default config values - --extra-helm-values Optional extra helm values to override the default config values + --coverage Generate a manifest which supports measuring code coverage of Antrea binaries. + --simulator Generate a manifest with antrea-agent simulator included. + --custom-adm-controller Generate a manifest with custom Antrea admission controller to validate/mutate resources. + --flexible-ipam Generate a manifest with flexible IPAM enabled. + --hw-offload Generate a manifest with hw-offload enabled in the antrea-ovs container. + --sriov Generate a manifest which enables use of kubelet API for SR-IOV device info. + --secondary-bridge (bridge) Generate a manifest which enables secondary network and creates a secondary OVS bridge. + --physical-interface (device) Specify the physical interface of the secondary OVS bridge. + --multicast Generate a manifest for multicast. + --multicast-interfaces Multicast interface names (default is empty). + --extra-helm-values-file Optional extra helm values file to override the default config values. + --extra-helm-values Optional extra helm values to override the default config values. + --verbose-log Generate a manifest with increased log-level (level 4) for Antrea agent and controller. + This option will work only in 'dev' mode. + --help, -h Print this message and exit. In 'release' mode, environment variables IMG_NAME and IMG_TAG must be set. @@ -75,9 +77,11 @@ COVERAGE=false K8S_115=false SIMULATOR=false CUSTOM_ADM_CONTROLLER=false +FLEXIBLE_IPAM=false HW_OFFLOAD=false SRIOV=false -FLEXIBLE_IPAM=false +SECONDARY_BRIDGE="" +PHYSICAL_INTERFACE="" MULTICAST=false MULTICAST_INTERFACES="" HELM_VALUES_FILES=() @@ -126,10 +130,6 @@ case $key in TUN_TYPE="$2" shift 2 ;; - --verbose-log) - VERBOSE_LOG=true - shift - ;; --on-delete) ON_DELETE=true shift @@ -146,6 +146,10 @@ case $key in CUSTOM_ADM_CONTROLLER=true shift ;; + --flexible-ipam) + FLEXIBLE_IPAM=true + shift + ;; --hw-offload) HW_OFFLOAD=true shift @@ -154,9 +158,13 @@ case $key in SRIOV=true shift ;; - --flexible-ipam) - FLEXIBLE_IPAM=true - shift + --secondary-bridge) + SECONDARY_BRIDGE="$2" + shift 2 + ;; + --physical-interface) + PHYSICAL_INTERFACE="$2" + shift 2 ;; --multicast) MULTICAST=true @@ -178,6 +186,10 @@ case $key in HELM_VALUES+=("$2") shift 2 ;; + --verbose-log) + VERBOSE_LOG=true + shift + ;; -h|--help) print_usage exit 0 @@ -322,6 +334,20 @@ EOF HELM_VALUES_FILES+=("$TMP_DIR/sriov.yml") fi +if [[ $SECONDARY_BRIDGE != "" ]]; then + if [[ $PHYSICAL_INTERFACE != "" ]]; then + ovs_bridges="[{bridgeName: $SECONDARY_BRIDGE, physicalInterfaces: [$PHYSICAL_INTERFACE]}]" + else + ovs_bridges="[{bridgeName: \"$SECONDARY_BRIDGE\"}]" + fi + cat << EOF > $TMP_DIR/secondary-network.yml +secondaryNetwork: + ovsBridges: $ovs_bridges +EOF + HELM_VALUES+=("featureGates.SecondaryNetwork=true" "featureGates.AntreaIPAM=true") + HELM_VALUES_FILES+=("$TMP_DIR/secondary-network.yml") +fi + if [ "$MODE" == "dev" ]; then if [[ -z "$IMG_NAME" ]]; then if $COVERAGE; then diff --git a/test/e2e-secondary-network/framework.go b/test/e2e-secondary-network/framework.go deleted file mode 100644 index f87e789f575..00000000000 --- a/test/e2e-secondary-network/framework.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2022 Antrea Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "log" - "time" - - antreae2e "antrea.io/antrea/test/e2e" -) - -type TestData struct { - e2eTestData *antreae2e.TestData - logsDirForTestCase string -} - -const ( - busyboxImage = "projects.registry.vmware.com/antrea/busybox" - defaultInterval = 1 * time.Second -) - -var testData *TestData - -type ClusterInfo struct { - controlPlaneNodeName string -} - -var clusterInfo ClusterInfo - -func (data *TestData) createClient(kubeconfigPath string) error { - e2edata = &antreae2e.TestData{} - if err := e2edata.CreateClient(kubeconfigPath); err != nil { - log.Fatalf("Error when creating K8s ClientSet: %v", err) - return err - } - data.e2eTestData = e2edata - return nil -} diff --git a/test/e2e-secondary-network/infra/prepare_cluster.sh b/test/e2e-secondary-network/infra/prepare_cluster.sh index 0c5d335b719..a5bc82dc8e3 100755 --- a/test/e2e-secondary-network/infra/prepare_cluster.sh +++ b/test/e2e-secondary-network/infra/prepare_cluster.sh @@ -18,14 +18,13 @@ THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" pushd $THIS_DIR _usage="Usage: $0 [--setup] [--cleanup] - [--network ] [--host-name ] [--help] + [--network ] [--host-name ] [--help] Deploy Antrea CNI and its native secondary network prerequisites on an existing/running K8s cluster control node. --setup Deploy all the pre-requisite plugins and CRDs for secondary network configuration. --cleanup Remove all the pre-requisite plugins and CRDs deployed for secondary network configuration. - --network sriov or veth (provide --network=sriov for sriov based secondary network configuration. - --network=veth for veth based secondary network configuration). + --network Secondary network type, can be sriov or vlan. --host-name K8s cluster control node's host name or IP address." _ssh_config_info="Prerequisite: Kubernetes(K8s) cluster is up and running at the local or remote server. @@ -64,12 +63,9 @@ function print_help { NAMESPACE="kube-system" ANTREA_DS="ds/antrea-agent" -WHEREABOUTS_CNI_REL_TAG=v0.6.1 -export URL_WHEREABOUTS_IP_POOLS="https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/$WHEREABOUTS_CNI_REL_TAG/doc/crds/whereabouts.cni.cncf.io_ippools.yaml" -export URL_WHEREABOUTS_OVERLAPPING_IP_RANGE_RES="https://raw.githubusercontent.com/k8snetworkplumbingwg/whereabouts/$WHEREABOUTS_CNI_REL_TAG/doc/crds/whereabouts.cni.cncf.io_overlappingrangeipreservations.yaml" export YAML_ANTREA='antrea.yml' export YAML_NET_ATTACH_DEF_CRD='network-attachment-definition-crd.yml' -export YAML_VIRTUAL_NET_INSTANCE='virtual-network-instance-crd.yml' +export YAML_SECONDARY_NETWORKS='secondary-networks.yml' action='' configure='' @@ -78,12 +74,12 @@ network='' function generateAntreaConfig() { genManifest=$THIS_DIR/../../../hack/generate-manifest.sh - echo $genManifest + echo $genManifest if [ "$1" == "sriov" ]; then - $genManifest --sriov --whereabouts --extra-helm-values "featureGates.SecondaryNetwork=true" > $YAML_ANTREA - elif [ "$1" == "veth" ]; then - $genManifest --whereabouts --extra-helm-values "featureGates.SecondaryNetwork=true" > $YAML_ANTREA - else + $genManifest --sriov --extra-helm-values "featureGates.SecondaryNetwork=true" > $YAML_ANTREA + elif [ "$1" == "vlan" ]; then + $genManifest --extra-helm-values "featureGates.SecondaryNetwork=true" > $YAML_ANTREAc + else echoerr "Incorrect network option $1. Failed to generate antrea.yml" exit 1 fi @@ -102,30 +98,13 @@ function AntreaCNI() { echoerr "Failed to $action Antrea CNI" exit 1 fi - # Setting timeout to report lack of progress after set time is elapsed + # Setting timeout to report lack of progress after set time is elapsed kubectl patch $ANTREA_DS -p '{"spec":{"progressDeadlineSeconds":30}}' -n $NAMESPACE kubectl -n $NAMESPACE rollout status $ANTREA_DS } -function WhereaboutsCNI() { - kubectl $action -f $URL_WHEREABOUTS_IP_POOLS - if [ $? -ne 0 ]; then - echoerr "Failed to $action Whereabouts CNI" - exit 1 - fi - echo "$action WhereaboutsCNI Done!" - - kubectl $action -f $URL_WHEREABOUTS_OVERLAPPING_IP_RANGE_RES - if [ $? -ne 0 ]; then - echoerr "Failed to $action Whereabouts overlapping IP Range reservation" - exit 1 - fi - echo "$action WhereaboutsOverlappingIpRangeRes Done!" -} - - function VirtualNetworks() { - kubectl $action -f $YAML_VIRTUAL_NET_INSTANCE + kubectl $action -f $YAML_SECONDARY_NETWORKS if [ $? -ne 0 ]; then echoerr "Failed to $action Virtual Network Instance: $instance" exit 1 @@ -147,7 +126,6 @@ function NetworkAttachmentDefinition() { function configureSecondaryNetworkPrerequisite() { NetworkAttachmentDefinition - WhereaboutsCNI AntreaCNI echo "Setup is up and running..." } @@ -181,10 +159,10 @@ while [[ $# -gt 0 ]]; do hostName="$2" shift 2 ;; - --network) - network=$2 - shift 2 - ;; + --network) + network=$2 + shift 2 + ;; --help) print_usage exit 0 @@ -198,7 +176,7 @@ done # Download all the prerequisite files. if [[ $configure == true ]]; then - # Generate antrea.yml with --sriov and --whereabouts config options. + # Generate antrea.yml with --sriov config option. generateAntreaConfig $network fi configureSecondaryNetworkPrerequisite diff --git a/test/e2e-secondary-network/infra/prepare_sriov.sh b/test/e2e-secondary-network/infra/prepare_sriov.sh index 18b8787faa5..df930780463 100755 --- a/test/e2e-secondary-network/infra/prepare_sriov.sh +++ b/test/e2e-secondary-network/infra/prepare_sriov.sh @@ -22,8 +22,8 @@ _usage="Usage: $0 [--setup] [--cleanup] Configure/cleanup SR-IOV VFs and deploy the SR-IOV device plugin at the remote single node cluster. --setup Configures the SR-IOV VFs and deploy the SR-IOV device plugin at the remote cluster (single node cluster). - --cleanup cleanup SR-IOV device plugin and VFs at the remote cluster (single node cluster). - --sriov-interface Base network interface (remote node scope) to create VFs. + --cleanup cleanup SR-IOV device plugin and VFs at the remote cluster (single node cluster). + --sriov-interface Base network interface (remote node scope) to create VFs. --sriov-vf-count Number of virtual functions to be created on the base network interface. --host-name K8s cluster's host name or IP address." @@ -63,7 +63,6 @@ function print_help { SRIOV_PLUGIN_REL_TAG=v3.5.1 -WHEREABOUTS_CNI_REL_TAG=v0.6.1 export URL_SRIOV_DP_CONFIG_MAP="https://raw.githubusercontent.com/k8snetworkplumbingwg/sriov-network-device-plugin/$SRIOV_PLUGIN_REL_TAG/deployments/configMap.yaml" export URL_SRIOV_DP_DAEMONSET="https://raw.githubusercontent.com/k8snetworkplumbingwg/sriov-network-device-plugin/$SRIOV_PLUGIN_REL_TAG/deployments/k8s-v1.16/sriovdp-daemonset.yaml" export YAML_ANTREA='antrea.yml' @@ -171,7 +170,7 @@ function updateKubeConfigForTargetCluster() { echoerr "Failed to copy the downloaded configuration files!" exit 1 fi - export KUBECONFIG=/tmp/kubernetes/$hostName/admin.conf + export KUBECONFIG=/tmp/kubernetes/$hostName/admin.conf } @@ -191,11 +190,11 @@ EOF fi SriovDevicePlugin fi - if [[ $configure == "true" ]]; then + if [[ $configure == "true" ]]; then echo "SR-IOV successfully configured at the remote server." - else - echo "SR-IOV successfully deconfigured at the remote server." - fi + else + echo "SR-IOV successfully deconfigured at the remote server." + fi } if [[ $# -eq 0 ]]; then diff --git a/test/e2e-secondary-network/infra/secondary-network-configuration.yml b/test/e2e-secondary-network/infra/secondary-network-configuration.yml deleted file mode 100755 index a0d9d43e3c7..00000000000 --- a/test/e2e-secondary-network/infra/secondary-network-configuration.yml +++ /dev/null @@ -1,14 +0,0 @@ -interface_type: - interfacetype: sriov -sriov_conf: - networkinterface: ens785f0 - - numberofvfs: 24 -vir_net: - totalnumberofvirtualnetworks: 3 - virtualnetworknames: ["virtual-network-instance1.yaml","virtual-network-instance2.yaml", "virtual-network-instance3.yaml"] -create_pod: - numberofpods: 3 - describe: [["testpodsec1",2,"virtual-net1,virtual-net2","eth1,eth2","sanity1.yaml"], - ["testpodsec2",2,"virtual-net1,virtual-net2","eth2,eth3","sanity2.yaml"], - ["testpodsec3",1,"virtual-net1","eth4","sanity3.yaml"]] diff --git a/test/e2e-secondary-network/infra/secondary-networks.yml b/test/e2e-secondary-network/infra/secondary-networks.yml new file mode 100644 index 00000000000..5495dd18394 --- /dev/null +++ b/test/e2e-secondary-network/infra/secondary-networks.yml @@ -0,0 +1,147 @@ +--- +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: secnet-ipv4-1 +spec: + ipVersion: 4 + ipRanges: + - cidr: "148.14.24.0/24" + gateway: "148.14.24.1" + prefixLength: 24 + +--- +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: secnet-ipv4-2 +spec: + ipVersion: 4 + ipRanges: + - start: "148.14.25.111" + end: "148.14.25.123" + gateway: "148.14.25.1" + prefixLength: 24 + +--- +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: secnet-ipv4-3 +spec: + ipVersion: 4 + ipRanges: + - cidr: "148.14.26.0/24" + gateway: "148.14.26.1" + prefixLength: 24 + vlan: 300 + +--- +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: secnet-ipv6-3 +spec: + ipVersion: 6 + ipRanges: + - cidr: "10:2400::0/96" + gateway: "10:2400::1" + prefixLength: 64 + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net1 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "sriov", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-1"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net2 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "sriov", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-2"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net3 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "sriov", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-3"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: vlan-net1 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "vlan", + "mtu": 1200, + "vlan": 100, + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-1"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: vlan-net2 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "vlan", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-2"] + } + }' + +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: vlan-net3 +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "antrea", + "networkType": "vlan", + "ipam": { + "type": "antrea", + "ippools": ["secnet-ipv4-3", "secnet-ipv6-3"] + } + }' diff --git a/test/e2e-secondary-network/infra/virtual-network-instance-crd.yml b/test/e2e-secondary-network/infra/virtual-network-instance-crd.yml deleted file mode 100644 index f83affc9572..00000000000 --- a/test/e2e-secondary-network/infra/virtual-network-instance-crd.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: virtual-net1 -spec: - config: '{ - "cniVersion": "0.3.0", - "type": "antrea", - "networkType": "sriov", - "ipam": { - "type": "whereabouts", - "datastore": "kubernetes", - "kubernetes": { - "kubeconfig": "/host/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" - }, - "range": "148.14.24.0/24" - } - }' - ---- - -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: virtual-net2 -spec: - config: '{ - "cniVersion": "0.3.0", - "type": "antrea", - "networkType": "sriov", - "ipam": { - "type": "whereabouts", - "datastore": "kubernetes", - "kubernetes": { - "kubeconfig": "/host/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" - }, - "range": "148.14.25.0/24" - } - }' - ---- - -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: virtual-net3 -spec: - config: '{ - "cniVersion": "0.3.0", - "type": "antrea", - "networkType": "sriov", - "ipam": { - "type": "whereabouts", - "datastore": "kubernetes", - "kubernetes": { - "kubeconfig": "/host/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" - }, - "range": "148.14.26.0/24" - } - }' - ---- diff --git a/test/e2e-secondary-network/main_test.go b/test/e2e-secondary-network/main_test.go index 968513812b1..28b7b06697d 100644 --- a/test/e2e-secondary-network/main_test.go +++ b/test/e2e-secondary-network/main_test.go @@ -16,80 +16,15 @@ // instantiates and initializes objects imported from pkg, and runs // the process -package e2e +package e2esecondary import ( - "flag" - "log" "os" - "path" "testing" antreae2e "antrea.io/antrea/test/e2e" ) -type TestOptions struct { - logsExportDir string - enableAntreaIPAM bool - skipCases string - linuxVMs string -} - -var e2edata *antreae2e.TestData -var testOptions TestOptions -var homeDir, _ = os.UserHomeDir() - -// setupLogging creates a temporary directory to export the test logs if necessary. If a directory -// was provided by the user, it checks that the directory exists. -func (tOptions *TestOptions) setupLogging() func() { - if tOptions.logsExportDir == "" { - name, err := os.MkdirTemp("", "antrea-e2e-secondary-test-") - if err != nil { - log.Fatalf("Error when creating temporary directory to export logs: %v", err) - } - log.Printf("Test logs (if any) will be exported under the '%s' directory", name) - tOptions.logsExportDir = name - // we will delete the temporary directory if no logs are exported - return func() { - if empty, _ := antreae2e.IsDirEmpty(name); empty { - log.Printf("Removing empty logs directory '%s'", name) - _ = os.Remove(name) - } else { - log.Printf("Logs exported under '%s', it is your responsibility to delete the directory when you no longer need it", name) - } - } - } - fInfo, err := os.Stat(tOptions.logsExportDir) - if err != nil { - log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.logsExportDir, err) - } - if !fInfo.Mode().IsDir() { - log.Fatalf("'%s' is not a valid directory", tOptions.logsExportDir) - } - // no-op cleanup function - return func() {} -} - -func testMain(m *testing.M) int { - flag.StringVar(&testOptions.logsExportDir, "logs-export-dir", "", "Export directory for test logs") - flag.BoolVar(&testOptions.enableAntreaIPAM, "antrea-ipam", false, "Run tests with AntreaIPAM") - flag.StringVar(&testOptions.skipCases, "skip", "", "Key words to skip cases") - flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") - flag.Parse() - - cleanupLogging := testOptions.setupLogging() - defer cleanupLogging() - - testData = &TestData{} - log.Println("Creating K8s ClientSet") - kubeconfigPath := path.Join(homeDir, ".kube", "secondary_network_cluster", "config") - if err := testData.createClient(kubeconfigPath); err != nil { - log.Fatalf("Error when creating K8s ClientSet: %v", err) - } - ret := m.Run() - return ret -} - func TestMain(m *testing.M) { - os.Exit(testMain(m)) + os.Exit(antreae2e.RunTests(m)) } diff --git a/test/e2e-secondary-network/secondary_network_test.go b/test/e2e-secondary-network/secondary_network_test.go index e5caff7cd77..cd17a5d78c0 100644 --- a/test/e2e-secondary-network/secondary_network_test.go +++ b/test/e2e-secondary-network/secondary_network_test.go @@ -12,123 +12,63 @@ // See the License for the specific language governing permissions and // limitations under the License. -package e2e +package e2esecondary import ( "fmt" - "log" "net" - "os" "strings" "testing" "time" logs "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" antreae2e "antrea.io/antrea/test/e2e" ) -// Structure to extract and store the secondary network configuration information parsed from secondary-network-configuration.yml. -type PodConfig struct { - InterfaceType struct { - interfaceType string `yaml:"interfacetype"` - } `yaml:"interface_type"` - SriovConf struct { - networkInterface string `yaml:"networkinterface"` - numberOfVfs int `yaml:"numberofvfs"` - } `yaml:"sriov_conf"` - VirNet struct { - totalNumberOfVirtualNetworks int `yaml:"totalnumberofvirtualnetworks"` - virtualNetworknames []string `yaml:"virtualnetworknames"` - } `yaml:"vir_net"` - CreatePod struct { - numberOfPods int `yaml:"numberofpods"` - describe [][]interface{} `yaml:"describe"` - } `yaml:"create_pod"` -} - -var service PodConfig - // Structure for extracting the variables for describing the Pod from secondary-network-configuration.yml file. -type describePodInfo struct { - nameOfPods string - countOfVirtualNetworksPerPod int - nameOfVirtualNetworkPerPod []string - nameOfInterfacePerPod []string +type testPodInfo struct { + podName string + nodeName string + // map from interface name to secondary network name. + interfaceNetworks map[string]string } -var podData []describePodInfo -var totalNumberOfPods int -var interfaceType string +type testData struct { + e2eTestData *antreae2e.TestData + networkType string + pods []*testPodInfo +} const ( - secondaryNetworkConfigYAML = "./infra/secondary-network-configuration.yml" - nameSpace = "kube-system" - ctrName = "busyboxpod" - testPodName = "testsecpod" - osType = "linux" - count = 5 - size = 40 - defaultTimeout = 10 * time.Second - reqName = "intel.com/intel_sriov_netdevice" - resNum = 3 -) -const ( - podName = iota - podVNsCount - podVirtualNetwork - podInterfaceName + networkTypeSriov = "sriov" + networkTypeVLAN = "vlan" + + // Namespace of NetworkAttachmentDefinition CRs. + attachDefNamespace = "default" + + containerName = "busybox" + podApp = "secondaryTest" + osType = "linux" + pingCount = 5 + pingSize = 40 + defaultTimeout = 10 * time.Second + sriovReqName = "intel.com/intel_sriov_netdevice" + sriovResNum = 3 ) -// setupTestWithSecondaryNetworkConfig sets up all the prerequisites for running the test including the antrea enabled and running, extracting Pod and secondary network interface information and setting log directory for the test -func (data *TestData) setupTestWithSecondaryNetworkConfig(tb testing.TB) (*TestData, error) { - // Extracting the Pods information from the secondary_network_configuration.yml file. - if err := data.extractPodsInfo(); err != nil { - tb.Errorf("Error in extracting Pods info from secondary-network-configuration.yml : %v", err) - return nil, err - } - // Set log directory for test execution. - if err := data.e2eTestData.SetupLogDirectoryForTest(tb.Name()); err != nil { - tb.Errorf("Error creating logs directory '%s': %v", data.logsDirForTestCase, err) - return nil, err - } - return data, nil -} - -// extractPodsInfo extracts the Pod and secondary network interface information for the creation of Podsfrom secondary-network-configuration.yaml file -func (data *TestData) extractPodsInfo() error { - var errYamlUnmarshal error - _, err := os.Stat(secondaryNetworkConfigYAML) - if err != nil { - return fmt.Errorf("Parsing of the Pod configuration file failed") - - } - secondaryNetworkConfigYAML, _ := os.ReadFile(secondaryNetworkConfigYAML) - errYamlUnmarshal = yaml.Unmarshal(secondaryNetworkConfigYAML, &service) - if errYamlUnmarshal != nil { - return fmt.Errorf("Parsing %s failed", secondaryNetworkConfigYAML) - } - interfaceType = service.InterfaceType.interfaceType - totalNumberOfPods = service.CreatePod.numberOfPods - for _, s := range service.CreatePod.describe { - output := describePodInfo{nameOfPods: s[podName].(string), countOfVirtualNetworksPerPod: s[podVNsCount].(int), nameOfVirtualNetworkPerPod: strings.Split(s[podVirtualNetwork].(string), ","), nameOfInterfacePerPod: strings.Split(s[podInterfaceName].(string), ",")} - podData = append(podData, output) - } - return nil -} - // formAnnotationStringOfPod forms the annotation string, used in the generation of each Pod YAML file. -func (data *TestData) formAnnotationStringOfPod(pod int) string { +func (data *testData) formAnnotationStringOfPod(pod *testPodInfo) string { var annotationString = "" - for xPodVN := 0; xPodVN < podData[pod].countOfVirtualNetworksPerPod; xPodVN++ { - var podNetworkSpec = "{\"name\": \"" + podData[pod].nameOfVirtualNetworkPerPod[xPodVN] + "\" ,\"interface\": \"" + podData[pod].nameOfInterfacePerPod[xPodVN] + "\" , \"type\": \"" + interfaceType + "\"}" + for i, n := range pod.interfaceNetworks { + podNetworkSpec := fmt.Sprintf("{\"name\": \"%s\", \"namespace\": \"%s\", \"interface\": \"%s\"}", + n, attachDefNamespace, i) if annotationString == "" { annotationString = "[" + podNetworkSpec } else { - annotationString = annotationString + "," + podNetworkSpec + annotationString = annotationString + ", " + podNetworkSpec } } annotationString = annotationString + "]" @@ -136,77 +76,101 @@ func (data *TestData) formAnnotationStringOfPod(pod int) string { } // createPodOnNode creates the Pod for the specific annotations as per the parsed Pod information using the NewPodBuilder API -func (data *TestData) createPodOnNode(t *testing.T, ns string, nodeName string) error { +func (data *testData) createPods(t *testing.T, ns string) error { var err error - for xPod := 0; xPod < totalNumberOfPods; xPod++ { - err := data.createPodForSecondaryNetwork(ns, nodeName, xPod, testPodName, resNum) + for _, pod := range data.pods { + err := data.createPodForSecondaryNetwork(ns, pod) if err != nil { - return fmt.Errorf("Error in creating pods.., err: %v", err) + return fmt.Errorf("error in creating pods.., err: %v", err) } } return err } -// getSecondaryInterface shows up the secondary interfaces created for the specific Pod and extracts the IP address for the same. -func (data *TestData) getSecondaryInterface(targetPod int, targetInterface int) (string, error) { - cmd := []string{"/bin/sh", "-c", fmt.Sprintf("ip addr show %s | grep \"inet\" | awk '{print $2}' | cut -d/ -f1", podData[targetPod].nameOfInterfacePerPod[targetInterface])} - stdout, _, err := data.e2eTestData.RunCommandFromPod(nameSpace, podData[targetPod].nameOfPods, ctrName, cmd) +// The Wrapper function createPodForSecondaryNetwork creates the Pod adding the annotation, arguments, commands, Node, container name, +// resource requests and limits as arguments with the NewPodBuilder API +func (data *testData) createPodForSecondaryNetwork(ns string, pod *testPodInfo) error { + podBuilder := antreae2e.NewPodBuilder(pod.podName, ns, antreae2e.BusyboxImage). + OnNode(pod.nodeName).WithContainerName(containerName). + WithCommand([]string{"sleep", "infinity"}). + WithAnnotations(map[string]string{ + "k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%s", data.formAnnotationStringOfPod(pod)), + }). + WithLabels(map[string]string{ + "App": fmt.Sprintf("%s", podApp), + }) + + if data.networkType == networkTypeSriov { + computeResources := resource.NewQuantity(sriovResNum, resource.DecimalSI) + podBuilder = podBuilder.WithResources(corev1.ResourceList{sriovReqName: *computeResources}, corev1.ResourceList{sriovReqName: *computeResources}) + } + return podBuilder.Create(data.e2eTestData) +} + +// getSecondaryInterface checks the secondary interfaces created for the specific Pod and returns its IPv4 address. +func (data *testData) getSecondaryInterface(targetPod *testPodInfo, interfaceName string) (string, error) { + cmd := []string{"/bin/sh", "-c", fmt.Sprintf("ip addr show %s | grep 'inet ' | awk '{print $2}' | cut -d/ -f1", interfaceName)} + stdout, _, err := data.e2eTestData.RunCommandFromPod(data.e2eTestData.GetTestNamespace(), targetPod.podName, containerName, cmd) stdout = strings.TrimSuffix(stdout, "\n") - if stdout == "" { - log.Fatalf("Error: Interface %s not found on %s. err: %v", podData[targetPod].nameOfInterfacePerPod[targetInterface], podData[targetPod].nameOfPods, err) + if err != nil || stdout == "" { + return "", fmt.Errorf("interface %s not found on %s. err: %v", interfaceName, targetPod.podName, err) } return stdout, nil } // checkSubnet checks if the IP address to be pinged has the same subnet as the Pod from which the IP Address is pinged. -func (data *TestData) checkSubnet(t *testing.T, sourcePod int, targetPod int, targetInterface int) (bool, error) { - for podCheckForSubnet := 0; podCheckForSubnet < podData[sourcePod].countOfVirtualNetworksPerPod; podCheckForSubnet++ { - if podData[sourcePod].nameOfVirtualNetworkPerPod[podCheckForSubnet] == podData[targetPod].nameOfVirtualNetworkPerPod[targetInterface] { - _, err := data.getSecondaryInterface(sourcePod, podCheckForSubnet) +func (data *testData) checkSubnet(t *testing.T, sourcePod, targetPod *testPodInfo, targetNetwork string) (bool, error) { + for i, n := range sourcePod.interfaceNetworks { + if n == targetNetwork { + _, err := data.getSecondaryInterface(sourcePod, i) if err != nil { - t.Logf("Error in ping: Interface %s for the source test Pod %s not created", podData[sourcePod].nameOfInterfacePerPod[podCheckForSubnet], podData[sourcePod].nameOfPods) return false, err } + return true, nil } } - return true, nil + return false, nil } -// pingBetweenInterfaces parses through all the created Podsand pings the other Pod if the IP Address of the secondary network interface of the Pod is in the same subnet. Sleep time of 3 seconds is ensured for the successful ping between the pods. -func (data *TestData) pingBetweenInterfaces(t *testing.T) error { - for sourcePod := 0; sourcePod < totalNumberOfPods; sourcePod++ { - for targetPod := 0; targetPod < totalNumberOfPods; targetPod++ { - for targetInterface := 0; targetInterface < podData[targetPod].countOfVirtualNetworksPerPod; targetInterface++ { - if podData[targetPod].nameOfPods == podData[sourcePod].nameOfPods { - continue - } - _, err := data.e2eTestData.PodWaitFor(defaultTimeout, podData[targetPod].nameOfPods, nameSpace, func(pod *corev1.Pod) (bool, error) { +// pingBetweenInterfaces parses through all the created Pods and pings the other Pod if the two Pods +// both have a secondary network interface on the same network. +func (data *testData) pingBetweenInterfaces(t *testing.T) error { + e2eTestData := data.e2eTestData + namespace := e2eTestData.GetTestNamespace() + for _, sourcePod := range data.pods { + for _, targetPod := range data.pods { + if targetPod.podName == sourcePod.podName { + continue + } + for i, n := range targetPod.interfaceNetworks { + _, err := e2eTestData.PodWaitFor(defaultTimeout, targetPod.podName, namespace, func(pod *corev1.Pod) (bool, error) { return pod.Status.Phase == corev1.PodRunning, nil }) if err != nil { - t.Logf("Error when waiting for the perftest client Pod: %s", podData[targetPod].nameOfPods) + return fmt.Errorf("error when waiting for Pod %s: %v", targetPod.podName, err) } - flag, _ := data.checkSubnet(t, sourcePod, targetPod, targetInterface) - if flag != false { - secondaryIpAddress, _ := data.getSecondaryInterface(targetPod, targetInterface) - ip := net.ParseIP(secondaryIpAddress) - if ip != nil { - var IPToPing antreae2e.PodIPs - if ip.To4() != nil { - IPToPing = antreae2e.PodIPs{IPv4: &ip} - } else { - IPToPing = antreae2e.PodIPs{IPv6: &ip} - } - err := data.e2eTestData.RunPingCommandFromTestPod(antreae2e.PodInfo{Name: podData[sourcePod].nameOfPods, OS: osType, NodeName: clusterInfo.controlPlaneNodeName, Namespace: nameSpace}, nameSpace, &IPToPing, ctrName, count, size) - if err == nil { - logs.Infof("Ping '%s' -> '%s'( Interface: %s, IP Address: %s): OK", podData[sourcePod].nameOfPods, podData[targetPod].nameOfPods, podData[targetPod].nameOfInterfacePerPod[targetInterface], secondaryIpAddress) - } else { - t.Logf("Ping '%s' -> '%s'( Interface: %s, IP Address: %s): ERROR (%v)", podData[sourcePod].nameOfPods, podData[targetPod].nameOfPods, podData[targetPod].nameOfInterfacePerPod[targetInterface], secondaryIpAddress, err) - } + matched, _ := data.checkSubnet(t, sourcePod, targetPod, n) + if matched { + secondaryIPAddress, err := data.getSecondaryInterface(targetPod, i) + if err != nil { + return err + } + ip := net.ParseIP(secondaryIPAddress) + if ip == nil { + return fmt.Errorf("failed to parse IP (%s) for interface %s of Pod %s", secondaryIPAddress, i, targetPod.podName) + } + var IPToPing antreae2e.PodIPs + if ip.To4() != nil { + IPToPing = antreae2e.PodIPs{IPv4: &ip} } else { - t.Logf("Error in Ping: Target interface %v of %v Pod not created", podData[targetPod].nameOfInterfacePerPod[targetInterface], podData[targetPod].nameOfPods) + IPToPing = antreae2e.PodIPs{IPv6: &ip} + } + if err := e2eTestData.RunPingCommandFromTestPod(antreae2e.PodInfo{Name: sourcePod.podName, OS: osType, NodeName: sourcePod.nodeName, Namespace: namespace}, + namespace, &IPToPing, containerName, pingCount, pingSize); err != nil { + return fmt.Errorf("ping '%s' -> '%s'(Interface: %s, IP Address: %s) failed: %v", sourcePod.podName, targetPod.podName, i, secondaryIPAddress, err) } + logs.Infof("ping '%s' -> '%s'( Interface: %s, IP Address: %s): OK", sourcePod.podName, targetPod.podName, i, secondaryIPAddress) } } } @@ -214,32 +178,69 @@ func (data *TestData) pingBetweenInterfaces(t *testing.T) error { return nil } -// The Wrapper function createPodForSecondaryNetwork creates the Pod adding the annotation, arguments, commands, Node, container name, -// resource requests and limits as arguments with the NewPodBuilder API -func (data *TestData) createPodForSecondaryNetwork(ns string, nodeName string, podNum int, testPodName string, resNum int64) error { - computeResources := resource.NewQuantity(resNum, resource.DecimalSI) - return antreae2e.NewPodBuilder(podData[podNum].nameOfPods, ns, busyboxImage).OnNode(nodeName).WithContainerName(ctrName).WithCommand([]string{"sleep", "infinity"}).WithAnnotations( - map[string]string{ - "k8s.v1.cni.cncf.io/networks": fmt.Sprintf("%s", data.formAnnotationStringOfPod(podNum)), - }).WithLabels( - map[string]string{ - "App": fmt.Sprintf("%s", testPodName), - }).WithResources(corev1.ResourceList{reqName: *computeResources}, corev1.ResourceList{reqName: *computeResources}).Create(data.e2eTestData) -} - -func TestNativeSecondaryNetwork(t *testing.T) { - // once the setupTestWithSecondaryNetworkConfig is successful, we have all the prerequisites enabled and running. - _, err := testData.setupTestWithSecondaryNetworkConfig(t) +func testSecondaryNetwork(t *testing.T, networkType string, pods []*testPodInfo) { + e2eTestData, err := antreae2e.SetupTest(t) if err != nil { - t.Logf("Error when setupTestWithSecondaryNetworkConfig: %v", err) + t.Fatalf("Error when setting up test: %v", err) } + defer antreae2e.TeardownTest(t, e2eTestData) + + testData := &testData{e2eTestData: e2eTestData, networkType: networkType, pods: pods} + t.Run("testCreateTestPodOnNode", func(t *testing.T) { - testData.createPodOnNode(t, nameSpace, clusterInfo.controlPlaneNodeName) + testData.createPods(t, e2eTestData.GetTestNamespace()) }) t.Run("testpingBetweenInterfaces", func(t *testing.T) { err := testData.pingBetweenInterfaces(t) if err != nil { - t.Logf("Error when pinging between interfaces: %v", err) + t.Fatalf("Error when pinging between interfaces: %v", err) } }) } + +func TestSriovNetwork(t *testing.T) { + // Create Pods on the control plane Node, assuming a single Node cluster for the SR-IOV + // test. + controlPlaneNodeName := antreae2e.NodeName(0) + pods := []*testPodInfo{ + { + podName: "sriov-pod1", + nodeName: controlPlaneNodeName, + interfaceNetworks: map[string]string{"eth1": "sriov-net1", "eth2": "sriov-net2"}, + }, + { + podName: "sriov-pod2", + nodeName: controlPlaneNodeName, + interfaceNetworks: map[string]string{"eth2": "sriov-net1", "eth3": "sriov-net3"}, + }, + { + podName: "sriov-pod3", + nodeName: controlPlaneNodeName, + interfaceNetworks: map[string]string{"eth4": "sriov-net1"}, + }, + } + testSecondaryNetwork(t, networkTypeSriov, pods) +} + +func TestVLANNetwork(t *testing.T) { + node1 := antreae2e.NodeName(1) + node2 := antreae2e.NodeName(2) + pods := []*testPodInfo{ + { + podName: "vlan-pod1", + nodeName: node1, + interfaceNetworks: map[string]string{"eth1": "vlan-net1", "eth2": "vlan-net2"}, + }, + { + podName: "vlan-pod2", + nodeName: node1, + interfaceNetworks: map[string]string{"eth1": "vlan-net1", "eth2": "vlan-net3"}, + }, + { + podName: "vlan-pod3", + nodeName: node2, + interfaceNetworks: map[string]string{"eth1": "vlan-net2"}, + }, + } + testSecondaryNetwork(t, networkTypeVLAN, pods) +} diff --git a/test/e2e/entry.go b/test/e2e/entry.go new file mode 100644 index 00000000000..2f17249c7a1 --- /dev/null +++ b/test/e2e/entry.go @@ -0,0 +1,186 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "flag" + "fmt" + "log" + "os" + "testing" +) + +// setupLogging creates a temporary directory to export the test logs if necessary. If a directory +// was provided by the user, it checks that the directory exists. +func (tOptions *TestOptions) setupLogging() func() { + if tOptions.logsExportDir == "" { + name, err := os.MkdirTemp("", "antrea-test-") + if err != nil { + log.Fatalf("Error when creating temporary directory to export logs: %v", err) + } + log.Printf("Test logs (if any) will be exported under the '%s' directory", name) + tOptions.logsExportDir = name + // we will delete the temporary directory if no logs are exported + return func() { + if empty, _ := IsDirEmpty(name); empty { + log.Printf("Removing empty logs directory '%s'", name) + _ = os.Remove(name) + } else { + log.Printf("Logs exported under '%s', it is your responsibility to delete the directory when you no longer need it", name) + } + } + } + fInfo, err := os.Stat(tOptions.logsExportDir) + if err != nil { + log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.logsExportDir, err) + } + if !fInfo.Mode().IsDir() { + log.Fatalf("'%s' is not a valid directory", tOptions.logsExportDir) + } + // no-op cleanup function + return func() {} +} + +// setupCoverage checks if the directory provided by the user exists. +func (tOptions *TestOptions) setupCoverage(data *TestData) func() { + if tOptions.coverageDir != "" { + fInfo, err := os.Stat(tOptions.coverageDir) + if err != nil { + log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.coverageDir, err) + } + if !fInfo.Mode().IsDir() { + log.Fatalf("'%s' is not a valid directory", tOptions.coverageDir) + } + + } + // cpNodeCoverageDir is a directory on the control-plane Node, where tests can deposit test + // coverage data. + log.Printf("Creating directory '%s' on Node '%s'\n", cpNodeCoverageDir, controlPlaneNodeName()) + rc, _, _, err := data.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("mkdir -p %s", cpNodeCoverageDir)) + if err != nil || rc != 0 { + log.Fatalf("Failed to create directory '%s' on control-plane Node", cpNodeCoverageDir) + } + return func() { + log.Printf("Removing directory '%s' on Node '%s'\n", cpNodeCoverageDir, controlPlaneNodeName()) + // best effort + data.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("rm -rf %s", cpNodeCoverageDir)) + } + +} + +// testMain is meant to be called by TestMain and enables the use of defer statements. +func testMain(m *testing.M) int { + flag.StringVar(&testOptions.providerName, "provider", "vagrant", "K8s test cluster provider") + flag.StringVar(&testOptions.providerConfigPath, "provider-cfg-path", "", "Optional config file for provider") + flag.StringVar(&testOptions.logsExportDir, "logs-export-dir", "", "Export directory for test logs") + flag.BoolVar(&testOptions.logsExportOnSuccess, "logs-export-on-success", false, "Export logs even when a test is successful") + flag.BoolVar(&testOptions.withBench, "benchtest", false, "Run tests include benchmark tests") + flag.BoolVar(&testOptions.enableCoverage, "coverage", false, "Run tests and measure coverage") + flag.BoolVar(&testOptions.enableAntreaIPAM, "antrea-ipam", false, "Run tests with AntreaIPAM") + flag.BoolVar(&testOptions.flowVisibility, "flow-visibility", false, "Run flow visibility tests") + flag.BoolVar(&testOptions.deployAntrea, "deploy-antrea", true, "Deploy Antrea before running tests") + flag.StringVar(&testOptions.coverageDir, "coverage-dir", "", "Directory for coverage data files") + flag.StringVar(&testOptions.skipCases, "skip-cases", "", "Key words to skip cases") + flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") + flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs") + flag.Parse() + + cleanupLogging := testOptions.setupLogging() + defer cleanupLogging() + + testData = &TestData{} + if err := testData.InitProvider(testOptions.providerName, testOptions.providerConfigPath); err != nil { + log.Fatalf("Error when initializing provider: %v", err) + } + log.Println("Creating K8s ClientSet") + kubeconfigPath, err := testData.provider.GetKubeconfigPath() + if err != nil { + log.Fatalf("Error when getting Kubeconfig path: %v", err) + } + if err := testData.CreateClient(kubeconfigPath); err != nil { + log.Fatalf("Error when creating K8s ClientSet: %v", err) + } + log.Println("Collecting information about K8s cluster") + if err := testData.collectClusterInfo(); err != nil { + log.Fatalf("Error when collecting information about K8s cluster: %v", err) + } + if clusterInfo.podV4NetworkCIDR != "" { + log.Printf("Pod IPv4 network: '%s'", clusterInfo.podV4NetworkCIDR) + } + if clusterInfo.podV6NetworkCIDR != "" { + log.Printf("Pod IPv6 network: '%s'", clusterInfo.podV6NetworkCIDR) + } + if clusterInfo.svcV4NetworkCIDR != "" { + log.Printf("Service IPv4 network: '%s'", clusterInfo.svcV4NetworkCIDR) + } + if clusterInfo.svcV6NetworkCIDR != "" { + log.Printf("Service IPv6 network: '%s'", clusterInfo.svcV6NetworkCIDR) + } + log.Printf("Num nodes: %d", clusterInfo.numNodes) + err = ensureAntreaRunning(testData) + if err != nil { + log.Fatalf("Error when deploying Antrea: %v", err) + } + // Collect PodCIDRs after Antrea is running as Antrea is responsible for allocating PodCIDRs in some cases. + // Polling is not needed here because antrea-agents won't be up and running if PodCIDRs of their Nodes are not set. + if err = testData.collectPodCIDRs(); err != nil { + log.Fatalf("Error collecting PodCIDRs: %v", err) + } + AntreaConfigMap, err = testData.GetAntreaConfigMap(antreaNamespace) + if err != nil { + log.Fatalf("Error when getting antrea-config configmap: %v", err) + } + if testOptions.enableCoverage { + cleanupCoverage := testOptions.setupCoverage(testData) + defer cleanupCoverage() + defer gracefulExitAntrea(testData) + } + ret := m.Run() + return ret +} + +func gracefulExitAntrea(testData *TestData) { + if err := testData.gracefulExitAntreaController(testOptions.coverageDir); err != nil { + log.Fatalf("Error when gracefully exit antrea controller: %v", err) + } + if err := testData.gracefulExitAntreaAgent(testOptions.coverageDir, "all"); err != nil { + log.Fatalf("Error when gracefully exit antrea agent: %v", err) + } + if err := testData.collectAntctlCovFilesFromControlPlaneNode(testOptions.coverageDir); err != nil { + log.Fatalf("Error when collecting antctl coverage files from control-plane Node: %v", err) + } +} + +// The following funcs are used in e2e-secondary-network. + +func RunTests(m *testing.M) int { + return testMain(m) +} + +func SetupTest(tb testing.TB) (*TestData, error) { + return setupTest(tb) +} + +func TeardownTest(tb testing.TB, data *TestData) { + teardownTest(tb, data) +} + +func NodeName(idx int) string { + return nodeName(idx) +} + +func (data *TestData) GetTestNamespace() string { + return data.testNamespace +} diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 45af883bccd..9a699e1cd32 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -253,10 +253,10 @@ func setupTest(tb testing.TB) (*TestData, error) { name = replacer.Replace(name) name = strings.ToLower(name) testData.testNamespace = randName(name + "-") - tb.Logf("Creating '%s' K8s Namespace", testData.testNamespace) if err := ensureAntreaRunning(testData); err != nil { return nil, err } + tb.Logf("Creating '%s' K8s Namespace", testData.testNamespace) if err := testData.CreateNamespace(testData.testNamespace, nil); err != nil { return nil, err } diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 7973fb7f9d2..dd5a1fb7d61 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -124,20 +124,19 @@ const ( antreaControllerConfName = "antrea-controller.conf" flowAggregatorConfName = "flow-aggregator.conf" - nameSuffixLength = 8 + BusyboxImage = "projects.registry.vmware.com/antrea/busybox" agnhostImage = "registry.k8s.io/e2e-test-images/agnhost:2.29" - busyboxImage = "projects.registry.vmware.com/antrea/busybox" mcjoinImage = "projects.registry.vmware.com/antrea/mcjoin:v2.9" nginxImage = "projects.registry.vmware.com/antrea/nginx:1.21.6-alpine" iisImage = "mcr.microsoft.com/windows/servercore/iis" toolboxImage = "projects.registry.vmware.com/antrea/toolbox:1.2-1" ipfixCollectorImage = "projects.registry.vmware.com/antrea/ipfix-collector:v0.6.2" - ipfixCollectorPort = "4739" - clickHouseHTTPPort = "8123" nginxLBService = "nginx-loadbalancer" + ipfixCollectorPort = "4739" + clickHouseHTTPPort = "8123" exporterFlowPollInterval = 1 * time.Second exporterActiveFlowExportTimeout = 2 * time.Second exporterIdleFlowExportTimeout = 1 * time.Second @@ -1450,7 +1449,7 @@ func (data *TestData) UpdatePod(namespace, name string, mutateFunc func(*corev1. // createBusyboxPodOnNode creates a Pod in the test namespace with a single busybox container. The // Pod will be scheduled on the specified Node (if nodeName is not empty). func (data *TestData) createBusyboxPodOnNode(name string, ns string, nodeName string, hostNetwork bool) error { - return NewPodBuilder(name, ns, busyboxImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).Create(data) + return NewPodBuilder(name, ns, BusyboxImage).OnNode(nodeName).WithCommand([]string{"sleep", "3600"}).WithHostNetwork(hostNetwork).Create(data) } // createMcJoinPodOnNode creates a Pod in the test namespace with a single mcjoin container. The @@ -2067,10 +2066,6 @@ func (data *TestData) deleteNetworkpolicy(policy *networkingv1.NetworkPolicy) er return nil } -func RandName(prefix string) string { - return prefix + randSeq(nameSuffixLength) -} - // A DNS-1123 subdomain must consist of lower case alphanumeric characters var lettersAndDigits = []rune("abcdefghijklmnopqrstuvwxyz0123456789") @@ -2086,6 +2081,7 @@ func randSeq(n int) string { // randName generates a DNS-1123 subdomain name func randName(prefix string) string { + nameSuffixLength := 8 return prefix + randSeq(nameSuffixLength) } diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index a1cd1ca0eb6..2d590e333f3 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -15,154 +15,10 @@ package e2e import ( - "flag" - "fmt" - "log" "os" "testing" ) -// setupLogging creates a temporary directory to export the test logs if necessary. If a directory -// was provided by the user, it checks that the directory exists. -func (tOptions *TestOptions) setupLogging() func() { - if tOptions.logsExportDir == "" { - name, err := os.MkdirTemp("", "antrea-test-") - if err != nil { - log.Fatalf("Error when creating temporary directory to export logs: %v", err) - } - log.Printf("Test logs (if any) will be exported under the '%s' directory", name) - tOptions.logsExportDir = name - // we will delete the temporary directory if no logs are exported - return func() { - if empty, _ := IsDirEmpty(name); empty { - log.Printf("Removing empty logs directory '%s'", name) - _ = os.Remove(name) - } else { - log.Printf("Logs exported under '%s', it is your responsibility to delete the directory when you no longer need it", name) - } - } - } - fInfo, err := os.Stat(tOptions.logsExportDir) - if err != nil { - log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.logsExportDir, err) - } - if !fInfo.Mode().IsDir() { - log.Fatalf("'%s' is not a valid directory", tOptions.logsExportDir) - } - // no-op cleanup function - return func() {} -} - -// setupCoverage checks if the directory provided by the user exists. -func (tOptions *TestOptions) setupCoverage(data *TestData) func() { - if tOptions.coverageDir != "" { - fInfo, err := os.Stat(tOptions.coverageDir) - if err != nil { - log.Fatalf("Cannot stat provided directory '%s': %v", tOptions.coverageDir, err) - } - if !fInfo.Mode().IsDir() { - log.Fatalf("'%s' is not a valid directory", tOptions.coverageDir) - } - - } - // cpNodeCoverageDir is a directory on the control-plane Node, where tests can deposit test - // coverage data. - log.Printf("Creating directory '%s' on Node '%s'\n", cpNodeCoverageDir, controlPlaneNodeName()) - rc, _, _, err := data.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("mkdir -p %s", cpNodeCoverageDir)) - if err != nil || rc != 0 { - log.Fatalf("Failed to create directory '%s' on control-plane Node", cpNodeCoverageDir) - } - return func() { - log.Printf("Removing directory '%s' on Node '%s'\n", cpNodeCoverageDir, controlPlaneNodeName()) - // best effort - data.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("rm -rf %s", cpNodeCoverageDir)) - } - -} - -// testMain is meant to be called by TestMain and enables the use of defer statements. -func testMain(m *testing.M) int { - flag.StringVar(&testOptions.providerName, "provider", "vagrant", "K8s test cluster provider") - flag.StringVar(&testOptions.providerConfigPath, "provider-cfg-path", "", "Optional config file for provider") - flag.StringVar(&testOptions.logsExportDir, "logs-export-dir", "", "Export directory for test logs") - flag.BoolVar(&testOptions.logsExportOnSuccess, "logs-export-on-success", false, "Export logs even when a test is successful") - flag.BoolVar(&testOptions.withBench, "benchtest", false, "Run tests include benchmark tests") - flag.BoolVar(&testOptions.enableCoverage, "coverage", false, "Run tests and measure coverage") - flag.BoolVar(&testOptions.enableAntreaIPAM, "antrea-ipam", false, "Run tests with AntreaIPAM") - flag.BoolVar(&testOptions.flowVisibility, "flow-visibility", false, "Run flow visibility tests") - flag.BoolVar(&testOptions.deployAntrea, "deploy-antrea", true, "Deploy Antrea before running tests") - flag.StringVar(&testOptions.coverageDir, "coverage-dir", "", "Directory for coverage data files") - flag.StringVar(&testOptions.skipCases, "skip-cases", "", "Key words to skip cases") - flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") - flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs") - flag.Parse() - - cleanupLogging := testOptions.setupLogging() - defer cleanupLogging() - - testData = &TestData{} - if err := testData.InitProvider(testOptions.providerName, testOptions.providerConfigPath); err != nil { - log.Fatalf("Error when initializing provider: %v", err) - } - log.Println("Creating K8s ClientSet") - kubeconfigPath, err := testData.provider.GetKubeconfigPath() - if err != nil { - log.Fatalf("Error when getting Kubeconfig path: %v", err) - } - if err := testData.CreateClient(kubeconfigPath); err != nil { - log.Fatalf("Error when creating K8s ClientSet: %v", err) - } - log.Println("Collecting information about K8s cluster") - if err := testData.collectClusterInfo(); err != nil { - log.Fatalf("Error when collecting information about K8s cluster: %v", err) - } - if clusterInfo.podV4NetworkCIDR != "" { - log.Printf("Pod IPv4 network: '%s'", clusterInfo.podV4NetworkCIDR) - } - if clusterInfo.podV6NetworkCIDR != "" { - log.Printf("Pod IPv6 network: '%s'", clusterInfo.podV6NetworkCIDR) - } - if clusterInfo.svcV4NetworkCIDR != "" { - log.Printf("Service IPv4 network: '%s'", clusterInfo.svcV4NetworkCIDR) - } - if clusterInfo.svcV6NetworkCIDR != "" { - log.Printf("Service IPv6 network: '%s'", clusterInfo.svcV6NetworkCIDR) - } - log.Printf("Num nodes: %d", clusterInfo.numNodes) - err = ensureAntreaRunning(testData) - if err != nil { - log.Fatalf("Error when deploying Antrea: %v", err) - } - // Collect PodCIDRs after Antrea is running as Antrea is responsible for allocating PodCIDRs in some cases. - // Polling is not needed here because antrea-agents won't be up and running if PodCIDRs of their Nodes are not set. - if err = testData.collectPodCIDRs(); err != nil { - log.Fatalf("Error collecting PodCIDRs: %v", err) - } - AntreaConfigMap, err = testData.GetAntreaConfigMap(antreaNamespace) - if err != nil { - log.Fatalf("Error when getting antrea-config configmap: %v", err) - } - if testOptions.enableCoverage { - cleanupCoverage := testOptions.setupCoverage(testData) - defer cleanupCoverage() - defer gracefulExitAntrea(testData) - } - ret := m.Run() - return ret -} - -func gracefulExitAntrea(testData *TestData) { - if err := testData.gracefulExitAntreaController(testOptions.coverageDir); err != nil { - log.Fatalf("Error when gracefully exit antrea controller: %v", err) - } - if err := testData.gracefulExitAntreaAgent(testOptions.coverageDir, "all"); err != nil { - log.Fatalf("Error when gracefully exit antrea agent: %v", err) - } - if err := testData.collectAntctlCovFilesFromControlPlaneNode(testOptions.coverageDir); err != nil { - log.Fatalf("Error when collecting antctl coverage files from control-plane Node: %v", err) - } -} - func TestMain(m *testing.M) { os.Exit(testMain(m)) } diff --git a/test/e2e/wireguard_test.go b/test/e2e/wireguard_test.go index b746ffe07a7..f39e6565751 100644 --- a/test/e2e/wireguard_test.go +++ b/test/e2e/wireguard_test.go @@ -141,7 +141,7 @@ func testServiceConnectivity(t *testing.T, data *TestData) { defer cleanup() // Create the a hostNetwork Pod on a Node different from the service's backend Pod, so the service traffic will be transferred across the tunnel. - require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, busyboxImage).OnNode(clientPodNode).WithCommand([]string{"sleep", "3600"}).InHostNetwork().Create(data)) + require.NoError(t, NewPodBuilder(clientPodName, data.testNamespace, BusyboxImage).OnNode(clientPodNode).WithCommand([]string{"sleep", "3600"}).InHostNetwork().Create(data)) defer data.DeletePodAndWait(defaultTimeout, clientPodName, data.testNamespace) require.NoError(t, data.podWaitForRunning(defaultTimeout, clientPodName, data.testNamespace))