Skip to content

Commit

Permalink
Add e2e test for VLAN secondary network (#5745)
Browse files Browse the repository at this point in the history
Extend secondary network e2e tests to support testing secondary VLAN
networks. Add a script to run the test on a Kind cluster, and enable
the test on Github CI.

Signed-off-by: Jianjun Shen <shenj@vmware.com>
  • Loading branch information
jianjuns authored Feb 15, 2024
1 parent a5e2792 commit 1fa4883
Show file tree
Hide file tree
Showing 24 changed files with 751 additions and 602 deletions.
44 changes: 44 additions & 0 deletions .github/workflows/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,50 @@ jobs:
path: log.tar.gz
retention-days: 30

test-secondary-network:
name: Antrea-native (VLAN) secondary network tests on a Kind cluster on Linux
needs: [build-antrea-coverage-image]
runs-on: [ubuntu-latest]
steps:
- name: Free disk space
# https://github.com/actions/virtual-environments/issues/709
run: |
sudo apt-get clean
df -h
- uses: actions/checkout@v4
with:
show-progress: false
- uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
- name: Download Antrea image from previous job
uses: actions/download-artifact@v4
with:
name: antrea-ubuntu-cov
- name: Load Antrea image
run: |
docker load -i antrea-ubuntu.tar
- name: Install Kind
run: |
KIND_VERSION=$(head -n1 ./ci/kind/version)
curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
chmod +x ./kind
sudo mv kind /usr/local/bin
- name: Run secondary network tests
run: |
mkdir log
ANTREA_LOG_DIR=$PWD/log ./ci/kind/test-secondary-network-kind.sh
- name: Tar log files
if: ${{ failure() }}
run: tar -czf log.tar.gz log
- name: Upload test log
uses: actions/upload-artifact@v4
if: ${{ failure() }}
with:
name: secondary-network.tar.gz
path: log.tar.gz
retention-days: 30

test-upgrade-from-N-1:
name: Upgrade from Antrea version N-1
needs: build-antrea-coverage-image
Expand Down
2 changes: 1 addition & 1 deletion build/charts/antrea/conf/antrea-agent.conf
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ secondaryNetwork:
{{- with .Values.secondaryNetwork }}
# Configuration of OVS bridges for secondary network.
ovsBridges:
{{- toYaml .ovsBridges | trim | nindent 6 }}
{{- toYaml .ovsBridges | trim | nindent 4 }}
{{- end }}

{{- end }}
2 changes: 1 addition & 1 deletion ci/kind/kind-setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ function configure_extra_networks {
i=$((i+1))
done

nodes="$(kind get nodes --name $CLUSTER_NAME | grep worker)"
nodes="$(kind get nodes --name $CLUSTER_NAME)"
for node in $nodes; do
for network in $networks; do
docker network connect $network $node >/dev/null 2>&1
Expand Down
2 changes: 1 addition & 1 deletion ci/kind/test-e2e-kind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ fi
COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.29" \
"projects.registry.vmware.com/antrea/busybox" \
"projects.registry.vmware.com/antrea/nginx:1.21.6-alpine" \
"projects.registry.vmware.com/antrea/toolbox:1.1-0")
"projects.registry.vmware.com/antrea/toolbox:1.3-0")

FLOW_VISIBILITY_IMAGE_LIST=("projects.registry.vmware.com/antrea/ipfix-collector:v0.8.2" \
"projects.registry.vmware.com/antrea/clickhouse-operator:0.21.0" \
Expand Down
126 changes: 126 additions & 0 deletions ci/kind/test-secondary-network-kind.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
#!/usr/bin/env bash

# Copyright 2024 Antrea Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eo pipefail

function echoerr {
>&2 echo "$@"
}

_usage="Usage: $0 [--setup-only|--test-only|--cleanup-only] [--help|-h]
--setup-only Only perform setting up the cluster and run test.
--test-only Only run test on current cluster. Not set up/clean up the cluster.
--cleanup-only Only perform cleaning up the cluster.
--help, -h Print this message and exit.
"

function print_usage {
echoerr -n "$_usage"
}

THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
TESTBED_CMD=$THIS_DIR"/kind-setup.sh"
ATTACHMENT_DEFINITION_YAML=$THIS_DIR"/../../test/e2e-secondary-network/infra/network-attachment-definition-crd.yml"
SECONDARY_NETWORKS_YAML=$THIS_DIR"/../../test/e2e-secondary-network/infra/secondary-networks.yml"
ANTREA_CHART="$THIS_DIR/../../build/charts/antrea"

TIMEOUT="5m"
# Antrea is deployed by this script. Do not deploy it again in the test.
TEST_OPTIONS="--logs-export-dir=$ANTREA_LOG_DIR --deploy-antrea=false"
EXTRA_NETWORK="20.20.20.0/24"

setup_only=false
cleanup_only=false
test_only=false

function quit {
result=$?
if [[ $setup_only || $test_only ]]; then
exit $result
fi
echoerr "Cleaning testbed"
$TESTBED_CMD destroy kind
}

while [[ $# -gt 0 ]]
do
key="$1"

case $key in
--setup-only)
setup_only=true
shift
;;
--cleanup-only)
cleanup_only=true
shift
;;
--test-only)
test_only=true
shift
;;
-h|--help)
print_usage
exit 0
;;
*) # unknown option
echoerr "Unknown option $1"
exit 1
;;
esac
done

if [[ $cleanup_only == "true" ]];then
$TESTBED_CMD destroy kind
exit 0
fi

trap "quit" INT EXIT

IMAGE_LIST=("projects.registry.vmware.com/antrea/toolbox:1.3-0" \
"antrea/antrea-agent-ubuntu:latest" \
"antrea/antrea-controller-ubuntu:latest")

printf -v IMAGES "%s " "${IMAGE_LIST[@]}"

function setup_cluster {
args=$1
echo "creating test bed with args $args"
eval "timeout 600 $TESTBED_CMD create kind $args"
}

function run_test {
echo "deploying Antrea to the kind cluster"
# Create a secondary OVS bridge with the Node's physical interface on the extra
# network.
helm install antrea $ANTREA_CHART --namespace kube-system \
--set featureGates.SecondaryNetwork=true,featureGates.AntreaIPAM=true \
--set-json secondaryNetwork.ovsBridges='[{"bridgeName": "br-secondary", "physicalInterfaces": ["eth1"]}]'

# Wait for antrea-controller start to make sure the IPPool validation webhook is ready.
kubectl rollout status --timeout=1m deployment.apps/antrea-controller -n kube-system
kubectl apply -f $ATTACHMENT_DEFINITION_YAML
kubectl apply -f $SECONDARY_NETWORKS_YAML

go test -v -timeout=$TIMEOUT antrea.io/antrea/test/e2e-secondary-network -run=TestVLANNetwork -provider=kind $TEST_OPTIONS
}

echo "======== Testing Antrea-native secondary network support =========="
if [[ $test_only == "false" ]];then
setup_cluster "--extra-networks \"$EXTRA_NETWORK\" --images \"$IMAGES\" --num-workers 1"
fi
run_test
exit 0
82 changes: 54 additions & 28 deletions hack/generate-manifest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,30 +20,32 @@ function echoerr {
>&2 echo "$@"
}

_usage="Usage: $0 [--mode (dev|release)] [--encap-mode] [--ipsec] [--tun (geneve|vxlan|gre|stt)] [--verbose-log] [--help|-h]
_usage="Usage: $0 [--mode (dev|release)] [--encap-mode (mode)] [--ipsec] [--tun (geneve|vxlan|gre|stt)] [--verbose-log]
Generate a YAML manifest for Antrea using Helm and print it to stdout.
--mode (dev|release) Choose the configuration variant that you need (default is 'dev')
--encap-mode Traffic encapsulation mode. (default is 'encap')
--cloud Generate a manifest appropriate for running Antrea in Public Cloud
--ipsec Generate a manifest with IPsec encryption of tunnel traffic enabled
--mode (dev|release) Choose the configuration variant that you need (default is 'dev').
--encap-mode (mode) Traffic encapsulation mode (default is 'encap').
--cloud Generate a manifest appropriate for running Antrea in Public Cloud.
--ipsec Generate a manifest with IPsec encryption of tunnel traffic enabled.
--feature-gates A comma-separated list of key=value pairs that describe feature gates, e.g. TrafficControl=true,Egress=false.
--proxy-all Generate a manifest with Antrea proxy with all Service support enabled
--tun (geneve|vxlan|gre|stt) Choose encap tunnel type from geneve, gre, stt and vxlan (default is geneve)
--verbose-log Generate a manifest with increased log-level (level 4) for Antrea agent and controller.
This option will work only in 'dev' mode.
--proxy-all Generate a manifest with Antrea proxy with all Service support enabled.
--tun (geneve|vxlan|gre|stt) Choose encap tunnel type from geneve, gre, stt and vxlan (default is geneve).
--on-delete Generate a manifest with antrea-agent's update strategy set to OnDelete.
This option will work only in 'dev' mode.
--coverage Generates a manifest which supports measuring code coverage of Antrea binaries.
--simulator Generates a manifest with antrea-agent simulator included
--custom-adm-controller Generates a manifest with custom Antrea admission controller to validate/mutate resources.
--hw-offload Generates a manifest with hw-offload enabled in the antrea-ovs container.
--sriov Generates a manifest which enables use of Kubelet API for SR-IOV device info.
--flexible-ipam Generates a manifest with flexible IPAM enabled.
--help, -h Print this message and exit
--multicast Generates a manifest for multicast.
--multicast-interfaces Multicast interface names (default is empty)
--extra-helm-values-file Optional extra helm values file to override the default config values
--extra-helm-values Optional extra helm values to override the default config values
--coverage Generate a manifest which supports measuring code coverage of Antrea binaries.
--simulator Generate a manifest with antrea-agent simulator included.
--custom-adm-controller Generate a manifest with custom Antrea admission controller to validate/mutate resources.
--flexible-ipam Generate a manifest with flexible IPAM enabled.
--hw-offload Generate a manifest with hw-offload enabled in the antrea-ovs container.
--sriov Generate a manifest which enables use of kubelet API for SR-IOV device info.
--secondary-bridge (bridge) Generate a manifest which enables secondary network and creates a secondary OVS bridge.
--physical-interface (device) Specify the physical interface of the secondary OVS bridge.
--multicast Generate a manifest for multicast.
--multicast-interfaces Multicast interface names (default is empty).
--extra-helm-values-file Optional extra helm values file to override the default config values.
--extra-helm-values Optional extra helm values to override the default config values.
--verbose-log Generate a manifest with increased log-level (level 4) for Antrea agent and controller.
This option will work only in 'dev' mode.
--help, -h Print this message and exit.
In 'release' mode, environment variables AGENT_IMG_NAME, CONTROLLER_IMG_NAME, and IMG_TAG must be set.
Expand Down Expand Up @@ -75,9 +77,11 @@ COVERAGE=false
K8S_115=false
SIMULATOR=false
CUSTOM_ADM_CONTROLLER=false
FLEXIBLE_IPAM=false
HW_OFFLOAD=false
SRIOV=false
FLEXIBLE_IPAM=false
SECONDARY_BRIDGE=""
PHYSICAL_INTERFACE=""
MULTICAST=false
MULTICAST_INTERFACES=""
HELM_VALUES_FILES=()
Expand Down Expand Up @@ -126,10 +130,6 @@ case $key in
TUN_TYPE="$2"
shift 2
;;
--verbose-log)
VERBOSE_LOG=true
shift
;;
--on-delete)
ON_DELETE=true
shift
Expand All @@ -146,6 +146,10 @@ case $key in
CUSTOM_ADM_CONTROLLER=true
shift
;;
--flexible-ipam)
FLEXIBLE_IPAM=true
shift
;;
--hw-offload)
HW_OFFLOAD=true
shift
Expand All @@ -154,9 +158,13 @@ case $key in
SRIOV=true
shift
;;
--flexible-ipam)
FLEXIBLE_IPAM=true
shift
--secondary-bridge)
SECONDARY_BRIDGE="$2"
shift 2
;;
--physical-interface)
PHYSICAL_INTERFACE="$2"
shift 2
;;
--multicast)
MULTICAST=true
Expand All @@ -178,6 +186,10 @@ case $key in
HELM_VALUES+=("$2")
shift 2
;;
--verbose-log)
VERBOSE_LOG=true
shift
;;
-h|--help)
print_usage
exit 0
Expand Down Expand Up @@ -322,6 +334,20 @@ EOF
HELM_VALUES_FILES+=("$TMP_DIR/sriov.yml")
fi

if [[ $SECONDARY_BRIDGE != "" ]]; then
if [[ $PHYSICAL_INTERFACE != "" ]]; then
ovs_bridges="[{bridgeName: $SECONDARY_BRIDGE, physicalInterfaces: [$PHYSICAL_INTERFACE]}]"
else
ovs_bridges="[{bridgeName: \"$SECONDARY_BRIDGE\"}]"
fi
cat << EOF > $TMP_DIR/secondary-network.yml
secondaryNetwork:
ovsBridges: $ovs_bridges
EOF
HELM_VALUES+=("featureGates.SecondaryNetwork=true" "featureGates.AntreaIPAM=true")
HELM_VALUES_FILES+=("$TMP_DIR/secondary-network.yml")
fi

if [ "$MODE" == "dev" ]; then
if [[ -z "$AGENT_IMG_NAME" ]]; then
if $COVERAGE; then
Expand Down
50 changes: 0 additions & 50 deletions test/e2e-secondary-network/framework.go

This file was deleted.

Loading

0 comments on commit 1fa4883

Please sign in to comment.