Skip to content

Commit

Permalink
Support deploying an FRR container in Kind network
Browse files Browse the repository at this point in the history 
This commit introduces a new parameter `--deploy-external-frr`
in `ci/kind/kind-setup.sh`, enabling to deploy an FRR container in
the Kind cluster network. This serves as the foundation for running
BGPPolicy e2e tests.

Signed-off-by: Hongliang Liu <lhongliang@vmware.com>
  • Loading branch information
@hongliangl
hongliangl committed Jul 12, 2024
1 parent 5cee770 commit 7a7a8bc
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 32 deletions.
49 changes: 36 additions & 13 deletions ci/kind/kind-setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ KUBE_PROXY_MODE="iptables"
PROMETHEUS=false
K8S_VERSION=""
KUBE_NODE_IPAM=true
DEPLOY_EXTERNAL_SERVER=false
DEPLOY_EXTERNAL_AGNHOST=false
DEPLOY_EXTERNAL_FRR=false
positional_args=()
options=()

Expand Down Expand Up @@ -79,8 +80,9 @@ where:
network. Note, '--extra-networks' and '--subnets' cannot be specified together.
--ip-family: specify the ip-family for the kind cluster, default is $IP_FAMILY.
--k8s-version: specify the Kubernetes version of the kind cluster, kind's default K8s version will be used if empty.
--deploy-external-server: deploy a container running as an external server for the cluster.
--all: delete all kind clusters
--deploy-external-agnhost: deploy a container running agnhost as an external server for the cluster, default is $DEPLOY_EXTERNAL_AGNHOST.
--deploy-external-frr: deploy a container running FRR as an external router for the cluster, default is $DEPLOY_EXTERNAL_FRR.
--all: delete all kind clusters.
--until: delete kind clusters that have been created before the specified duration.
"

Expand Down Expand Up @@ -411,7 +413,7 @@ EOF
configure_networks
configure_extra_networks
configure_vlan_subnets
setup_external_server
setup_external_servers
load_images

if [[ $ANTREA_CNI == true ]]; then
Expand Down Expand Up @@ -441,7 +443,7 @@ function destroy {
else
kind delete cluster --name $CLUSTER_NAME
fi
destroy_external_server
destroy_external_servers
delete_networks
delete_vlan_subnets
}
Expand All @@ -456,14 +458,30 @@ function printUnixTimestamp {
}

function setup_external_server {
if [[ $DEPLOY_EXTERNAL_SERVER == true ]]; then
docker run -d --name antrea-external-server-$RANDOM --network kind -it --rm registry.k8s.io/e2e-test-images/agnhost:2.40 netexec &> /dev/null
if [[ $DEPLOY_EXTERNAL_AGNHOST == true ]]; then
docker run -d --name antrea-external-agnhost-$RANDOM --network kind -it --rm registry.k8s.io/e2e-test-images/agnhost:2.40 netexec &> /dev/null
fi

if [[ $DEPLOY_EXTERNAL_FRR == true ]]; then
docker run -d \
--name antrea-external-frr-$RANDOM \
--network kind --cap-add=NET_BIND_SERVICE \
--cap-add=NET_ADMIN \
--cap-add=NET_RAW \
--cap-add=SYS_ADMIN \
-it \
--rm \
frrouting/frr:v8.4.0 \
bash -c "/bin/sed -i s/bgpd=no/bgpd=yes/g /etc/frr/daemons && /sbin/tini -- /usr/lib/frr/docker-start" &> /dev/null
fi
}

function destroy_external_server {
echo "Deleting external server"
cid=$(docker ps -f name="^antrea-external-server" --format '{{.ID}}')
function destroy_external_servers {
echo "Deleting external servers"
cid=$(docker ps -f name="^antrea-external-agnhost" --format '{{.ID}}')
docker rm -f $cid &> /dev/null || true

cid=$(docker ps -f name="^antrea-external-frr" --format '{{.ID}}')
docker rm -f $cid &> /dev/null || true
}

Expand Down Expand Up @@ -583,9 +601,14 @@ while [[ $# -gt 0 ]]
K8S_VERSION="$2"
shift 2
;;
--deploy-external-server)
add_option "--deploy-external-server" "create"
DEPLOY_EXTERNAL_SERVER=true
--deploy-external-agnhost)
add_option "--deploy-external-agnhost" "create"
DEPLOY_EXTERNAL_AGNHOST=true
shift
;;
--deploy-external-frr)
add_option "--deploy-external-frr" "create"
DEPLOY_EXTERNAL_FRR=true
shift
;;
--all)
Expand Down
45 changes: 32 additions & 13 deletions ci/kind/test-e2e-kind.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,21 @@ function echoerr {

_usage="Usage: $0 [--encap-mode <mode>] [--ip-family <v4|v6|dual>] [--coverage] [--help|-h]
--encap-mode Traffic encapsulation mode. (default is 'encap').
--ip-family Configures the ipFamily for the KinD cluster.
--ip-family Configure the ipFamily for the KinD cluster.
--feature-gates A comma-separated list of key=value pairs that describe feature gates, e.g. AntreaProxy=true,Egress=false.
--run Run only tests matching the regexp.
--proxy-all Enables Antrea proxy with all Service support.
--proxy-all Enable Antrea proxy with all Service support.
--no-kube-proxy Don't deploy kube-proxy.
--load-balancer-mode LoadBalancer mode.
--node-ipam Enables Antrea NodeIPAM.
--multicast Enables Multicast.
--node-ipam Enable Antrea NodeIPAM.
--multicast Enable Multicast.
--bgp-policy Enable Antrea BGPPolicy.
--flow-visibility Only run flow visibility related e2e tests.
--networkpolicy-evaluation Configures additional NetworkPolicy evaluation level when running e2e tests.
--extra-network Creates an extra network that worker Nodes will connect to. Cannot be specified with the hybrid mode.
--extra-vlan Creates an subnet-based VLAN that worker Nodes will connect to.
--networkpolicy-evaluation Configure additional NetworkPolicy evaluation level when running e2e tests.
--extra-network Create an extra network that worker Nodes will connect to. Cannot be specified with the hybrid mode.
--extra-vlan Create an subnet-based VLAN that worker Nodes will connect to.
--skip A comma-separated list of keywords, with which tests should be skipped.
--coverage Enables measure Antrea code coverage when running e2e tests on kind.
--coverage Enable measure Antrea code coverage when running e2e tests on kind.
--setup-only Only perform setting up the cluster and run test.
--cleanup-only Only perform cleaning up the cluster.
--test-only Only run test on current cluster. Not set up/clean up the cluster.
Expand Down Expand Up @@ -77,6 +78,7 @@ no_kube_proxy=false
load_balancer_mode=""
node_ipam=false
multicast=false
bgp_policy=false
flow_visibility=false
np_evaluation=false
extra_network=false
Expand Down Expand Up @@ -124,6 +126,10 @@ case $key in
multicast=true
shift
;;
--bgp-policy)
bgp_policy=true
shift
;;
--ip-family)
ipfamily="$2"
shift 2
Expand Down Expand Up @@ -236,6 +242,9 @@ fi
if $multicast; then
manifest_args="$manifest_args --multicast"
fi
if $bgp_policy; then
manifest_args="$manifest_args --feature-gates BGPPolicy=true"
fi
if $flow_visibility; then
manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES"
fi
Expand Down Expand Up @@ -314,8 +323,12 @@ function setup_cluster {
if $extra_network && [[ "$mode" != "hybrid" ]]; then
args="$args --extra-networks \"20.20.30.0/24\""
fi
# Deploy an external server which could be used when testing Pod-to-External traffic.
args="$args --deploy-external-server $vlan_args"
# Deploy an external agnhost which could be used when testing Pod-to-External traffic.
args="$args --deploy-external-agnhost $vlan_args"
# Deploy an external FRR which could be used when testing BGPPolicy.
if $bgp_policy; then
args="$args --deploy-external-frr"
fi

echo "creating test bed with args $args"
eval "timeout 600 $TESTBED_CMD create kind $args"
Expand Down Expand Up @@ -379,9 +392,15 @@ function run_test {
np_evaluation_flag="--networkpolicy-evaluation"
fi

external_server_cid=$(docker ps -f name="^antrea-external-server" --format '{{.ID}}')
external_server_ips=$(docker inspect $external_server_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')
EXTRA_ARGS="$vlan_args --external-server-ips $external_server_ips"
external_agnhost_cid=$(docker ps -f name="^antrea-external-agnhost" --format '{{.ID}}')
external_agnhost_ips=$(docker inspect $external_agnhost_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')
EXTRA_ARGS="$vlan_args --external-agnhost-ips $external_agnhost_ips"

if $bgp_policy; then
external_frr_cid=$(docker ps -f name="^antrea-external-frr" --format '{{.ID}}')
external_frr_ips=$(docker inspect $external_frr_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')
EXTRA_ARGS="$EXTRA_ARGS --external-frr-cid $external_frr_cid --external-frr-ips $external_frr_ips"
fi

go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS

Expand Down
4 changes: 3 additions & 1 deletion test/e2e/fixtures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -685,7 +685,9 @@ func testMain(m *testing.M) int {
flag.StringVar(&testOptions.skipCases, "skip-cases", "", "Key words to skip cases")
flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs")
flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs")
flag.StringVar(&testOptions.externalServerIPs, "external-server-ips", "", "IP addresses of external server, at most one IP per IP family")
flag.StringVar(&testOptions.externalAgnhostIPs, "external-agnhost-ips", "", "IP addresses of external agnhost, at most one IP per IP family")
flag.StringVar(&testOptions.externalFRRIPs, "external-frr-ips", "", "IP addresses of external FRR, at most one IP per IP family")
flag.StringVar(&testOptions.externalFRRCID, "external-frr-cid", "", "Container ID of external FRR")
flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family")
flag.IntVar(&testOptions.vlanID, "vlan-id", 0, "ID of the VLAN network the Nodes reside in")
flag.Parse()
Expand Down
40 changes: 35 additions & 5 deletions test/e2e/framework.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ type ExternalInfo struct {
vlanSubnetIPv6 string
vlanGatewayIPv6 string
vlanID int

externalFRRIPv4 string
externalFRRIPv6 string
externalFRRCID string
}

var clusterInfo ClusterInfo
Expand All @@ -213,9 +217,16 @@ type TestOptions struct {
// the home directory of the control-plane Node. Note it doesn't affect the tests that redeploy Antrea themselves.
deployAntrea bool

externalServerIPs string
vlanSubnets string
vlanID int
externalAgnhostIPs string
vlanSubnets string
vlanID int

externalFRRIPs string
// FRR cannot currently be configured remotely over networking. As a result, the e2e tests for BGPPolicy can only
// be run in a Kind cluster, where the FRR container can be configured using Docker exec with the container ID.
// TODO: Introduce a BGP router implementation that can be configured remotely over networking to replace FRR.
// This would allow the e2e tests for BGPPolicy to be run in environments other than just a Kind cluster.
externalFRRCID string
}

type flowVisibilityTestOptions struct {
Expand Down Expand Up @@ -498,14 +509,14 @@ func (data *TestData) RunCommandOnNodeExt(nodeName, cmd string, envs map[string]
}

func (data *TestData) collectExternalInfo() error {
ips := strings.Split(testOptions.externalServerIPs, ",")
ips := strings.Split(testOptions.externalAgnhostIPs, ",")
for _, ip := range ips {
if ip == "" {
continue
}
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return fmt.Errorf("invalid external server IP %s", ip)
return fmt.Errorf("invalid external agnhost IP %s", ip)
}
if parsedIP.To4() != nil {
externalInfo.externalServerIPv4 = ip
Expand All @@ -532,6 +543,25 @@ func (data *TestData) collectExternalInfo() error {
}
}
externalInfo.vlanID = testOptions.vlanID

frrIPs := strings.Split(testOptions.externalFRRIPs, ",")
for _, ip := range frrIPs {
if ip == "" {
continue
}
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return fmt.Errorf("invalid external FRR IP %s", ip)
}
if parsedIP.To4() != nil {
externalInfo.externalFRRIPv4 = ip
} else {
externalInfo.externalFRRIPv6 = ip
}
}

externalInfo.externalFRRCID = testOptions.externalFRRCID

return nil
}

Expand Down

0 comments on commit 7a7a8bc

Please sign in to comment.