Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support deploying one FRR container in Kind network #6488

Merged
merged 1 commit into from
Jul 22, 2024
Merged

Support deploying one FRR container in Kind network #6488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 37 additions & 14 deletions ci/kind/kind-setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ KUBE_PROXY_MODE="iptables"
PROMETHEUS=false
K8S_VERSION=""
KUBE_NODE_IPAM=true
DEPLOY_EXTERNAL_SERVER=false
DEPLOY_EXTERNAL_AGNHOST=false
DEPLOY_EXTERNAL_FRR=false
positional_args=()
options=()

Expand Down Expand Up @@ -79,8 +80,9 @@ where:
network. Note, '--extra-networks' and '--subnets' cannot be specified together.
--ip-family: specify the ip-family for the kind cluster, default is $IP_FAMILY.
--k8s-version: specify the Kubernetes version of the kind cluster, kind's default K8s version will be used if empty.
--deploy-external-server: deploy a container running as an external server for the cluster.
--all: delete all kind clusters
--deploy-external-agnhost: deploy a container running agnhost as an external server for the cluster, default is $DEPLOY_EXTERNAL_AGNHOST.
--deploy-external-frr: deploy a container running FRR as an external router for the cluster, default is $DEPLOY_EXTERNAL_FRR.
--all: delete all kind clusters.
--until: delete kind clusters that have been created before the specified duration.
"

Expand Down Expand Up @@ -411,7 +413,7 @@ EOF
configure_networks
configure_extra_networks
configure_vlan_subnets
setup_external_server
setup_external_servers
load_images

if [[ $ANTREA_CNI == true ]]; then
Expand Down Expand Up @@ -441,7 +443,7 @@ function destroy {
else
kind delete cluster --name $CLUSTER_NAME
fi
destroy_external_server
destroy_external_servers
delete_networks
delete_vlan_subnets
}
Expand All @@ -455,15 +457,31 @@ function printUnixTimestamp {
fi
}

function setup_external_server {
if [[ $DEPLOY_EXTERNAL_SERVER == true ]]; then
docker run -d --name antrea-external-server-$RANDOM --network kind -it --rm registry.k8s.io/e2e-test-images/agnhost:2.40 netexec &> /dev/null
function setup_external_servers {
if [[ $DEPLOY_EXTERNAL_AGNHOST == true ]]; then
docker run -d --name antrea-external-agnhost-$RANDOM --network kind -it --rm registry.k8s.io/e2e-test-images/agnhost:2.40 netexec &> /dev/null
fi

if [[ $DEPLOY_EXTERNAL_FRR == true ]]; then
docker run -d \
--name antrea-external-frr-$RANDOM \
--network kind --cap-add=NET_BIND_SERVICE \
--cap-add=NET_ADMIN \
--cap-add=NET_RAW \
--cap-add=SYS_ADMIN \
-it \
--rm \
frrouting/frr:v8.4.0 \
bash -c "/bin/sed -i s/bgpd=no/bgpd=yes/g /etc/frr/daemons && /sbin/tini -- /usr/lib/frr/docker-start" &> /dev/null
fi
}

function destroy_external_server {
echo "Deleting external server"
cid=$(docker ps -f name="^antrea-external-server" --format '{{.ID}}')
function destroy_external_servers {
echo "Deleting external servers"
cid=$(docker ps -f name="^antrea-external-agnhost" --format '{{.ID}}')
hongliangl marked this conversation as resolved.
Show resolved Hide resolved
docker rm -f $cid &> /dev/null || true

cid=$(docker ps -f name="^antrea-external-frr" --format '{{.ID}}')
docker rm -f $cid &> /dev/null || true
}

Expand Down Expand Up @@ -583,9 +601,14 @@ while [[ $# -gt 0 ]]
K8S_VERSION="$2"
shift 2
;;
--deploy-external-server)
add_option "--deploy-external-server" "create"
DEPLOY_EXTERNAL_SERVER=true
--deploy-external-agnhost)
add_option "--deploy-external-agnhost" "create"
DEPLOY_EXTERNAL_AGNHOST=true
shift
;;
--deploy-external-frr)
add_option "--deploy-external-frr" "create"
DEPLOY_EXTERNAL_FRR=true
shift
;;
--all)
Expand Down
45 changes: 32 additions & 13 deletions ci/kind/test-e2e-kind.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,21 @@ function echoerr {

_usage="Usage: $0 [--encap-mode <mode>] [--ip-family <v4|v6|dual>] [--coverage] [--help|-h]
--encap-mode Traffic encapsulation mode. (default is 'encap').
--ip-family Configures the ipFamily for the KinD cluster.
--ip-family Configure the ipFamily for the KinD cluster.
--feature-gates A comma-separated list of key=value pairs that describe feature gates, e.g. AntreaProxy=true,Egress=false.
--run Run only tests matching the regexp.
--proxy-all Enables Antrea proxy with all Service support.
--proxy-all Enable Antrea proxy with all Service support.
--no-kube-proxy Don't deploy kube-proxy.
--load-balancer-mode LoadBalancer mode.
--node-ipam Enables Antrea NodeIPAM.
--multicast Enables Multicast.
--node-ipam Enable Antrea NodeIPAM.
--multicast Enable Multicast.
--bgp-policy Enable Antrea BGPPolicy.
--flow-visibility Only run flow visibility related e2e tests.
--networkpolicy-evaluation Configures additional NetworkPolicy evaluation level when running e2e tests.
--extra-network Creates an extra network that worker Nodes will connect to. Cannot be specified with the hybrid mode.
--extra-vlan Creates an subnet-based VLAN that worker Nodes will connect to.
--networkpolicy-evaluation Configure additional NetworkPolicy evaluation level when running e2e tests.
--extra-network Create an extra network that worker Nodes will connect to. Cannot be specified with the hybrid mode.
--extra-vlan Create an subnet-based VLAN that worker Nodes will connect to.
--skip A comma-separated list of keywords, with which tests should be skipped.
--coverage Enables measure Antrea code coverage when running e2e tests on kind.
--coverage Enable measure Antrea code coverage when running e2e tests on kind.
--setup-only Only perform setting up the cluster and run test.
--cleanup-only Only perform cleaning up the cluster.
--test-only Only run test on current cluster. Not set up/clean up the cluster.
Expand Down Expand Up @@ -77,6 +78,7 @@ no_kube_proxy=false
load_balancer_mode=""
node_ipam=false
multicast=false
bgp_policy=false
flow_visibility=false
np_evaluation=false
extra_network=false
Expand Down Expand Up @@ -124,6 +126,10 @@ case $key in
multicast=true
shift
;;
--bgp-policy)
bgp_policy=true
shift
;;
--ip-family)
ipfamily="$2"
shift 2
Expand Down Expand Up @@ -236,6 +242,9 @@ fi
if $multicast; then
manifest_args="$manifest_args --multicast"
fi
if $bgp_policy; then
manifest_args="$manifest_args --feature-gates BGPPolicy=true"
fi
if $flow_visibility; then
manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES"
fi
Expand Down Expand Up @@ -314,8 +323,12 @@ function setup_cluster {
if $extra_network && [[ "$mode" != "hybrid" ]]; then
args="$args --extra-networks \"20.20.30.0/24\""
fi
# Deploy an external server which could be used when testing Pod-to-External traffic.
args="$args --deploy-external-server $vlan_args"
# Deploy an external agnhost which could be used when testing Pod-to-External traffic.
args="$args --deploy-external-agnhost $vlan_args"
# Deploy an external FRR which could be used when testing BGPPolicy.
if $bgp_policy; then
args="$args --deploy-external-frr"
fi

echo "creating test bed with args $args"
eval "timeout 600 $TESTBED_CMD create kind $args"
Expand Down Expand Up @@ -379,9 +392,15 @@ function run_test {
np_evaluation_flag="--networkpolicy-evaluation"
fi

external_server_cid=$(docker ps -f name="^antrea-external-server" --format '{{.ID}}')
external_server_ips=$(docker inspect $external_server_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')
EXTRA_ARGS="$vlan_args --external-server-ips $external_server_ips"
external_agnhost_cid=$(docker ps -f name="^antrea-external-agnhost" --format '{{.ID}}')
external_agnhost_ips=$(docker inspect $external_agnhost_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')
EXTRA_ARGS="$vlan_args --external-agnhost-ips $external_agnhost_ips"

if $bgp_policy; then
external_frr_cid=$(docker ps -f name="^antrea-external-frr" --format '{{.ID}}')
external_frr_ips=$(docker inspect $external_frr_cid -f '{{.NetworkSettings.Networks.kind.IPAddress}},{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')
EXTRA_ARGS="$EXTRA_ARGS --external-frr-cid $external_frr_cid --external-frr-ips $external_frr_ips"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why we need to pass external_frr_cid to test code? I think this would cause the test can only run in kind cluster.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought we could run some commands in the FRR container by passing the cid to test code.

I think this would cause the test can only run in kind cluster.

A question: is the external agnhost container only supported in Kind cluster? I thought that it was the same for the external frr container.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought we could run some commands in the FRR container by passing the cid to test code.

Is there a public API to access the FRR application, like "/clientip" of agnhost container?

A question: is the external agnhost container only supported in Kind cluster? I thought that it was the same for the external frr container.

The test code doesn't make any assumption on how the external agnhost server is deployed so technically you can specify a VM IP or a public IP. It only needs the server's IP.
By passing the container ID to test code, I assume you would then use it to run some validation, then the code can be only ran in a kind cluster, and can't work with real BGP router.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a public API to access the FRR application, like "/clientip" of agnhost container?

I'm afraid we cannot configure it through network. I found that FRR application doesn't provide any API based TCP/IP. It can be configured through Unix Socket API locally.

By passing the container ID to test code, I assume you would then use it to run some validation, then the code can be only ran in a kind cluster, and can't work with real BGP router.

I use the container ID to configure the FRR application using the command like docker exec -it <container ID> <command>.

fi

go test -v -timeout=$timeout $RUN_OPT antrea.io/antrea/test/e2e $flow_visibility_args -provider=kind --logs-export-dir=$ANTREA_LOG_DIR $np_evaluation_flag --skip-cases=$skiplist $coverage_args $EXTRA_ARGS

Expand Down
4 changes: 3 additions & 1 deletion test/e2e/fixtures.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -685,7 +685,9 @@ func testMain(m *testing.M) int {
flag.StringVar(&testOptions.skipCases, "skip-cases", "", "Key words to skip cases")
flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs")
flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs")
flag.StringVar(&testOptions.externalServerIPs, "external-server-ips", "", "IP addresses of external server, at most one IP per IP family")
flag.StringVar(&testOptions.externalAgnhostIPs, "external-agnhost-ips", "", "IP addresses of external agnhost, at most one IP per IP family")
flag.StringVar(&testOptions.externalFRRIPs, "external-frr-ips", "", "IP addresses of external FRR, at most one IP per IP family")
flag.StringVar(&testOptions.externalFRRCID, "external-frr-cid", "", "Container ID of external FRR")
flag.StringVar(&testOptions.vlanSubnets, "vlan-subnets", "", "IP subnets of the VLAN network the Nodes reside in, at most one subnet per IP family")
flag.IntVar(&testOptions.vlanID, "vlan-id", 0, "ID of the VLAN network the Nodes reside in")
flag.Parse()
Expand Down
40 changes: 35 additions & 5 deletions test/e2e/framework.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ type ExternalInfo struct {
vlanSubnetIPv6 string
vlanGatewayIPv6 string
vlanID int

externalFRRIPv4 string
XinShuYang marked this conversation as resolved.
Show resolved Hide resolved
externalFRRIPv6 string
externalFRRCID string
}

var clusterInfo ClusterInfo
Expand All @@ -213,9 +217,16 @@ type TestOptions struct {
// the home directory of the control-plane Node. Note it doesn't affect the tests that redeploy Antrea themselves.
deployAntrea bool

externalServerIPs string
vlanSubnets string
vlanID int
externalAgnhostIPs string
vlanSubnets string
vlanID int

externalFRRIPs string
// FRR cannot currently be configured remotely over networking. As a result, the e2e tests for BGPPolicy can only
// be run in a Kind cluster, where the FRR container can be configured using Docker exec with the container ID.
// TODO: Introduce a BGP router implementation that can be configured remotely over networking to replace FRR.
// This would allow the e2e tests for BGPPolicy to be run in environments other than just a Kind cluster.
externalFRRCID string
}

type flowVisibilityTestOptions struct {
Expand Down Expand Up @@ -498,14 +509,14 @@ func (data *TestData) RunCommandOnNodeExt(nodeName, cmd string, envs map[string]
}

func (data *TestData) collectExternalInfo() error {
ips := strings.Split(testOptions.externalServerIPs, ",")
ips := strings.Split(testOptions.externalAgnhostIPs, ",")
for _, ip := range ips {
if ip == "" {
continue
}
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return fmt.Errorf("invalid external server IP %s", ip)
return fmt.Errorf("invalid external agnhost IP %s", ip)
}
if parsedIP.To4() != nil {
externalInfo.externalServerIPv4 = ip
Expand All @@ -532,6 +543,25 @@ func (data *TestData) collectExternalInfo() error {
}
}
externalInfo.vlanID = testOptions.vlanID

frrIPs := strings.Split(testOptions.externalFRRIPs, ",")
for _, ip := range frrIPs {
if ip == "" {
continue
}
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return fmt.Errorf("invalid external FRR IP %s", ip)
}
if parsedIP.To4() != nil {
externalInfo.externalFRRIPv4 = ip
} else {
externalInfo.externalFRRIPv6 = ip
}
}

externalInfo.externalFRRCID = testOptions.externalFRRCID

return nil
}

Expand Down