From f11e33bac7514673443fe4d386b0c63f0a204d93 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Wed, 27 Mar 2024 18:43:04 +0900 Subject: [PATCH 1/2] sock-addr: local-vip cicd added --- cicd/common.sh | 13 +++++++---- cicd/tcplb-local/config.sh | 40 ++++++++++++++++++++++++++++++++++ cicd/tcplb-local/rmconfig.sh | 18 +++++++++++++++ cicd/tcplb-local/validation.sh | 36 ++++++++++++++++++++++++++++++ loxilb-ebpf | 2 +- loxinet/loxinet.go | 6 ++--- 6 files changed, 106 insertions(+), 9 deletions(-) create mode 100755 cicd/tcplb-local/config.sh create mode 100755 cicd/tcplb-local/rmconfig.sh create mode 100755 cicd/tcplb-local/validation.sh diff --git a/cicd/common.sh b/cicd/common.sh index e8725ea42..5ff24a960 100644 --- a/cicd/common.sh +++ b/cicd/common.sh @@ -14,6 +14,7 @@ hexist="$vrn$hn" lxdocker="ghcr.io/loxilb-io/loxilb:latest" hostdocker="ghcr.io/nicolaka/netshoot:latest" cluster_opts="" +extra_opts="" ka_opts="" var=$(lsb_release -r | cut -f2) #if [[ $var == *"22.04"* ]];then @@ -82,6 +83,10 @@ spawn_docker_host() { fi shift 2 ;; + -e | --extra-args) + extra_opts="$2" + shift 2 + ;; -*|--*) echo "Unknown option $1" exit @@ -104,12 +109,12 @@ spawn_docker_host() { fi if [[ ! -z ${ka+x} ]]; then sudo mkdir -p /etc/shared/$dname/ - docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config --name $dname $lxdocker + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --pid=host --cgroupns=host --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config --name $dname $lxdocker get_llb_peerIP $dname - docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts $extra_opts else - docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts - docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --pid=host --cgroupns=host --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $extra_opts fi elif [[ "$dtype" == "host" ]]; then if [[ ! -z "$bpath" ]]; then diff --git a/cicd/tcplb-local/config.sh b/cicd/tcplb-local/config.sh new file mode 100755 index 000000000..c3b7087a7 --- /dev/null +++ b/cicd/tcplb-local/config.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +source ../common.sh + +echo "#########################################" +echo "Spawning all hosts" +echo "#########################################" + +spawn_docker_host --dock-type loxilb --dock-name llb1 --extra-args "--localvip" +spawn_docker_host --dock-type host --dock-name l3h1 +spawn_docker_host --dock-type host --dock-name l3ep1 +spawn_docker_host --dock-type host --dock-name l3ep2 +spawn_docker_host --dock-type host --dock-name l3ep3 + +echo "#########################################" +echo "Connecting and configuring hosts" +echo "#########################################" + + +connect_docker_hosts l3h1 llb1 +connect_docker_hosts l3ep1 llb1 +connect_docker_hosts l3ep2 llb1 +connect_docker_hosts l3ep3 llb1 + +sleep 5 + +#L3 config +config_docker_host --host1 l3h1 --host2 llb1 --ptype phy --addr 10.10.10.1/24 --gw 10.10.10.254 +config_docker_host --host1 l3ep1 --host2 llb1 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254 +config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254 +config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254 +config_docker_host --host1 llb1 --host2 l3h1 --ptype phy --addr 10.10.10.254/24 +config_docker_host --host1 llb1 --host2 l3ep1 --ptype phy --addr 31.31.31.254/24 +config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24 +config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24 + +$dexec llb1 ip addr add 10.10.10.3/32 dev lo + +sleep 5 +create_lb_rule llb1 10.10.10.3 --tcp=2020:8080 --endpoints=10.10.10.3:1 diff --git a/cicd/tcplb-local/rmconfig.sh b/cicd/tcplb-local/rmconfig.sh new file mode 100755 index 000000000..9ea80577f --- /dev/null +++ b/cicd/tcplb-local/rmconfig.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +source ../common.sh + +disconnect_docker_hosts l3h1 llb1 +disconnect_docker_hosts l3ep1 llb1 +disconnect_docker_hosts l3ep2 llb1 +disconnect_docker_hosts l3ep3 llb1 + +delete_docker_host llb1 +delete_docker_host l3h1 +delete_docker_host l3ep1 +delete_docker_host l3ep2 +delete_docker_host l3ep3 + +echo "#########################################" +echo "Deleted testbed" +echo "#########################################" diff --git a/cicd/tcplb-local/validation.sh b/cicd/tcplb-local/validation.sh new file mode 100755 index 000000000..5e0aa5a56 --- /dev/null +++ b/cicd/tcplb-local/validation.sh @@ -0,0 +1,36 @@ +#!/bin/bash +source ../common.sh +echo SCENARIO-tcplb-local +$hexec llb1 node ../common/tcp_server.js server1 & + +sleep 5 +code=0 +servIP=( "10.10.10.3" ) +servArr=( "server1" ) +ep=( "10.10.10.3" ) + +for k in {0..0} +do +echo "Testing Service IP: ${servIP[k]}" +lcode=0 +for j in {0..2} +do + res=$($hexec llb1 curl --max-time 10 -s ${servIP[k]}:2020) + echo $res + if [[ $res != "${servArr[k]}" ]] + then + lcode=1 + fi + sleep 1 +done +if [[ $lcode == 0 ]] +then + echo SCENARIO-tcplb-local with ${servIP[k]} [OK] +else + echo SCENARIO-tcplb-local with ${servIP[k]} [FAILED] + code=1 +fi +done + +sudo killall -9 node 2>&1 > /dev/null +exit $code diff --git a/loxilb-ebpf b/loxilb-ebpf index 2f8f97336..a4c5c3c31 160000 --- a/loxilb-ebpf +++ b/loxilb-ebpf @@ -1 +1 @@ -Subproject commit 2f8f973361bc4e06b6c00fe3b913e5633594a95a +Subproject commit a4c5c3c31d70ce8b1c0910abef436322f6b35499 diff --git a/loxinet/loxinet.go b/loxinet/loxinet.go index 8bdea558e..10fe938dd 100644 --- a/loxinet/loxinet.go +++ b/loxinet/loxinet.go @@ -56,8 +56,7 @@ const ( BpfFsCheckFile = "/opt/loxilb/dp/bpf/intf_map" ARPAcceptAll = "sysctl net.ipv4.conf.all.arp_accept=1" ARPAcceptDfl = "sysctl net.ipv4.conf.default.arp_accept=1" - UnMountCG2 = "umount /sys/fs/cgroup/unified || mkdir -p /sys/fs/cgroup/unified" - MountCG2 = "mount -t cgroup2 -o rw,relatime,nsdelegate,memory_recursiveprot cgroup2 /sys/fs/cgroup/unified" + MkMountCG2 = "/usr/local/sbin/mkllb_cgroup" ) type loxiNetH struct { @@ -238,8 +237,7 @@ func loxiNetInit() { if !opts.Opts.BgpPeerMode { if mh.locVIP { - RunCommand(UnMountCG2, false) - RunCommand(MountCG2, false) + RunCommand(MkMountCG2, false) } // Initialize the ebpf datapath subsystem mh.dpEbpf = DpEbpfInit(clusterMode, mh.rssEn, mh.eHooks, mh.locVIP, mh.self, -1) From c4d3813fea7e75007e50e9e9ef4cb79a3fb8fb07 Mon Sep 17 00:00:00 2001 From: Trekkie Coder Date: Wed, 27 Mar 2024 18:48:42 +0900 Subject: [PATCH 2/2] dockerfile: updated for sock-addr support --- Dockerfile | 2 ++ loxilb-ebpf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 606d1ff77..bdd4d8dc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,7 @@ RUN mkdir -p /opt/loxilb && \ git clone --recurse-submodules https://github.com/loxilb-io/loxilb /root/loxilb-io/loxilb/ && \ cd /root/loxilb-io/loxilb/ && go get . && if [ "$arch" = "arm64" ] ; then DOCKER_BUILDX_ARM64=true make; \ else make ;fi && cp loxilb-ebpf/utils/mkllb_bpffs.sh /usr/local/sbin/mkllb_bpffs && \ + cp loxilb-ebpf/utils/mkllb_cgroup.sh /usr/local/sbin/mkllb_cgroup && \ cp api/certification/* /opt/loxilb/cert/ && cd - && \ cp /root/loxilb-io/loxilb/loxilb-ebpf/kernel/loxilb_dp_debug /usr/local/sbin/loxilb_dp_debug && \ cp /root/loxilb-io/loxilb/loxilb /usr/local/sbin/loxilb && \ @@ -87,6 +88,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends sudo \ COPY --from=build /usr/lib64/libbpf* /usr/lib64/ COPY --from=build /usr/local/go/bin /usr/local/go/bin COPY --from=build /usr/local/sbin/mkllb_bpffs /usr/local/sbin/mkllb_bpffs +COPY --from=build /usr/local/sbin/mkllb_cgroup /usr/local/sbin/mkllb_cgroup COPY --from=build /usr/local/sbin/loxilb_dp_debug /usr/local/sbin/loxilb_dp_debug COPY --from=build /usr/local/sbin/loxicmd /usr/local/sbin/loxicmd COPY --from=build /opt/loxilb /opt/loxilb diff --git a/loxilb-ebpf b/loxilb-ebpf index a4c5c3c31..8ec9cff76 160000 --- a/loxilb-ebpf +++ b/loxilb-ebpf @@ -1 +1 @@ -Subproject commit a4c5c3c31d70ce8b1c0910abef436322f6b35499 +Subproject commit 8ec9cff76ef7ff5ab5c0683b31427dabd78b345f