test/cni: work-around conntrack lingering rules #313
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| pull_request: | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| PLATFORMS: linux/amd64 | |
| permissions: | |
| packages: write | |
| jobs: | |
| check_unit: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| combination: | |
| #Debug | |
| - { kernel: 'Linux 6.8', runner: 'ubuntu-24.04', clangs: 'clang-14 clang-15 clang-16 clang-17 clang-18', debug: '1' } | |
| #NDEBUG | |
| - { kernel: 'Linux 6.8', runner: 'ubuntu-24.04', clangs: 'clang-14 clang-15 clang-16 clang-17 clang-18', debug: '0' } | |
| runs-on: ${{ matrix.combination.runner }} | |
| steps: | |
| - name: "Checkout sfunnel" | |
| uses: actions/checkout@v4 | |
| with: | |
| path: sfunnel | |
| fetch-depth: 0 | |
| fetch-tags: 1 | |
| - name: "Install deps..." | |
| run: | | |
| sudo add-apt-repository universe | |
| sudo apt-get update | |
| sudo apt-get install -y ${{matrix.combination.clangs}} make iproute2 \ | |
| bridge-utils python3-scapy python3-pip libbpf-dev \ | |
| libelf-dev linux-headers-generic \ | |
| linux-libc-dev llvm iptables | |
| sudo ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/asm | |
| if [[ "${{ matrix.combination.runner }}" == "ubuntu-24.04" ]]; then | |
| sudo apt-get install python3-pytest | |
| else | |
| sudo pip3 install pytest #pytest binary missing in python3-pytest | |
| fi | |
| - name: "Allow test traffic in iptables/nftables" | |
| run: | | |
| sudo iptables -L -n | |
| sudo iptables -t nat -L -n | |
| sudo iptables -I FORWARD -s 11.1.1.1 -j ACCEPT | |
| - name: "Run Linux NS tests on '${{matrix.combination.clangs}}'..." | |
| run: | | |
| cd sfunnel/test/ns | |
| for clang in ${{matrix.combination.clangs}}; do | |
| echo "[${clang}] Running tests ..." | |
| make VERBOSE=1 CLANG=${clang} DEBUG=${{matrix.combination.debug}} || ( echo "FAILED: test failed with '${clang}' debug='${{matrix.combination.debug}}'" && exit 1 ) | |
| make clean 2>&1 > /dev/null | |
| echo "[${clang}]" | |
| done | |
| check_cni_example: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| combination: | |
| - { runner: 'ubuntu-24.04', cni: 'cilium', nodes: '1' } | |
| - { runner: 'ubuntu-24.04', cni: 'cilium', nodes: '2' } | |
| - { runner: 'ubuntu-24.04', cni: 'calico', nodes: '2' } | |
| - { runner: 'ubuntu-24.04', cni: 'flannel', nodes: '2' } | |
| - { runner: 'ubuntu-24.04', cni: 'kindnet', nodes: '2' } | |
| runs-on: ${{ matrix.combination.runner }} | |
| steps: | |
| - name: "Checkout sfunnel" | |
| uses: actions/checkout@v4 | |
| with: | |
| path: sfunnel | |
| fetch-depth: 0 | |
| fetch-tags: 1 | |
| - name: "Install deps..." | |
| run: | | |
| sudo add-apt-repository universe | |
| sudo apt-get update | |
| sudo apt-get install -y clang make iproute2 iperf \ | |
| bridge-utils python3-scapy python3-pip libbpf-dev \ | |
| libelf-dev linux-headers-generic \ | |
| linux-libc-dev llvm iptables | |
| sudo ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/asm | |
| if [[ "${{ matrix.combination.runner }}" == "ubuntu-24.04" ]]; then | |
| sudo apt-get install python3-pytest | |
| else | |
| sudo pip3 install pytest #pytest binary missing in python3-pytest | |
| fi | |
| - name: "[cilium] Install CLI" | |
| if: matrix.combination.cni == 'cilium' | |
| run: | | |
| CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt) | |
| GOOS=$(go env GOOS) | |
| GOARCH=$(go env GOARCH) | |
| wget https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-${GOOS}-${GOARCH}.tar.gz | |
| sudo tar -C /usr/local/bin -xzvf cilium-${GOOS}-${GOARCH}.tar.gz | |
| rm cilium-${GOOS}-${GOARCH}.tar.gz | |
| - name: "Run LoadBalancer test nodes='${{ matrix.combination.nodes }}' cni='${{ matrix.combination.cni }}'..." | |
| env: | |
| CNI: ${{matrix.combination.cni}} | |
| NODES: ${{matrix.combination.nodes}} | |
| run: | | |
| cd sfunnel/test/cni/ | |
| make NODES=${NODES} CNI=${CNI} VERBOSE=1 || ( echo "FAILED: test failed with cni='${CNI}' nodes='${NODES}'" && exit 1 ) | |
| echo "Pods (after positive test):" | |
| minikube kubectl -- get pods -o wide | |
| - name: "Run ClusterIP test nodes='${{ matrix.combination.nodes }}' cni='${{ matrix.combination.cni }}'..." | |
| env: | |
| CNI: ${{matrix.combination.cni}} | |
| NODES: ${{matrix.combination.nodes}} | |
| MAX_ITERATIONS: 15 | |
| timeout-minutes: 5 | |
| run: | | |
| cd sfunnel/example/k8s | |
| minikube kubectl -- apply -f client.yaml | |
| minikube kubectl -- get pods -o wide | |
| echo "Waiting for all pods to be running (filter spureous starts)..." | |
| while [[ "$(minikube kubectl -- get pods | grep client | grep -v Running)" != "" ]]; do | |
| echo "Not ready, waiting..." | |
| minikube kubectl -- get pods -o wide | |
| sleep 1; | |
| done | |
| for POD in $(minikube kubectl -- get pods | grep client | awk '{print $1}'); do | |
| echo "Waiting for '${POD}' to have completed at least one query..." | |
| i=1 | |
| while true; do | |
| if [[ "$(minikube kubectl -- logs ${POD} | grep 'Serving pod')" != "" ]]; then | |
| break | |
| fi | |
| if [[ $i -eq ${MAX_ITERATIONS} ]]; then | |
| echo "ERROR: '${POD}' never converged..." | |
| minikube kubectl -- logs ${POD} | |
| exit 1 | |
| fi | |
| ((i++)) | |
| sleep 3 | |
| done | |
| done | |
| - name: "[NEGATIVE test] Check LoadBalancer affinity test must fail nodes='${{ matrix.combination.nodes }}' cni='${{ matrix.combination.cni }}'..." | |
| env: | |
| CNI: ${{matrix.combination.cni}} | |
| NODES: ${{matrix.combination.nodes}} | |
| MAX_ITERATIONS: 15 | |
| run: | | |
| cd sfunnel/test/cni/ | |
| #Remove funneling | |
| make VERBOSE=1 _unload | |
| minikube kubectl -- apply -k ../../.github/workflows/negative/ | |
| #This MUST now fail | |
| echo "Pods (before negative test):" | |
| minikube kubectl -- get pods -o wide | |
| ( make VERBOSE=1 _check_affinity && ( echo "FAILED: negative test succeeded with cni='${CNI}' nodes='${NODES}' when it should have failed!" && exit 1 ) ) || true | |
| - name: "Performance test nodes='${{ matrix.combination.nodes }}' cni='${{ matrix.combination.cni }}'..." | |
| env: | |
| CNI: ${{matrix.combination.cni}} | |
| NODES: ${{matrix.combination.nodes}} | |
| run: | | |
| cd sfunnel/test/cni/ | |
| make VERBOSE=1 _check_perf | |
| #Create the markdown report (TODO improve txt) | |
| echo "# Performance report" >> $GITHUB_STEP_SUMMARY | |
| echo "CNI: ${CNI}" >> $GITHUB_STEP_SUMMARY | |
| echo "Number of nodes: ${NODES}" >> $GITHUB_STEP_SUMMARY | |
| echo "## Results" >> $GITHUB_STEP_SUMMARY | |
| cat .last_perf_report.txt >> $GITHUB_STEP_SUMMARY | |
| docker_build_test_publish: | |
| needs: [check_unit, check_cni_example] | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: "Checkout sfunnel" | |
| uses: actions/checkout@v4 | |
| with: | |
| path: sfunnel | |
| fetch-depth: 0 | |
| fetch-tags: 1 | |
| - name: "Set up Docker buildx" | |
| uses: docker/setup-buildx-action@v3 | |
| - name: "Login to GitHub Container Registry (ghcr.io)" | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{github.actor}} | |
| password: ${{secrets.GITHUB_TOKEN}} | |
| - name: "Build container" | |
| run: | | |
| #Cross-build | |
| cd sfunnel | |
| echo "Fix mess with tags in actions/checkout..." | |
| git fetch -f && git fetch -f --tags | |
| docker buildx build --platform ${PLATFORMS} -t sfunnel --build-arg VERSION="$(git describe)" --build-arg COMMIT="${GITHUB_SHA}" --load -f docker/Dockerfile . | |
| - name: "[TEST] Run container without env. nor file. Should fail..." | |
| run: | | |
| set -o pipefail | |
| set +e | |
| docker run --privileged --network=host -v /var/run/netns:/var/run/netns -e DEBUG=1 -e IFACES=lo sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -eq 0 ]; then | |
| echo "ERROR: container succeded when it should have FAILED!" | |
| exit 1 | |
| fi | |
| set -e | |
| grep "\[ERROR\] Neither '" output || (echo "ERROR: unable to validate it raises the correct ruleset error" && exit 1) | |
| - name: "[TEST] Run container with ruleset file..." | |
| run: | | |
| RULE="ip saddr 127.0.0.1 udp dport 80 actions unfunnel udp" | |
| echo "$RULE" > ruleset | |
| set -o pipefail | |
| docker run --privileged -v `pwd`/ruleset:/etc/sfunnel/ruleset 2>&1 sfunnel:latest | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| grep "Recompiling sfunnel BPF program" output || (echo "ERROR: unable to validate it loads the rulset file" && exit 1) | |
| grep "$RULE" output || (echo "ERROR: unable to validate it loads the ruleset file" && exit 1) | |
| - name: "[TEST] Run container with invalid SFUNNEL_RULESET. Should fail..." | |
| run: | | |
| set -o pipefail | |
| set +e | |
| docker run --privileged --network=host -e SFUNNEL_RULESET="XXX" -e DEBUG=1 -e IFACES=lo sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -eq 0 ]; then | |
| echo "ERROR: container succeded when it should have FAILED!" | |
| exit 1 | |
| fi | |
| set -e | |
| - name: "[TEST] Run container with ruleset via SFUNNEL_RULESET (no override)..." | |
| run: | | |
| RULE="ip saddr 127.0.0.2 udp dport 80 actions unfunnel udp" | |
| set -o pipefail | |
| docker run -e SFUNNEL_RULESET="$RULE" --privileged sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| grep "SFUNNEL_RULESET='$RULE'" output || (echo "ERROR: unable to validate it loads ruleset via SFUNNEL_RULESET" && exit 1) | |
| grep "Recompiling sfunnel BPF program" output || (echo "ERROR: unable to validate it loads ruleset via SFUNNEL_RULESET" && exit 1) | |
| grep "$RULE" output || (echo "ERROR: unable to validate it loads custom ruleset via SFUNNEL_RULESET" && exit 1) | |
| - name: "[TEST] Run container with ruleset via SFUNNEL_RULESET (override)..." | |
| run: | | |
| RULE="ip saddr 127.0.0.2 udp dport 80 actions unfunnel udp" #Should override ruleset file with 127.0.0.1 | |
| set -o pipefail | |
| docker run -e SFUNNEL_RULESET="$RULE" --privileged -v `pwd`/ruleset:/etc/sfunnel/ruleset sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| grep "SFUNNEL_RULESET='$RULE'" output || (echo "ERROR: unable to validate it loads ruleset via SFUNNEL_RULESET" && exit 1) | |
| grep "Recompiling sfunnel BPF program" output || (echo "ERROR: unable to validate it loads ruleset via SFUNNEL_RULESET" && exit 1) | |
| grep "$RULE" output || (echo "ERROR: unable to validate it loads custom ruleset via SFUNNEL_RULESET" && exit 1) | |
| - name: "[TEST] Run container with custom params ..." | |
| run: | | |
| set -o pipefail | |
| docker run -e N_ATTEMPTS=7 -e RETRY_DELAY=3 -e IFACES="lo" -e DEBUG=1 -v `pwd`/ruleset:/etc/sfunnel/ruleset --privileged sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| grep "\$DEBUG='1'" output || (echo "ERROR: unable to validate it loads params (DEBUG)" && exit 1) | |
| grep "\-DDEBUG=1" output || (echo "ERROR: unable to validate it loads params (DEBUG), CFLAGS" && exit 1) | |
| grep "\$N_ATTEMPTS='7'" output || (echo "ERROR: unable to validate it loads params (N_ATTEMPTS)" && exit 1) | |
| grep "\$RETRY_DELAY='3'" output || (echo "ERROR: unable to validate it loads params (RETRY_DELAY)" && exit 1) | |
| grep "\$IFACES='lo'" output || (echo "ERROR: unable to validate it loads params (IFACES)" && exit 1) | |
| #Must recompile due to DEBUG=1 | |
| grep "Recompiling sfunnel BPF program" output || (echo "ERROR: unable to validate it loads params (DEBUG)" && exit 1) | |
| grep "\-DDEBUG=1" output || (echo "ERROR: unable to validate it loads params (DEBUG, CFLAGS)" && exit 1) | |
| - name: "[TEST] Run container with DEBUG=1 ..." | |
| run: | | |
| RULE="ip saddr 127.0.0.1 udp dport 80 actions unfunnel udp" | |
| echo "$RULE" > ruleset | |
| set -o pipefail | |
| docker run --privileged -e DEBUG=1 -v `pwd`/ruleset:/etc/sfunnel/ruleset sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| grep "\$DEBUG='1'" output || (echo "ERROR: unable to validate it loads custom file ruleset DEBUG=1" && exit 1) | |
| grep "\-DDEBUG=1" output || (echo "ERROR: unable to validate it loads custom file ruleset DEBUG=1, CFLAGS" && exit 1) | |
| grep "Recompiling sfunnel BPF program" output || (echo "ERROR: unable to validate it loads custom file ruleset DEBUG=1" && exit 1) | |
| grep "$RULE" output || (echo "ERROR: unable to validate it loads custom file ruleset DEBUG=1" && exit 1) | |
| - name: "[TEST] Run container with IFACES=invalid ..." | |
| run: | | |
| set -o pipefail | |
| set +e | |
| docker run --privileged -e DEBUG=1 -e IFACES=invalid -v `pwd`/ruleset:/etc/sfunnel/ruleset sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -eq 0 ]; then | |
| echo "ERROR: container succeded when it should have FAILED!" | |
| exit 1 | |
| fi | |
| set -e | |
| - name: "[TEST] Run container with DIRECTION=invalid ..." | |
| run: | | |
| set -o pipefail | |
| set +e | |
| docker run --privileged -e DEBUG=1 -e DIRECTION=invalid -v `pwd`/ruleset:/etc/sfunnel/ruleset sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -eq 0 ]; then | |
| echo "ERROR: container succeded when it should have FAILED!" | |
| exit 1 | |
| fi | |
| set -e | |
| grep "FATAL: Invalid traffic direction" output || (echo "ERROR: unable to validate container correctly fails when DIRECTION is invalid" && exit 1) | |
| - name: "[TEST] Run container with DIRECTION=ingress ..." | |
| run: | | |
| set -o pipefail | |
| docker run --privileged -e DEBUG=1 -e DIRECTION=ingress -v `pwd`/ruleset:/etc/sfunnel/ruleset sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| (grep "Attaching BPF program" output | grep "direction 'ingress'") || (echo "ERROR: unable to validate container attaches to DIRECTION=ingress" && exit 1) | |
| - name: "[TEST] Run container with DIRECTION=egress ..." | |
| run: | | |
| set -o pipefail | |
| docker run --privileged -e DEBUG=1 -e DIRECTION=egress -v `pwd`/ruleset:/etc/sfunnel/ruleset sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| (grep "Attaching BPF program" output | grep "direction 'egress'") || (echo "ERROR: unable to validate container attaches to DIRECTION=egress" && exit 1) | |
| - name: "[TEST] Run container with DIRECTION=both ..." | |
| run: | | |
| set -o pipefail | |
| docker run --privileged -e DEBUG=1 -e DIRECTION=both -v `pwd`/ruleset:/etc/sfunnel/ruleset sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| (grep "Attaching BPF program" output | grep "direction 'ingress'") || (echo "ERROR: unable to validate container attaches to DIRECTION=both" && exit 1) | |
| (grep "Attaching BPF program" output | grep "direction 'egress'") || (echo "ERROR: unable to validate container attaches to DIRECTION=both" && exit 1) | |
| - name: "[TEST] Run container with CLEAN=1 ..." | |
| run: | | |
| set -o pipefail | |
| #Create a pair of veths | |
| sudo ip link add type veth | |
| sudo ip link set up dev veth0 | |
| sudo ip link set up dev veth1 | |
| #First attach to veth | |
| docker run --privileged --network=host -e DEBUG=1 -e DIRECTION=both -e IFACES=veth0 -v `pwd`/ruleset:/etc/sfunnel/ruleset sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| (tc filter show dev veth0 ingress | grep sfunnel) || (echo "ERROR: unable to validate container attaches to ingress" && exit 1) | |
| (tc filter show dev veth0 egress | grep sfunnel) || (echo "ERROR: unable to validate container attaches to egress" && exit 1) | |
| #Run CLEAN=1 | |
| docker run --privileged --network=host -e DEBUG=1 -e CLEAN=1 -e DIRECTION=both -e IFACES=veth0 sfunnel:latest 2>&1 | tee output | |
| [[ "$(tc filter show dev veth0 ingress)" == "" ]] || (echo "ERROR: unable to validate container removes BPF programs from ingress" && exit 1) | |
| [[ "$(tc filter show dev veth0 egress)" == "" ]] || (echo "ERROR: unable to validate container removes BPF programs from egress" && exit 1) | |
| - name: "[TEST] Run container with NETNS=test_ns ..." | |
| run: | | |
| set -o pipefail | |
| #Create a pair of veths | |
| sudo ip netns add test_ns | |
| sudo ip netns exec test_ns ip link set up dev lo | |
| #First run with an invalid netns, make sure it fails | |
| set +e | |
| docker run --privileged --network=host -v /var/run/netns:/var/run/netns -v `pwd`/ruleset:/etc/sfunnel/ruleset -e NETNS=test_ns_invalid -e DEBUG=1 -e IFACES=lo sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -eq 0 ]; then | |
| echo "ERROR: container succeded when it should have FAILED!" | |
| exit 1 | |
| fi | |
| #Then run it with a valid netns but invalid IFACE, and make sure it propagates the error code | |
| docker run --privileged --network=host -v /var/run/netns:/var/run/netns -e NETNS=test_ns -e DEBUG=1 -e IFACES=lo2 sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -eq 0 ]; then | |
| echo "ERROR: container succeded when it should have FAILED!" | |
| exit 1 | |
| fi | |
| set -e | |
| #Successful run | |
| docker run --privileged --network=host -v /var/run/netns:/var/run/netns -v `pwd`/ruleset:/etc/sfunnel/ruleset -e NETNS=test_ns -e DEBUG=1 -e IFACES=lo sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| [[ "$(sudo ip netns exec test_ns tc filter show dev lo ingress | grep sfunnel)" != "" ]] || (echo "ERROR: unable to validate container attaches to ingress when NETNS is set" && exit 1) | |
| (grep "\$DEBUG='1'" output) || (echo "ERROR: unable to validate env. variables are passed to the NETNS execution" && exit 1) | |
| #Successful run with SFUNNEL_RULESET | |
| RULE="ip saddr 127.0.0.2 udp dport 80 actions unfunnel udp" | |
| docker run --privileged --network=host -v /var/run/netns:/var/run/netns -e NETNS=test_ns -e DEBUG=1 -e SFUNNEL_RULESET="$RULE" -e IFACES=lo sfunnel:latest 2>&1 | tee output | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo "ERROR: container execution FAILED!" | |
| exit 1 | |
| fi | |
| - name: "Push to ghcr" | |
| run: | | |
| cd sfunnel | |
| export TAG=$(git describe HEAD | sed 's/-.*$//g' | tr -d "v") | |
| export EXACT_TAG=$(git describe --exact-match --match "v*" || echo "") | |
| echo "TAG=${TAG}, EXACT_TAG=${EXACT_TAG}" | |
| if [[ "${EXACT_TAG}" != "" ]]; then | |
| echo "Pushing to ghcr.io..." | |
| docker buildx build --platform ${PLATFORMS} --build-arg VERSION="$(git describe)" --build-arg COMMIT="${GITHUB_SHA}" --push -f docker/Dockerfile . --tag ghcr.io/${GITHUB_REPOSITORY}:${TAG} | |
| fi |