Merge pull request #5151 from rancher-sandbox/add-bats-traefik-test

Test traefik with kubernetes.ingress.localhost-only

bats/tests/helpers/commands.bash

@@ -45,6 +45,9 @@ ctrctl() {
         nerdctl "$@"
     fi
 }
+curl() {
+    command "curl$EXE" "$@"
+}
 docker() {
     docker_exe --context $RD_DOCKER_CONTEXT "$@"
 }

bats/tests/helpers/vm.bash

@@ -17,9 +17,9 @@ pkill_by_path() {
 }
 
 clear_iptables_chain() {
-    local CHAIN=$1
+    local chain=$1
     local rule
-    wsl sudo iptables -L | awk '/^Chain ${CHAIN}/ {print $2}' | while IFS= read -r rule; do
+    wsl sudo iptables -L | awk '/^Chain ${chain}/ {print $2}' | while IFS= read -r rule; do
         wsl sudo iptables -X "$rule"
     done
 }

bats/tests/k8s/traefik.bats

@@ -8,23 +8,22 @@
 load '../helpers/load'
 
 local_setup() {
+    if using_networking_tunnel && ! using_windows_exe; then
+        # BUG BUG BUG not yet implemented
+        skip "Test does not yet work from inside a WSL distro when using networking tunnel, since it requires WSL integration"
+    fi
     needs_port 80
 }
 
-@test 'factory reset' {
-    factory_reset
-}
-
-@test 'start k8s' {
-    start_kubernetes --kubernetes.options.traefik=true
-    wait_for_apiserver
-}
-
-get_host() {
-    if is_windows; then
-        echo "127.0.0.1"
+skip_unless_host_ip() {
+    if using_windows_exe; then
+        HOST_IP=$(netsh.exe interface ip show addresses 'vEthernet (WSL)' | grep -Po 'IP Address:\s+\K[\d.]+')
     else
-        echo "localhost"
+        # TODO determine if the Lima VM has its own IP address
+        HOST_IP=""
+    fi
+    if [[ -z $HOST_IP ]]; then
+        skip "Test requires a routable host ip address"
     fi
 }
 
@@ -45,18 +44,47 @@ assert_traefik_pods_are_down() {
 }
 
 assert_traefik_pods_are_up() {
-    run kubectl get --all-namespaces pods
+    ip_regex="^([0-9]{1,3}\.){3}[0-9]{1,3}$"
+    run kubectl -n kube-system get service traefik -o jsonpath="{.status.loadBalancer.ingress[0].ip}"
+    [[ $output =~ $ip_regex ]]
+}
 
-    if [[ $output != *"connection refused"* ]] &&
-        [[ $output != *"No resources found"* ]] &&
-        [[ $output != *"ContainerCreating"* ]] &&
-        [[ $output != *"Pending"* ]] &&
-        [[ $output != *"Terminating"* ]] &&
-        [[ $output =~ "traefik" ]]; then
-        return 0
-    else
-        return 1
+assert_curl() {
+    try --max 30 --delay 10 curl --silent --head "$@"
+    assert_success
+    assert_output --regexp 'HTTP/[0-9.]* 404'
+}
+
+refute_curl() {
+    run curl --head "$@"
+    assert_output --partial "curl: (7) Failed to connect"
+}
+
+assert_traefik() {
+    assert_curl "http://$1:80"
+    assert_curl --insecure "https://$1:443"
+}
+
+refute_traefik() {
+    refute_curl "http://$1:80"
+    refute_curl --insecure "https://$1:443"
+}
+
+assert_traefik_on_localhost() {
+    if using_networking_tunnel && ! using_windows_exe; then
+        # BUG BUG BUG not yet implemented
+        skip "Test does not yet work from inside a WSL distro when using networking tunnel"
     fi
+    assert_traefik localhost
+}
+
+@test 'factory reset' {
+    factory_reset
+}
+
+@test 'start k8s' {
+    start_kubernetes --kubernetes.options.traefik=true
+    wait_for_apiserver
 }
 
 @test 'disable traefik' {
@@ -67,33 +95,73 @@ assert_traefik_pods_are_up() {
     k3s_pid=$(get_service_pid k3s)
 
     # Disable traefik
-    rdctl set --kubernetes.options.traefik=FALSE
+    rdctl set --kubernetes.options.traefik=false
 
     # Wait until k3s has restarted
-    try --max 30 --delay 5 refute_service_pid k3s "$(k3s_pid)"
+    try --max 30 --delay 5 refute_service_pid k3s "${k3s_pid}"
 
     wait_for_apiserver
     # Check if the traefik pods go down
     try --max 30 --delay 10 assert_traefik_pods_are_down
 }
 
+@test 'no connection on localhost' {
+    refute_traefik localhost
+}
+
+@test 'no connection on host-ip' {
+    skip_unless_host_ip
+    refute_traefik "$HOST_IP"
+}
+
 @test 'enable traefik' {
     local k3s_pid
     k3s_pid=$(get_service_pid k3s)
 
     # Enable traefik
-    rdctl set --kubernetes.options.traefik=TRUE
+    rdctl set --kubernetes.options.traefik
 
     # Wait until k3s has restarted
-    try --max 30 --delay 5 refute_service_pid k3s "$(k3s_pid)"
+    try --max 30 --delay 5 refute_service_pid k3s "${k3s_pid}"
 
     wait_for_apiserver
     # Check if the traefik pods come up
     try --max 30 --delay 10 assert_traefik_pods_are_up
-    run try --max 30 --delay 10 curl --head "http://$(get_host):80"
-    assert_success
-    assert_output --regexp 'HTTP/[0-9.]* 404'
-    run try --max 30 --delay 10 curl --head --insecure "https://$(get_host):443"
-    assert_success
-    assert_output --regexp 'HTTP/[0-9.]* 404'
+}
+
+@test 'curl traefik via localhost' {
+    assert_traefik_on_localhost
+}
+
+@test 'curl traefik via host-ip while kubernetes.ingress.localhost-only is false' {
+    skip_unless_host_ip
+    assert_traefik "$HOST_IP"
+}
+
+@test 'set kubernetes.ingress.localhost-only to true' {
+    skip_unless_host_ip
+    if ! is_windows; then
+        skip "kubernetes.ingress.localhost-only is a Windows-only setting"
+    fi
+    rdctl set --kubernetes.options.traefik --kubernetes.ingress.localhost-only
+    wait_for_apiserver
+    # Check if the traefik pods come up
+    try --max 30 --delay 10 assert_traefik_pods_are_up
+}
+
+@test 'curl traefik via localhost while kubernetes.ingress.localhost-only is true' {
+    if ! is_windows; then
+        skip "Test requires kubernetes.ingress.localhost-only to be true"
+    fi
+    assert_traefik_on_localhost
+}
+
+@test 'curl traefik via host-ip while kubernetes.ingress.localhost-only is true' {
+    if ! is_windows; then
+        skip "Test requires kubernetes.ingress.localhost-only to be true"
+    fi
+    skip_unless_host_ip
+
+    # traefik should not be accessible on other interface
+    refute_traefik "$HOST_IP"
 }