Separate k3d dev scripts for upstream/downstream (#3166)

* Separate k3d dev scripts for upstream/downstream

* support PORT_OFFSET for upstream in both, to avoid conflicts with
  host ports
* for simplicity downstreams always have a number in their name

* fixup! Separate k3d dev scripts for upstream/downstream

* fixup! fixup! Separate k3d dev scripts for upstream/downstream

dev/README.md

@@ -74,7 +74,12 @@ export CHARTS_BRANCH=<branch_created_in_previous_step>
 ```
 3. Ensuring you have access to a running cluster in which to install Rancher.
 For instance, a set of k3d clusters (1 upstream + 1 downstream) can be created
-on your local machine through `dev/setup-k3ds`.
+on your local machine through:
+
+```
+dev/setup-k3d
+dev/setup-k3ds-downstream
+```
 
 4. Commenting out the pairs of lines (name + value) setting the following
 values for installing Rancher through Helm in script
@@ -149,7 +154,7 @@ export FLEET_E2E_CLUSTER_DOWNSTREAM=rancher-desktop
 export FLEET_E2E_CLUSTER=k3d-upstream
 export FLEET_E2E_CLUSTER_DOWNSTREAM=k3d-upstream
 
-# running multi-cluster tests in k3d (setup-k3ds)
+# running multi-cluster tests in k3d (setup-k3d;setup-k3ds-downstream)
 export FLEET_E2E_CLUSTER=k3d-upstream
 export FLEET_E2E_CLUSTER_DOWNSTREAM=k3d-downstream
 

dev/setup-k3d

@@ -1,17 +1,27 @@
 #!/bin/bash
+# Description: Create the management cluster
 
 set -euxo pipefail
 
+# k3d version list k3s
+# https://hub.docker.com/r/rancher/k3s/tags
+# k3d_args="-i docker.io/rancher/k3s:v1.22.15-k3s1"
+
 args=${k3d_args-}
 docker_mirror=${docker_mirror-}
+unique_api_port=${unique_api_port-36443}
+unique_tls_port=${unique_tls_port-443}
+METRICS_GITJOB_PORT=${METRICS_GITJOB_PORT-}
+METRICS_CONTROLLER_PORT=${METRICS_CONTROLLER_PORT-}
+
 name=${1-upstream}
-i=${2-0}
+offs=${2-0}
 
-if [ ! -z METRICS_CONTROLLER_PORT ]; then
+if [ -n "$METRICS_CONTROLLER_PORT" ]; then
     args="$args -p "${METRICS_CONTROLLER_PORT}:${METRICS_CONTROLLER_PORT}@server:0""
 fi
 
-if [ ! -z METRICS_GITJOB_PORT ]; then
+if [ -n "$METRICS_GITJOB_PORT" ]; then
     args="$args -p "${METRICS_GITJOB_PORT}:${METRICS_GITJOB_PORT}@server:0""
 fi
 
@@ -28,4 +38,12 @@ EOF
   args="$args --registry-config $TMP_CONFIG"
 fi
 
-k3d cluster create "$name" --servers 3 --api-port $(( 36443 + i )) -p "$(( 8080 + i )):8080@server:0" -p "$(( 8081 + i )):8081@server:0" -p "$(( 8082 + i )):8082@server:0" -p "$(( 8443 + i )):443@server:0" $args
+k3d cluster create "$name" \
+  --servers 3 \
+  --api-port "$unique_api_port" \
+  -p "$(( 8080 + offs )):8080@server:0" \
+  -p "$(( 8081 + offs )):8081@server:0" \
+  -p "$(( 8082 + offs )):8082@server:0" \
+  -p "$unique_tls_port:443@server:0" \
+  --k3s-arg '--tls-san=k3d-upstream-server-0@server:0' \
+  $args

dev/setup-k3ds → dev/setup-k3ds-downstream

@@ -1,10 +1,12 @@
 #!/bin/bash
+# Description: Create n downstream clusters
 
 set -euxo pipefail
 
 args=${k3d_args---network fleet}
 docker_mirror=${docker_mirror-}
-FLEET_E2E_DS_CLUSTER_COUNT=${FLEET_E2E_DS_CLUSTER_COUNT:-1}
+name="downstream"
+FLEET_E2E_DS_CLUSTER_COUNT=${FLEET_E2E_DS_CLUSTER_COUNT-1}
 
 if [ -n "$docker_mirror" ]; then
   TMP_CONFIG="$(mktemp)"
@@ -19,29 +21,13 @@ EOF
   args="$args --registry-config $TMP_CONFIG"
 fi
 
-# k3d version list k3s
-# https://hub.docker.com/r/rancher/k3s/tags
-#args="$args -i docker.io/rancher/k3s:v1.22.15-k3s1"
-
-k3d cluster create upstream \
-  --servers 3 \
-  --api-port 36443 \
-  -p '80:80@server:0' \
-  -p '443:443@server:0' \
-  --k3s-arg '--tls-san=k3d-upstream-server-0@server:0' \
-  $args
-
 for i in $(seq 1 "$FLEET_E2E_DS_CLUSTER_COUNT"); do
-  cluster="downstream"
-  if [[ $i -ne 1 ]]; then
-    cluster="${cluster}${i}"
-  fi
-  k3d cluster create "${cluster}" \
+  k3d cluster create "$name$i" \
     --servers 1 \
     --api-port $((36443 + i)) \
     -p "$((4080 + (1000 * i))):80@server:0" \
     -p "$((3443 + i)):443@server:0" \
-    --k3s-arg "--tls-san=k3d-${cluster}-server-0@server:0" \
+    --k3s-arg "--tls-san=k3d-$name$i-server-0@server:0" \
     $args
 done
 

dev/setup-multi-cluster

@@ -11,7 +11,14 @@ FLEET_E2E_DS_CLUSTER_COUNT=${FLEET_E2E_DS_CLUSTER_COUNT:-1}
 # Cleans with settings sourced, so it should be rather selective.
 ./dev/k3d-clean
 
-./dev/setup-k3ds
+PORT_OFFSET=0
+if [ -z "$external_ip" ];
+then
+ PORT_OFFSET=$(( RANDOM % 10001 ))
+fi
+
+./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" "$PORT_OFFSET"
+./dev/setup-k3ds-downstream
 ./dev/build-fleet
 ./dev/import-images-k3d
 ./dev/setup-fleet-multi-cluster

dev/setup-single-cluster

@@ -9,13 +9,13 @@ source ./dev/setup-cluster-config
 # Cleans with settings sourced, so it should be rather selective.
 ./dev/k3d-clean
 
-RANDOM_PORT=0
-if [ -z ${external_ip} ];
+PORT_OFFSET=0
+if [ -z "$external_ip" ];
 then
- RANDOM_PORT=$(( RANDOM % 10001 ))
+ PORT_OFFSET=$(( RANDOM % 10001 ))
 fi
 
-./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" "${RANDOM_PORT}"
+./dev/setup-k3d "${FLEET_E2E_CLUSTER#k3d-}" "$PORT_OFFSET"
 ./dev/build-fleet
 ./dev/import-images-k3d
 ./dev/setup-fleet "${FLEET_E2E_CLUSTER#k3d-}" '[