diff --git a/.github/workflows/terratest.yaml b/.github/workflows/terratest.yaml index 007ecaf828..b7efc5a11a 100644 --- a/.github/workflows/terratest.yaml +++ b/.github/workflows/terratest.yaml @@ -46,8 +46,21 @@ jobs: cluster-name: "test-gslb2" args: -c k3d/test-gslb2.yaml + - name: Create 3rd k3s Cluster + uses: AbsaOSS/k3d-action@v1.5.0 + with: + cluster-name: "test-gslb3" + args: -c k3d/test-gslb3.yaml + - name: K8GB deployment - run: make deploy-candidate + run: | + make deploy-test-version list-running-pods CLUSTERS_NUMBER=3 + echo "Cluster 1 (eu):" + kubectl get no -owide --context=k3d-test-gslb1 + echo "Cluster 2 (us):" + kubectl get no -owide --context=k3d-test-gslb2 + echo "Cluster 3 (cz):" + kubectl get no -owide --context=k3d-test-gslb3 - name: Terratest run: make terratest diff --git a/.github/workflows/upgrade-testing.yaml b/.github/workflows/upgrade-testing.yaml index 7dbacc87be..386ce4e303 100644 --- a/.github/workflows/upgrade-testing.yaml +++ b/.github/workflows/upgrade-testing.yaml @@ -46,11 +46,17 @@ jobs: cluster-name: "test-gslb2" args: -c k3d/test-gslb2.yaml + - name: Create 3rd k3s Cluster + uses: AbsaOSS/k3d-action@v1.5.0 + with: + cluster-name: "test-gslb3" + args: -c k3d/test-gslb3.yaml + - name: K8GB deploy stable version - run: make deploy-stable-version + run: make deploy-stable-version list-running-pods CLUSTERS_NUMBER=3 - name: K8GB deploy test version - run: make deploy-test-version + run: make deploy-test-version list-running-pods CLUSTERS_NUMBER=3 - name: Terratest run: make terratest diff --git a/.gitignore b/.gitignore index 4a3f493caf..d270cde4d5 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ chart/k8gb/charts/*.tgz # gh-pages | Jekyll generated files _site + +# generated k3d configs +k3d/test-gslb[!1-3].yaml diff --git a/Makefile b/Makefile index a6febb3843..1e57b0e298 100644 --- a/Makefile +++ b/Makefile @@ -24,8 +24,11 @@ endif ############################### # CONSTANTS ############################### -CLUSTER_GSLB1 = test-gslb1 -CLUSTER_GSLB2 = test-gslb2 +CLUSTERS_NUMBER ?= 2 +CLUSTER_IDS = $(shell seq $(CLUSTERS_NUMBER)) +CLUSTER_NAME ?= test-gslb +CLUSTER_GEO_TAGS ?= eu us cz af ru ap uk ca +CHART ?= k8gb/k8gb CLUSTER_GSLB_NETWORK = k3d-action-bridge-network GSLB_DOMAIN ?= cloud.example.com REPO = absaoss/k8gb @@ -33,7 +36,6 @@ VALUES_YAML ?= "" PODINFO_IMAGE_REPO ?= ghcr.io/stefanprodan/podinfo HELM_ARGS ?= K8GB_COREDNS_IP ?= kubectl get svc k8gb-coredns -n k8gb -o custom-columns='IP:spec.clusterIP' --no-headers -CLUSTER_GSLB2_HELM_ARGS ?= --set k8gb.clusterGeoTag='us' --set k8gb.extGslbClustersGeoTags='eu' --set k8gb.hostAlias.hostnames='{gslb-ns-eu-cloud.example.com}' LOG_FORMAT ?= simple LOG_LEVEL ?= debug CONTROLLER_GEN_VERSION ?= v0.7.0 @@ -52,6 +54,7 @@ DEMO_DELAY ?=5 ifndef NO_COLOR YELLOW=\033[0;33m CYAN=\033[1;36m +RED=\033[31m # no color NC=\033[0m endif @@ -112,59 +115,94 @@ demo: ## Execute end-to-end demo # spin-up local environment .PHONY: deploy-full-local-setup -deploy-full-local-setup: ## Deploy full local multicluster setup (k3d >= 4.2.0) - $(call create-local-cluster,$(CLUSTER_GSLB1)) - $(call create-local-cluster,$(CLUSTER_GSLB2)) +deploy-full-local-setup: ensure-cluster-size ## Deploy full local multicluster setup (k3d >= 4.2.0) + @echo "\n$(YELLOW)Creating $(CLUSTERS_NUMBER) k8s clusters$(NC)" + @for c in $(CLUSTER_IDS); do \ + $(MAKE) create-local-cluster CLUSTER_NAME=$(CLUSTER_NAME)$$c ;\ + done - $(call deploy-local-cluster,$(CLUSTER_GSLB1),$(CLUSTER_GSLB2),$(VERSION),,'k8gb/k8gb') - $(call deploy-local-cluster,$(CLUSTER_GSLB2),$(CLUSTER_GSLB1),$(VERSION),$(CLUSTER_GSLB2_HELM_ARGS),'k8gb/k8gb') - - @echo "\n$(YELLOW)Deploy test apps $(NC)" - $(call deploy-test-apps,$(CLUSTER_GSLB1)) - $(call deploy-test-apps,$(CLUSTER_GSLB2)) + $(MAKE) deploy-stable-version .PHONY: deploy-stable-version deploy-stable-version: - @echo "\n$(YELLOW) import $(CYAN)k8gb:$(STABLE_VERSION) $(YELLOW)to $(CYAN)$(CLUSTER_GSLB1), $(CLUSTER_GSLB2) $(NC)" - $(call deploy-local-cluster,$(CLUSTER_GSLB1),$(CLUSTER_GSLB2),$(STABLE_VERSION),,'k8gb/k8gb') - $(call deploy-local-cluster,$(CLUSTER_GSLB2),$(CLUSTER_GSLB1),$(STABLE_VERSION),$(CLUSTER_GSLB2_HELM_ARGS),'k8gb/k8gb') - - $(call list-running-pods,$(CLUSTER_GSLB1)) - $(call list-running-pods,$(CLUSTER_GSLB2)) + @for c in $(CLUSTER_IDS); do \ + $(MAKE) deploy-local-cluster CLUSTER_ID=$$c ;\ + done .PHONY: deploy-test-version deploy-test-version: ## Upgrade k8gb to the test version on existing clusters - @echo "\n$(YELLOW)import k8gb docker image to $(CYAN)$(CLUSTER_GSLB1), $(CLUSTER_GSLB2) $(NC)" + @echo "\n$(YELLOW)import k8gb docker image to all $(CLUSTERS_NUMBER) clusters$(NC)" + + @for c in $(CLUSTER_IDS); do \ + echo "\n$(CYAN)$(CLUSTER_NAME)$$c:$(NC)" ;\ + k3d image import $(REPO):$(SEMVER)-amd64 -c $(CLUSTER_NAME)$$c ;\ + done + + @for c in $(CLUSTER_IDS); do \ + $(MAKE) deploy-local-cluster CLUSTER_ID=$$c VERSION=$(SEMVER)-amd64 CHART='./chart/k8gb' ;\ + done + +.PHONY: list-running-pods +list-running-pods: + @for c in $(CLUSTER_IDS); do \ + echo "\n$(YELLOW)Local cluster $(CYAN)$(CLUSTER_NAME)$$c $(NC)" ;\ + kubectl get pods -A --context=k3d-$(CLUSTER_NAME)$$c ;\ + done + +create-local-cluster: + @echo "\n$(YELLOW)Create local cluster $(CYAN)$(CLUSTER_NAME) $(NC)" + k3d cluster create -c k3d/$(CLUSTER_NAME).yaml + +.PHONY: deploy-local-cluster +deploy-local-cluster: + @if [ -z $(CLUSTER_ID) ]; then echo invalid CLUSTER_ID value && exit 1; fi + @echo "\n$(YELLOW)Deploy local cluster $(CYAN)$(CLUSTER_NAME)$(CLUSTER_ID) $(NC)" + kubectl config use-context k3d-$(CLUSTER_NAME)$(CLUSTER_ID) - k3d image import $(REPO):$(SEMVER)-amd64 -c $(CLUSTER_GSLB1) - k3d image import $(REPO):$(SEMVER)-amd64 -c $(CLUSTER_GSLB2) + @echo "\n$(YELLOW)Create namespace $(NC)" + kubectl apply -f deploy/namespace.yaml - @echo "\n$(YELLOW)Upgrade GSLB operator from $(STABLE_VERSION) to $(SEMVER)-amd64 on k3d-$(CLUSTER_GSLB1) $(NC)" - kubectl config use-context k3d-$(CLUSTER_GSLB1) - $(call deploy-k8gb-with-helm,$(CLUSTER_GSLB1),$(CLUSTER_GSLB2),$(SEMVER)-amd64,,'./chart/k8gb') + @echo "\n$(YELLOW)Deploy GSLB operator from $(VERSION) $(NC)" + $(MAKE) deploy-k8gb-with-helm - @echo "\n$(YELLOW)Upgrade GSLB operator from $(STABLE_VERSION) to $(SEMVER)-amd64 on k3d-$(CLUSTER_GSLB2) $(NC)" - kubectl config use-context k3d-$(CLUSTER_GSLB2) - $(call deploy-k8gb-with-helm,$(CLUSTER_GSLB2),$(CLUSTER_GSLB1),$(SEMVER)-amd64,$(CLUSTER_GSLB2_HELM_ARGS),'./chart/k8gb') + @echo "\n$(YELLOW)Deploy Ingress $(NC)" + helm repo add --force-update nginx-stable https://kubernetes.github.io/ingress-nginx + helm repo update + helm -n k8gb upgrade -i nginx-ingress nginx-stable/ingress-nginx \ + --version 3.24.0 -f deploy/ingress/nginx-ingress-values.yaml - $(call list-running-pods,$(CLUSTER_GSLB1)) - $(call list-running-pods,$(CLUSTER_GSLB2)) + @echo "\n$(YELLOW)Deploy GSLB cr $(NC)" + $(MAKE) deploy-gslb-cr -.PHONY: upgrade-candidate -upgrade-candidate: release-images deploy-test-version + $(call deploy-test-apps) -.PHONY: deploy-candidate -deploy-candidate: ## Deploy test k8gb version together with CRs and test apps on top of existing clusters - @echo "\n$(YELLOW)import k8gb docker image to $(CYAN)$(CLUSTER_GSLB1), $(CLUSTER_GSLB2) $(NC)" + @echo "\n$(YELLOW)Wait until Ingress controller is ready $(NC)" + $(call wait-for-ingress) - k3d image import $(REPO):$(SEMVER)-amd64 -c $(CLUSTER_GSLB1) - k3d image import $(REPO):$(SEMVER)-amd64 -c $(CLUSTER_GSLB2) + @echo "\n$(CYAN)$(CLUSTER_NAME)$(CLUSTER_ID) $(YELLOW)deployed! $(NC)" - $(call deploy-local-cluster,$(CLUSTER_GSLB1),$(CLUSTER_GSLB2),$(SEMVER)-amd64,,'./chart/k8gb') - $(call deploy-local-cluster,$(CLUSTER_GSLB2),$(CLUSTER_GSLB1),$(SEMVER)-amd64,$(CLUSTER_GSLB2_HELM_ARGS),'./chart/k8gb') +.PHONY: deploy-gslb-cr +deploy-gslb-cr: ## Apply Gslb Custom Resources + kubectl apply -f deploy/crds/test-namespace.yaml + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml) + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml) + +.PHONY: upgrade-candidate +upgrade-candidate: release-images deploy-test-version - $(call list-running-pods,$(CLUSTER_GSLB1)) - $(call list-running-pods,$(CLUSTER_GSLB2)) +.PHONY: deploy-k8gb-with-helm +deploy-k8gb-with-helm: + @if [ -z $(CLUSTER_ID) ]; then echo invalid CLUSTER_ID value && exit 1; fi + helm repo add --force-update k8gb https://www.k8gb.io + cd chart/k8gb && helm dependency update + helm -n k8gb upgrade -i k8gb $(CHART) -f $(VALUES_YAML) \ + --set $(call get-helm-args,$(CLUSTER_ID)) \ + --set k8gb.reconcileRequeueSeconds=10 \ + --set k8gb.dnsZoneNegTTL=10 \ + --set k8gb.imageTag=${VERSION:"stable"=""} \ + --set k8gb.log.format=$(LOG_FORMAT) \ + --set k8gb.log.level=$(LOG_LEVEL) \ + --wait --timeout=2m0s .PHONY: deploy-gslb-operator deploy-gslb-operator: ## Deploy k8gb operator @@ -174,33 +212,25 @@ deploy-gslb-operator: ## Deploy k8gb operator --set k8gb.log.format=$(LOG_FORMAT) --set k8gb.log.level=$(LOG_LEVEL) -.PHONY: deploy-gslb-cr -deploy-gslb-cr: ## Apply Gslb Custom Resources - kubectl apply -f deploy/crds/test-namespace.yaml - $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml) - $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml) - -.PHONY: deploy-test-apps -deploy-test-apps: ## Deploy testing workloads - kubectl apply -f deploy/crds/test-namespace.yaml - $(call deploy-test-apps) - # destroy local test environment .PHONY: destroy-full-local-setup destroy-full-local-setup: ## Destroy full local multicluster setup - k3d cluster delete $(CLUSTER_GSLB1) - k3d cluster delete $(CLUSTER_GSLB2) + @for c in $(CLUSTER_IDS); do \ + k3d cluster delete $(CLUSTER_NAME)$$c ;\ + done docker network rm $(CLUSTER_GSLB_NETWORK) .PHONY: deploy-prometheus deploy-prometheus: - $(call deploy-prometheus,$(CLUSTER_GSLB1)) - $(call deploy-prometheus,$(CLUSTER_GSLB2)) + @for c in $(CLUSTER_IDS); do \ + $(call deploy-prometheus,$(CLUSTER_NAME)$$c) ;\ + done .PHONY: uninstall-prometheus uninstall-prometheus: - $(call uninstall-prometheus,$(CLUSTER_GSLB1)) - $(call uninstall-prometheus,$(CLUSTER_GSLB2)) + @for c in $(CLUSTER_IDS); do \ + $(call uninstall-prometheus,$(CLUSTER_NAME)$$c) ;\ + done .PHONY: deploy-grafana deploy-grafana: @@ -248,6 +278,16 @@ docker-manifest: --os linux --arch arm64 docker manifest push ${IMG} +.PHONY: ensure-cluster-size +ensure-cluster-size: + @if [ $(CLUSTERS_NUMBER) -gt 8 ] ; then \ + echo "$(RED)$(CLUSTERS_NUMBER) clusters is probably way too many$(NC)" ;\ + echo "$(RED)you will probably hit resource limits or port collisions, gook luck you are on your own$(NC)" ;\ + fi + @if [ $(CLUSTERS_NUMBER) -gt 3 ] ; then \ + ./k3d/generate-yaml.sh $(CLUSTERS_NUMBER) ;\ + fi + .PHONY: goreleaser goreleaser: go install github.com/goreleaser/goreleaser@v0.184.0 @@ -368,6 +408,12 @@ test-failover: # executes terra-tests .PHONY: terratest terratest: # Run terratest suite + @$(eval RUNNING_CLUSTERS := $(shell k3d cluster list --no-headers | grep $(CLUSTER_NAME) -c)) + @if [ $(RUNNING_CLUSTERS) != 3 ] ; then \ + echo "$(RED)Make sure the you run the tests againt 3 running clusters$(NC)" ;\ + echo "$(RED)Currently $(RUNNING_CLUSTERS) running clusters were discovered$(NC)" ;\ + exit 1 ;\ + fi cd terratest/test/ && go mod download && go test -v -timeout 15m -parallel=12 .PHONY: website @@ -394,47 +440,6 @@ help: ## Show this help # FUNCTIONS ############################### -define create-local-cluster - @echo "\n$(YELLOW)Deploy local cluster $(CYAN)$1 $(NC)" - k3d cluster create -c k3d/$1.yaml -endef - -define deploy-k8gb-with-helm - helm repo add --force-update k8gb https://www.k8gb.io - cd chart/k8gb && helm dependency update - helm -n k8gb upgrade -i k8gb $5 -f $(VALUES_YAML) \ - --set k8gb.hostAlias.enabled=true \ - --set k8gb.hostAlias.ip="`$(call get-host-alias-ip,k3d-$1,k3d-$2)`" \ - --set k8gb.reconcileRequeueSeconds=10 \ - --set k8gb.dnsZoneNegTTL=10 \ - --set k8gb.imageTag=$3 $4 \ - --set k8gb.log.format=$(LOG_FORMAT) \ - --set k8gb.log.level=$(LOG_LEVEL) \ - --wait --timeout=2m0s -endef - -define deploy-local-cluster - @echo "\n$(YELLOW)Local cluster $(CYAN)$1 $(NC)" - kubectl config use-context k3d-$1 - - @echo "\n$(YELLOW)Create namespace $(NC)" - kubectl apply -f deploy/namespace.yaml - - @echo "\n$(YELLOW)Deploy GSLB operator from ${3} $(NC)" - $(call deploy-k8gb-with-helm,$1,$2,${3:"stable"=""},$4,$5) - - @echo "\n$(YELLOW)Deploy Ingress $(NC)" - helm repo add --force-update nginx-stable https://kubernetes.github.io/ingress-nginx - helm repo update - helm -n k8gb upgrade -i nginx-ingress nginx-stable/ingress-nginx \ - --version 3.24.0 -f deploy/ingress/nginx-ingress-values.yaml - - @echo "\n$(YELLOW)Wait until Ingress controller is ready $(NC)" - $(call wait-for-ingress) - - @echo "\n$(CYAN)$1 $(YELLOW)deployed! $(NC)" -endef - define apply-cr sed -i 's/cloud\.example\.com/$(GSLB_DOMAIN)/g' "$1" kubectl apply -f "$1" @@ -442,14 +447,6 @@ define apply-cr endef define deploy-test-apps - $(if $1,@kubectl config use-context k3d-$1, \ - @echo "Current context: $(shell kubectl config current-context)") - - @echo "\n$(YELLOW)Deploy GSLB CR $(NC)" - kubectl apply -f deploy/crds/test-namespace.yaml - $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml) - $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml) - @echo "\n$(YELLOW)Deploy podinfo $(NC)" kubectl apply -f deploy/test-apps helm repo add podinfo https://stefanprodan.github.io/podinfo @@ -464,12 +461,24 @@ define get-cluster-geo-tag kubectl -n k8gb describe deploy k8gb | awk '/CLUSTER_GEO_TAG/ { printf $$2 }' endef +nth-geo-tag = $(subst $1_,,$(filter $1_%, $(join $(addsuffix _,$(CLUSTER_IDS)),$(CLUSTER_GEO_TAGS)))) + +define get-ext-tags +$(shell echo $(foreach cl,$(filter-out $1,$(shell seq $(CLUSTERS_NUMBER))),$(call nth-geo-tag,$(cl))) + | sed 's/ /\\,/g') +endef + +define get-helm-args +k8gb.clusterGeoTag='$(call nth-geo-tag,$1)' --set k8gb.extGslbClustersGeoTags='$(call get-ext-tags,$1)' \ +$(foreach cl,$(shell seq $(CLUSTERS_NUMBER)), --set "k8gb.hostAliases[$$(( $(cl) - 1 ))].ip=$(shell $(call get-host-alias-ip,k3d-$(CLUSTER_NAME)$1,k3d-$(CLUSTER_NAME)$(cl)))" --set "k8gb.hostAliases[$$(( $(cl) - 1 ))].hostnames={gslb-ns-$(call nth-geo-tag,$(cl))-cloud.example.com}") +endef + # get-host-alias-ip switch to second context ($2), search for IP and switch back to first context ($1) # function returns one IP address define get-host-alias-ip - kubectl config use-context $2 > /dev/null && \ - kubectl get nodes $2-agent-0 -o custom-columns='IP:status.addresses[0].address' --no-headers && \ - kubectl config use-context $1 > /dev/null +kubectl config use-context $2 > /dev/null && \ +kubectl get nodes $2-agent-0 -o custom-columns='IP:status.addresses[0].address' --no-headers && \ +kubectl config use-context $1 > /dev/null endef define hit-testapp-host @@ -483,7 +492,8 @@ define init-test-strategy kubectl apply -f $1 kubectl config use-context k3d-test-gslb1 kubectl apply -f $1 - $(call testapp-set-replicas,2) + $(MAKE) start-test-app + endef define testapp-set-replicas @@ -531,32 +541,24 @@ define debug dlv $1 endef -define list-running-pods - @echo "\n$(YELLOW)Local cluster $(CYAN)$1 $(NC)" - kubectl get pods -A --context=k3d-$1 -endef - define deploy-prometheus - @echo "\n$(YELLOW)Local cluster $(CYAN)$1$(NC)" - - @echo "\n$(YELLOW)Set annotations on pods that will be scraped by prometheus$(NC)" - kubectl annotate pods -l name=k8gb -n k8gb --overwrite prometheus.io/scrape="true" --context=k3d-$1 - kubectl annotate pods -l name=k8gb -n k8gb --overwrite prometheus.io/port="8080" --context=k3d-$1 - - @echo "\n$(YELLOW)install prometheus $(NC)" - helm repo add prometheus-community https://prometheus-community.github.io/helm-charts - helm repo update + echo "\n$(YELLOW)Local cluster $(CYAN)$1$(NC)" ;\ + echo "\n$(YELLOW)Set annotations on pods that will be scraped by prometheus$(NC)" ;\ + kubectl annotate pods -l name=k8gb -n k8gb --overwrite prometheus.io/scrape="true" --context=k3d-$1 ;\ + kubectl annotate pods -l name=k8gb -n k8gb --overwrite prometheus.io/port="8080" --context=k3d-$1 ;\ + echo "\n$(YELLOW)install prometheus $(NC)" ;\ + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts ;\ + helm repo update ;\ helm -n k8gb upgrade -i prometheus prometheus-community/prometheus -f deploy/prometheus/values.yaml \ --version 14.2.0 \ --wait --timeout=2m0s \ - --kube-context=k3d-$1 + --kube-context=k3d-$1 ; endef define uninstall-prometheus - @echo "\n$(YELLOW)Local cluster $(CYAN)$1$(NC)" - - @echo "\n$(YELLOW)uninstall prometheus $(NC)" - helm uninstall prometheus -n k8gb --kube-context=k3d-$1 - kubectl annotate pods -l name=k8gb -n k8gb prometheus.io/scrape- --context=k3d-$1 - kubectl annotate pods -l name=k8gb -n k8gb prometheus.io/port- --context=k3d-$1 + echo "\n$(YELLOW)Local cluster $(CYAN)$1$(NC)" ;\ + echo "\n$(YELLOW)uninstall prometheus $(NC)" ;\ + helm uninstall prometheus -n k8gb --kube-context=k3d-$1 ;\ + kubectl annotate pods -l name=k8gb -n k8gb prometheus.io/scrape- --context=k3d-$1 ;\ + kubectl annotate pods -l name=k8gb -n k8gb prometheus.io/port- --context=k3d-$1 ; endef diff --git a/chart/k8gb/.helmignore b/chart/k8gb/.helmignore index 50af031725..6e49137e5a 100644 --- a/chart/k8gb/.helmignore +++ b/chart/k8gb/.helmignore @@ -20,3 +20,6 @@ .idea/ *.tmproj .vscode/ + +LICENSE +README.md diff --git a/chart/k8gb/templates/operator.yaml b/chart/k8gb/templates/operator.yaml index 614673ce7e..97865367bc 100644 --- a/chart/k8gb/templates/operator.yaml +++ b/chart/k8gb/templates/operator.yaml @@ -15,13 +15,9 @@ spec: labels: name: k8gb spec: - {{- if .Values.k8gb.hostAlias.enabled }} + {{- if .Values.k8gb.hostAliases }} hostAliases: - - ip: "{{ .Values.k8gb.hostAlias.ip }}" - hostnames: - {{- range.Values.k8gb.hostAlias.hostnames }} - - {{ . }} - {{- end }} + {{- toYaml .Values.k8gb.hostAliases | nindent 8 }} {{- end }} serviceAccountName: k8gb containers: diff --git a/chart/k8gb/values.yaml b/chart/k8gb/values.yaml index 739d39e356..0e57fc088a 100644 --- a/chart/k8gb/values.yaml +++ b/chart/k8gb/values.yaml @@ -23,12 +23,14 @@ k8gb: clusterGeoTag: "eu" # -- comma-separated list of external gslb geo tags to pair with extGslbClustersGeoTags: "us" - hostAlias: - # -- use https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ inside operator pod. Useful for advanced testing scenarios and to break dependency on EdgeDNS for cross k8gb collaboration - enabled: false - ip: "172.17.0.1" - hostnames: - - "gslb-ns-us-cloud.example.com" + # -- use https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ inside operator pod. Useful for advanced testing scenarios and to break dependency on EdgeDNS for cross k8gb collaboration + hostAliases: + # - ip: "172.17.0.1" + # hostnames: + # - "gslb-ns-us-cloud.example.com" + # - ip: "172.17.0.2" + # hostnames: + # - "gslb-ns-eu-cloud.example.com" # -- Reconcile time in seconds reconcileRequeueSeconds: 30 log: diff --git a/k3d/generate-yaml.sh b/k3d/generate-yaml.sh new file mode 100755 index 0000000000..a144481eb2 --- /dev/null +++ b/k3d/generate-yaml.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# use this simple script for generating additional config files for k3d +# note: make sure the generated ports doesn't collide with your local environment +# for instance on mac port 88 might be taken by kdc + +[[ $# != 1 ]] && echo "Usage: $0 " && exit 1 +[[ -z "${1##*[!0-9]*}" ]] && echo "'$1' is not a positive integer" && exit 1 +UPTO=$1 +DIR="${DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )}" + +echo Generating configs for $UPTO clusters.. +for c in $(seq $UPTO); do + export CLUSTER_INDEX=$(( 1 + $c )) + export PORT_HTTP=$(( 80 + $c )) + export PORT_HTTPS=$(( 443 + $c )) + export PORT_PROM=$(( 9080 + $c )) + export PORT_DNS=$(( 5053 + $c )) + cat ${DIR}/gslb.yaml.tmpl | envsubst > ${DIR}/test-gslb${CLUSTER_INDEX}.yaml +done diff --git a/k3d/gslb.yaml.tmpl b/k3d/gslb.yaml.tmpl new file mode 100644 index 0000000000..52596d7540 --- /dev/null +++ b/k3d/gslb.yaml.tmpl @@ -0,0 +1,25 @@ +apiVersion: k3d.io/v1alpha2 +kind: Simple +name: test-gslb$CLUSTER_INDEX +image: docker.io/rancher/k3s:v1.21.2-k3s1 +agents: 1 +network: k3d-action-bridge-network +ports: +- port: $PORT_HTTP:80 + nodeFilters: + - agent[0] +- port: $PORT_HTTPS:443 + nodeFilters: + - agent[0] +- port: $PORT_PROM:30090 + nodeFilters: + - agent[0] +- port: $PORT_DNS:53/udp + nodeFilters: + - agent[0] +options: + k3d: + disableLoadbalancer: true + k3s: # options passed on to K3s itself + extraServerArgs: # additional arguments passed to the `k3s server` command; same as `--k3s-server-arg` + - --no-deploy=traefik,servicelb,metrics-server,local-storage diff --git a/k3d/test-gslb3.yaml b/k3d/test-gslb3.yaml new file mode 100644 index 0000000000..7a3ce79e82 --- /dev/null +++ b/k3d/test-gslb3.yaml @@ -0,0 +1,25 @@ +apiVersion: k3d.io/v1alpha2 +kind: Simple +name: test-gslb3 +image: docker.io/rancher/k3s:v1.21.2-k3s1 +agents: 1 +network: k3d-action-bridge-network +ports: +- port: 82:80 + nodeFilters: + - agent[0] +- port: 445:443 + nodeFilters: + - agent[0] +- port: 9082:30090 + nodeFilters: + - agent[0] +- port: 5055:53/udp + nodeFilters: + - agent[0] +options: + k3d: + disableLoadbalancer: true + k3s: # options passed on to K3s itself + extraServerArgs: # additional arguments passed to the `k3s server` command; same as `--k3s-server-arg` + - --no-deploy=traefik,servicelb,metrics-server,local-storage diff --git a/terratest/test/init.go b/terratest/test/init.go index 3abaa14357..ecfd01b02d 100644 --- a/terratest/test/init.go +++ b/terratest/test/init.go @@ -30,6 +30,7 @@ var ( func init() { p1, _ := env.GetEnvAsIntOrFallback("DNS_SERVER1_PORT", 5053) p2, _ := env.GetEnvAsIntOrFallback("DNS_SERVER2_PORT", 5054) + p3, _ := env.GetEnvAsIntOrFallback("DNS_SERVER3_PORT", 5055) settings = utils.TestSettings{ DNSZone: env.GetEnvAsStringOrFallback("GSLB_DOMAIN", "cloud.example.com"), PrimaryGeoTag: env.GetEnvAsStringOrFallback("PRIMARY_GEO_TAG", "eu"), @@ -38,8 +39,11 @@ func init() { Port1: p1, DNSServer2: env.GetEnvAsStringOrFallback("DNS_SERVER2", "localhost"), Port2: p2, + DNSServer3: env.GetEnvAsStringOrFallback("DNS_SERVER3", "localhost"), + Port3: p3, Cluster1: env.GetEnvAsStringOrFallback("K8GB_CLUSTER1", "k3d-test-gslb1"), Cluster2: env.GetEnvAsStringOrFallback("K8GB_CLUSTER2", "k3d-test-gslb2"), + Cluster3: env.GetEnvAsStringOrFallback("K8GB_CLUSTER3", "k3d-test-gslb3"), PodinfoImage: env.GetEnvAsStringOrFallback("PODINFO_IMAGE_REPO", "ghcr.io/stefanprodan/podinfo"), } } diff --git a/terratest/test/k8gb_full_roundrobin_test.go b/terratest/test/k8gb_full_roundrobin_test.go index 06f7af5404..20ec8d1b0d 100644 --- a/terratest/test/k8gb_full_roundrobin_test.go +++ b/terratest/test/k8gb_full_roundrobin_test.go @@ -29,60 +29,99 @@ func TestFullRoundRobin(t *testing.T) { const host = "roundrobin-test.cloud.example.com" const gslbPath = "../examples/roundrobin2.yaml" - instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + instanceEU, err := utils.NewWorkflow(t, settings.Cluster1, settings.Port1). WithGslb(gslbPath, host). WithTestApp("eu"). Start() require.NoError(t, err) defer instanceEU.Kill() - instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + instanceUS, err := utils.NewWorkflow(t, settings.Cluster2, settings.Port2). WithGslb(gslbPath, host). WithTestApp("us"). Start() require.NoError(t, err) defer instanceUS.Kill() + instanceCZ, err := utils.NewWorkflow(t, settings.Cluster3, settings.Port3). + WithGslb(gslbPath, host). + WithTestApp("cz"). + Start() + require.NoError(t, err) + defer instanceCZ.Kill() - t.Run("round-robin on two concurrent clusters with podinfo running", func(t *testing.T) { + t.Run("round-robin on three concurrent clusters with podinfo running", func(t *testing.T) { err = instanceEU.WaitForAppIsRunning() require.NoError(t, err) err = instanceUS.WaitForAppIsRunning() require.NoError(t, err) + err = instanceCZ.WaitForAppIsRunning() + require.NoError(t, err) }) euLocalTargets := instanceEU.GetLocalTargets() usLocalTargets := instanceUS.GetLocalTargets() - expectedIPs := append(euLocalTargets, usLocalTargets...) + czLocalTargets := instanceCZ.GetLocalTargets() + euAndCZTargets := append(euLocalTargets, czLocalTargets...) + czAndUSTargets := append(czLocalTargets, usLocalTargets...) + expectedIPs := append(euAndCZTargets, usLocalTargets...) - t.Run("kill podinfo on the second cluster", func(t *testing.T) { + t.Run("kill podinfo on the second (us) cluster", func(t *testing.T) { instanceUS.StopTestApp() - err = instanceEU.WaitForExpected(euLocalTargets) + err = instanceEU.WaitForExpected(euAndCZTargets) require.NoError(t, err) + err = instanceUS.WaitForExpected(euAndCZTargets) + require.NoError(t, err) + err = instanceCZ.WaitForExpected(euAndCZTargets) + require.NoError(t, err) + }) + + t.Run("kill podinfo on the third (cz) cluster", func(t *testing.T) { + instanceCZ.StopTestApp() err = instanceUS.WaitForExpected(euLocalTargets) require.NoError(t, err) + err = instanceEU.WaitForExpected(euLocalTargets) + require.NoError(t, err) + err = instanceCZ.WaitForExpected(euLocalTargets) + require.NoError(t, err) }) - t.Run("kill podinfo on the first cluster", func(t *testing.T) { + t.Run("kill podinfo on the first (eu) cluster", func(t *testing.T) { instanceEU.StopTestApp() err = instanceUS.WaitForExpected([]string{}) require.NoError(t, err) err = instanceEU.WaitForExpected([]string{}) require.NoError(t, err) + err = instanceCZ.WaitForExpected([]string{}) + require.NoError(t, err) + }) + + t.Run("start podinfo on the third (cz) cluster", func(t *testing.T) { + instanceCZ.StartTestApp() + err = instanceEU.WaitForExpected(czLocalTargets) + require.NoError(t, err) + err = instanceUS.WaitForExpected(czLocalTargets) + require.NoError(t, err) + err = instanceCZ.WaitForExpected(czLocalTargets) + require.NoError(t, err) }) - t.Run("start podinfo on the second cluster", func(t *testing.T) { + t.Run("start podinfo on the second (us) cluster", func(t *testing.T) { instanceUS.StartTestApp() - err = instanceEU.WaitForExpected(usLocalTargets) + err = instanceEU.WaitForExpected(czAndUSTargets) require.NoError(t, err) - err = instanceUS.WaitForExpected(usLocalTargets) + err = instanceUS.WaitForExpected(czAndUSTargets) + require.NoError(t, err) + err = instanceCZ.WaitForExpected(czAndUSTargets) require.NoError(t, err) }) - t.Run("start podinfo on the first cluster", func(t *testing.T) { + t.Run("start podinfo on the first (eu) cluster", func(t *testing.T) { // start app in the both clusters instanceEU.StartTestApp() err = instanceEU.WaitForExpected(expectedIPs) require.NoError(t, err) err = instanceUS.WaitForExpected(expectedIPs) require.NoError(t, err) + err = instanceCZ.WaitForExpected(expectedIPs) + require.NoError(t, err) }) } diff --git a/terratest/utils/test_settings.go b/terratest/utils/test_settings.go index 0aa82d0631..9af2a5a32f 100644 --- a/terratest/utils/test_settings.go +++ b/terratest/utils/test_settings.go @@ -25,7 +25,10 @@ type TestSettings struct { Port1 int DNSServer2 string Port2 int + DNSServer3 string + Port3 int Cluster1 string Cluster2 string + Cluster3 string PodinfoImage string }