diff --git a/.github/workflows/terratest.yaml b/.github/workflows/terratest.yaml index 4e954f6c4f..f3bd5565e9 100644 --- a/.github/workflows/terratest.yaml +++ b/.github/workflows/terratest.yaml @@ -25,14 +25,6 @@ jobs: config: deploy/kind/cluster2-terratest.yaml wait: 180s - name: K8GB deployment - env: - NODE_ROLE: control-plane - TEST_CURRENT_COMMIT: yes - run: | - export VERSION=$(make version) - ./deploy/full.sh - kubectl get pods -A - make use-second-context - kubectl get pods -A + run: make deploy-full-terratest-setup - name: Terratest run: make terratest diff --git a/Makefile b/Makefile index 344d4446bf..53e4ac3adb 100644 --- a/Makefile +++ b/Makefile @@ -1,267 +1,161 @@ -REPO ?= absaoss/k8gb -# Current Operator version -VERSION ?= $$(helm show chart chart/k8gb/|awk '/appVersion:/ {print $$2}') -K8GB_IMAGE_TAG ?= v$(VERSION) -# Default bundle image tag -BUNDLE_IMG ?= controller-bundle:$(VERSION) -# Options for 'bundle-build' -ifneq ($(origin CHANNELS), undefined) -BUNDLE_CHANNELS := --channels=$(CHANNELS) -endif -ifneq ($(origin DEFAULT_CHANNEL), undefined) -BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) -endif -BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) - -# Image URL to use all building/pushing image targets -IMG ?= $(REPO):$(K8GB_IMAGE_TAG) -# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) -CRD_OPTIONS ?= "crd:trivialVersions=true" - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -all: manager - -# Run tests -test: lint generate fmt vet manifests - go test ./... -coverprofile cover.out - -# Build manager binary -manager: generate fmt vet - go build -o bin/manager main.go - -# Run against the configured Kubernetes cluster in ~/.kube/config -run: generate fmt vet manifests - go run ./main.go - -# Install CRDs into a cluster -install: manifests kustomize - $(KUSTOMIZE) build config/crd | kubectl apply -f - - -# Uninstall CRDs from a cluster -uninstall: manifests kustomize - $(KUSTOMIZE) build config/crd | kubectl delete -f - +############################### +# CONSTANTS +############################### +CLUSTER_GSLB1 = test-gslb1 +CLUSTER_GSLB2 = test-gslb2 +GSLB_DOMAIN ?= cloud.example.com +REPO = absaoss/k8gb +VALUES_YAML ?= chart/k8gb/values.yaml +PODINFO_IMAGE_REPO ?= stefanprodan/podinfo +HELM_ARGS ?= --set k8gb.clusterGeoTag='us' --set k8gb.extGslbClustersGeoTags='eu' --set k8gb.hostAlias.hostnames='{test-gslb-ns-eu.example.com,test-gslb-failover-ns-eu.example.com}' +K8GB_COREDNS_IP ?= kubectl get svc k8gb-coredns -n k8gb -o custom-columns='IP:spec.clusterIP' --no-headers +ETCD_DEBUG_IMAGE ?= quay.io/coreos/etcd:v3.2.25 -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests kustomize - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | kubectl apply -f - +# terratest +GITACTION_TERRATEST_DOCKER_REPO_PORT ?= 5000 +GITACTION_TERRATEST_DOCKER_REPO_NAME ?= kind-registry +GITACTION_TERRATEST_DOCKER_REPO_IMAGE ?= registry:2 -# Generate manifests e.g. CRD, RBAC etc. -manifests: controller-gen - $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases +# log +YELLOW=\033[0;33m +CYAN=\033[1;36m +# no Color +NC=\033[0m -# Run go fmt against code -fmt: - go fmt ./... +NO_VALUE ?= no_value -# Run go vet against code -vet: - go vet ./... +############################### +# VARIABLES +############################### -# Generate code -generate: controller-gen - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." +# shell script inspects whether kind registry is running or not +KIND_REGISTRY_RUNNING ?= $(shell docker inspect -f '{{.State.Running}}' "test-gslb2-control-plane" || true) -# Build the docker image -docker-build: test - docker build . -t ${IMG} +VERSION ?= $(shell helm show chart chart/k8gb/|awk '/appVersion:/ {print $$2}') -# Push the docker image -docker-push: - docker push ${IMG} +# image URL to use all building/pushing image targets +IMG ?= $(REPO):v$(VERSION) -# Build and push the docker image exclusively for testing using commit hash -docker-test-build-push: test - $(call docker-test-build-push) +# default bundle image tag +BUNDLE_IMG ?= controller-bundle:$(VERSION) -# find or download controller-gen -# download controller-gen if necessary -controller-gen: -ifeq (, $(shell which controller-gen)) - @{ \ - set -e ;\ - CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$CONTROLLER_GEN_TMP_DIR ;\ - go mod init tmp ;\ - go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\ - rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ - } -CONTROLLER_GEN=$(GOBIN)/controller-gen -else -CONTROLLER_GEN=$(shell which controller-gen) +# options for 'bundle-build' +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) endif - -kustomize: -ifeq (, $(shell which kustomize)) - @{ \ - set -e ;\ - KUSTOMIZE_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$KUSTOMIZE_GEN_TMP_DIR ;\ - go mod init tmp ;\ - go get sigs.k8s.io/kustomize/kustomize/v3@v3.5.4 ;\ - rm -rf $$KUSTOMIZE_GEN_TMP_DIR ;\ - } -KUSTOMIZE=$(GOBIN)/kustomize -else -KUSTOMIZE=$(shell which kustomize) +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) -# Generate bundle manifests and metadata, then validate generated files. -.PHONY: bundle -bundle: manifests - operator-sdk generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) - $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) - operator-sdk bundle validate ./bundle +# create GOBIN if not specified +ifndef GOBIN +GOBIN=$(shell go env GOPATH)/bin +endif -# Build the bundle image. -.PHONY: bundle-build -bundle-build: - docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . +CONTROLLER_GEN_PATH ?= $(shell which controller-gen || echo $(NO_VALUE)) -## Special k8gb make part +KUSTOMIZE_PATH ?= $(shell which kustomize || echo $(NO_VALUE)) -VALUES_YAML ?= chart/k8gb/values.yaml -HELM_ARGS ?= -ETCD_DEBUG_IMAGE ?= quay.io/coreos/etcd:v3.2.25 -GSLB_DOMAIN ?= cloud.example.com -HOST_ALIAS_IP1 ?= 172.17.0.9 -HOST_ALIAS_IP2 ?= 172.17.0.5 -K8GB_IMAGE_REPO ?= absaoss/k8gb -K8GB_IMAGE_TAG ?= v$(VERSION) -K8GB_COREDNS_IP ?= kubectl get svc k8gb-coredns -n k8gb -o custom-columns='IP:spec.clusterIP' --no-headers -PODINFO_IMAGE_REPO ?= stefanprodan/podinfo COMMIT_HASH ?= $(shell git rev-parse --short HEAD) -.PHONY: debug-local -debug-local: create-test-ns - kubectl apply -f ./deploy/crds/k8gb.absa.oss_gslbs_crd.yaml - kubectl apply -f ./deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml - operator-sdk run --local --namespace=test-gslb --enable-delve - -.PHONY: lint -lint: - golangci-lint run - -.PHONY: terratest -terratest: - cd terratest/test/ && go mod download && go test -v - -.PHONY: dns-tools -dns-tools: - kubectl -n k8gb get svc k8gb-coredns - kubectl -n k8gb run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools - -.PHONY: dns-smoke-test -dns-smoke-test: - kubectl -n k8gb run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools --command -- /usr/bin/dig @k8gb-coredns app3.cloud.example.com - -.PHONY: deploy-local-cluster -deploy-local-cluster: - kind create cluster --config=deploy/kind/cluster.yaml --name test-gslb1 +############################### +# TARGETS +############################### -.PHONY: deploy-two-local-clusters -deploy-two-local-clusters: - kind create cluster --config=deploy/kind/cluster.yaml --name test-gslb1 - kind create cluster --config=deploy/kind/cluster2.yaml --name test-gslb2 +.PHONY: clean-test-apps +clean-test-apps: + kubectl delete -f deploy/test-apps + helm -n test-gslb uninstall backend + helm -n test-gslb uninstall frontend -.PHONY: use-first-context -use-first-context: - kubectl config use-context kind-test-gslb1 +# see: https://dev4devs.com/2019/05/04/operator-framework-how-to-debug-golang-operator-projects/ +.PHONY: debug-idea +debug-idea: export WATCH_NAMESPACE=test-gslb +debug-idea: + $(call debug,debug --headless --listen=:2345 --api-version=2) -.PHONY: use-second-context -use-second-context: - kubectl config use-context kind-test-gslb2 +.PHONY: debug-test-etcd +debug-test-etcd: + kubectl run --rm -i --tty --env="ETCDCTL_API=3" --env="ETCDCTL_ENDPOINTS=http://etcd-cluster-client:2379" --namespace k8gb etcd-test --image "$(ETCD_DEBUG_IMAGE)" --restart=Never -- /bin/sh -.PHONY: deploy-first-k8gb -deploy-first-k8gb: HELM_ARGS = --set k8gb.hostAlias.enabled=true --set k8gb.hostAlias.ip="$(HOST_ALIAS_IP1)" --set k8gb.imageRepo=$(K8GB_IMAGE_REPO) -deploy-first-k8gb: deploy-gslb-operator deploy-local-ingress +.PHONY: demo-roundrobin +demo-roundrobin: + @$(call demo-host, "app3.cloud.example.com") -.PHONY: deploy-second-k8gb -deploy-second-k8gb: HELM_ARGS = --set k8gb.hostAlias.enabled=true --set k8gb.clusterGeoTag="us" --set k8gb.extGslbClustersGeoTags="eu" --set k8gb.hostAlias.hostnames="{test-gslb-ns-eu.example.com,test-gslb-failover-ns-eu.example.com}" --set k8gb.hostAlias.ip="$(HOST_ALIAS_IP2)" --set k8gb.imageRepo=$(K8GB_IMAGE_REPO) -deploy-second-k8gb: deploy-gslb-operator deploy-local-ingress +.PHONY: demo-failover +demo-failover: + @$(call demo-host, "failover.cloud.example.com") +# spin-up local environment .PHONY: deploy-full-local-setup -deploy-full-local-setup: deploy-two-local-clusters - ADDITIONAL_TARGETS=deploy-test-apps VERSION=$(VERSION) ./deploy/full.sh - -.PHONY: destroy-full-local-setup -destroy-full-local-setup: destroy-two-local-clusters +deploy-full-local-setup: + $(call create-local-cluster,$(CLUSTER_GSLB1),"deploy/kind/cluster.yaml") + $(call create-local-cluster,$(CLUSTER_GSLB2),"deploy/kind/cluster2.yaml") -.PHONY: destroy-local-cluster -destroy-local-cluster: - kind delete cluster --name test-gslb1 + $(call deploy-local-cluster,$(CLUSTER_GSLB1),$(CLUSTER_GSLB2),worker,absaoss/k8gb,) -.PHONY: destroy-two-local-clusters -destroy-two-local-clusters: - kind delete cluster --name test-gslb1 - kind delete cluster --name test-gslb2 + $(call deploy-local-cluster,$(CLUSTER_GSLB2),$(CLUSTER_GSLB1),worker,absaoss/k8gb,$(HELM_ARGS)) -.PHONY: create-k8gb-ns -create-k8gb-ns: - kubectl apply -f deploy/namespace.yaml - -.PHONY: create-test-ns -create-test-ns: - kubectl apply -f deploy/crds/test-namespace.yaml +# triggered by terraform GitHub Action. Clusters already exists. GO is not installed yet +.PHONY: deploy-full-terratest-setup +deploy-full-terratest-setup: + @echo "\n$(YELLOW)create docker registry$(NC) see: https://kind.sigs.k8s.io/docs/user/local-registry/" + docker run -d --restart=always -p "$(GITACTION_TERRATEST_DOCKER_REPO_PORT):5000" --name "$(GITACTION_TERRATEST_DOCKER_REPO_NAME)" "$(GITACTION_TERRATEST_DOCKER_REPO_IMAGE)" -.PHONY: deploy-local-ingress -deploy-local-ingress: create-k8gb-ns - helm repo add --force-update stable https://kubernetes-charts.storage.googleapis.com - helm repo update - helm -n k8gb upgrade -i nginx-ingress stable/nginx-ingress --version 1.41.1 -f deploy/ingress/nginx-ingress-values.yaml + @echo "\n$(YELLOW)build k8gb docker and push to registry $(NC)" + docker build . -t k8gb:$(COMMIT_HASH) + docker tag k8gb:$(COMMIT_HASH) localhost:$(GITACTION_TERRATEST_DOCKER_REPO_PORT)/k8gb:v$(COMMIT_HASH) + docker push localhost:$(GITACTION_TERRATEST_DOCKER_REPO_PORT)/k8gb:v$(COMMIT_HASH) -.PHONY: wait-for-nginx-ingress-ready -wait-for-nginx-ingress-ready: - kubectl -n k8gb wait --for=condition=Ready pod -l app=nginx-ingress --timeout=600s + @echo "\n$(YELLOW)Change version in Chart.yaml $(CYAN) $(VERSION) to $(COMMIT_HASH)$(NC)" + sed -i "s/$(VERSION)/$(COMMIT_HASH)/g" chart/k8gb/Chart.yaml -.PHONY: deploy-gslb-operator -deploy-gslb-operator: create-k8gb-ns - cd chart/k8gb && helm dependency update - helm -n k8gb upgrade -i k8gb chart/k8gb -f $(VALUES_YAML) $(HELM_ARGS) + $(call deploy-local-cluster,$(CLUSTER_GSLB1),$(CLUSTER_GSLB2),control-plane,localhost:$(GITACTION_TERRATEST_DOCKER_REPO_PORT)/k8gb,) + $(call deploy-local-cluster,$(CLUSTER_GSLB2),$(CLUSTER_GSLB1),control-plane,localhost:$(GITACTION_TERRATEST_DOCKER_REPO_PORT)/k8gb,$(HELM_ARGS)) -.PHONY: wait-for-gslb-ready -wait-for-gslb-ready: - kubectl -n k8gb wait --for=condition=Ready pod -l app=etcd --timeout=600s + @echo "\n$(YELLOW)Local cluster $(CYAN)$(CLUSTER_GSLB2) $(NC)" + kubectl get pods -A + @echo "\n$(YELLOW)Local cluster $(CYAN)$(CLUSTER_GSLB1) $(NC)" + kubectl config use-context kind-$(CLUSTER_GSLB1) + kubectl get pods -A # workaround until https://github.com/crossplaneio/crossplane/issues/1170 solved .PHONY: deploy-gslb-operator-14 -deploy-gslb-operator-14: create-k8gb-ns +deploy-gslb-operator-14: + kubectl apply -f deploy/namespace.yaml cd chart/k8gb && helm dependency update helm -n k8gb template k8gb chart/k8gb -f $(VALUES_YAML) | kubectl -n k8gb --validate=false apply -f - -.PHONY: deploy-gslb-cr -deploy-gslb-cr: create-test-ns - $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml) - $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml) +# destroy local test environment +.PHONY: destroy-full-local-setup +destroy-full-local-setup: + $(call destroy-local-cluster,$(CLUSTER_GSLB1)) + $(call destroy-local-cluster,$(CLUSTER_GSLB2)) -.PHONY: deploy-test-apps -deploy-test-apps: create-test-ns - kubectl apply -f deploy/test-apps - helm repo add podinfo https://stefanprodan.github.io/podinfo - helm upgrade --install frontend --namespace test-gslb -f deploy/test-apps/podinfo/podinfo-values.yaml --set ui.message="`$(call get-cluster-geo-tag)`" --set image.repository="$(PODINFO_IMAGE_REPO)" podinfo/podinfo +.PHONY: dns-tools +dns-tools: + @kubectl -n k8gb get svc k8gb-coredns + @kubectl -n k8gb run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools -.PHONY: clean-test-apps -clean-test-apps: - kubectl delete -f deploy/test-apps - helm -n test-gslb uninstall backend - helm -n test-gslb uninstall frontend +.PHONY: dns-smoke-test +dns-smoke-test: + kubectl -n k8gb run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools --command -- /usr/bin/dig @k8gb-coredns app3.cloud.example.com -.PHONY: debug-test-etcd -debug-test-etcd: - kubectl run --rm -i --tty --env="ETCDCTL_API=3" --env="ETCDCTL_ENDPOINTS=http://etcd-cluster-client:2379" --namespace k8gb etcd-test --image "$(ETCD_DEBUG_IMAGE)" --restart=Never -- /bin/sh +# build the docker image +.PHONY: docker-build +docker-build: test + docker build . -t $(IMG) -.PHONY: infoblox-secret -infoblox-secret: - kubectl -n k8gb create secret generic external-dns \ - --from-literal=EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME=$${WAPI_USERNAME} \ - --from-literal=EXTERNAL_DNS_INFOBLOX_WAPI_PASSWORD=$${WAPI_PASSWORD} +# push the docker image +.PHONY: docker-push +docker-push: + docker push $(IMG) + +# build and push the docker image exclusively for testing using commit hash +.PHONY: docker-test-build-push +docker-test-build-push: test + $(call docker-test-build-push) .PHONY: init-failover init-failover: @@ -271,6 +165,46 @@ init-failover: init-round-robin: $(call init-test-strategy, "deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml") +# creates infoblox secret in current cluster +.PHONY: infoblox-secret +infoblox-secret: + kubectl -n k8gb create secret generic external-dns \ + --from-literal=EXTERNAL_DNS_INFOBLOX_WAPI_USERNAME=$${WAPI_USERNAME} \ + --from-literal=EXTERNAL_DNS_INFOBLOX_WAPI_PASSWORD=$${WAPI_PASSWORD} + +# install CRDs into a cluster +.PHONY: install +install: + $(call manifest) + $(KUSTOMIZE_PATH) build config/crd | kubectl apply -f - + +# run all linters from .golangci.yaml; see: https://golangci-lint.run/usage/install/#local-installation +.PHONY: lint +lint: + golangci-lint run + +# retrieves all targets +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' + +# build manager binary +.PHONY: manager +manager: lint + $(call generate) + go build -o bin/manager main.go + +# remove clusters and redeploy +.PHONY: reset +reset: destroy-full-local-setup deploy-full-local-setup + +# run against the configured Kubernetes cluster in ~/.kube/config +.PHONY: run +run: lint + $(call generate) + $(call manifest) + go run ./main.go + .PHONY: stop-test-app stop-test-app: $(call testapp-set-replicas,0) @@ -279,6 +213,13 @@ stop-test-app: start-test-app: $(call testapp-set-replicas,2) +# run tests +.PHONY: test +test: lint + $(call generate) + $(call manifest) + go test ./... -coverprofile cover.out + .PHONY: test-round-robin test-round-robin: @$(call hit-testapp-host, "app3.cloud.example.com") @@ -287,18 +228,110 @@ test-round-robin: test-failover: @$(call hit-testapp-host, "failover.cloud.example.com") -.PHONY: demo-roundrobin -demo-roundrobin: - @$(call demo-host, "app3.cloud.example.com") +# executes terra-tests +.PHONY: terratest +terratest: + cd terratest/test/ && go mod download && go test -v -.PHONY: demo-failover -demo-failover: - @$(call demo-host, "failover.cloud.example.com") +# uninstall CRDs from a cluster +.PHONY: uninstall +uninstall: + $(call manifest) + $(call install-kustomize-if-not-exists) + $(KUSTOMIZE_PATH) build config/crd | kubectl delete -f - .PHONY: version version: @echo $(VERSION) +############################### +# FUNCTIONS +############################### + +define create-local-cluster + @echo "\n$(YELLOW)Deploy local cluster $(CYAN)$1 $(NC)" + kind create cluster --name=$1 --config=$2 +endef + +define deploy-local-cluster + @echo "\n$(YELLOW)Local cluster $(CYAN)$1 $(NC)" + kubectl config use-context kind-$1 + + @echo "\n$(YELLOW)Create namespace $(NC)" + kubectl apply -f deploy/namespace.yaml + + @echo "\n$(YELLOW)Deploy GSLB operator from $4 $(NC)" + cd chart/k8gb && helm dependency update + helm -n k8gb upgrade -i k8gb chart/k8gb -f $(VALUES_YAML) \ + --set k8gb.hostAlias.enabled=true \ + --set k8gb.hostAlias.ip="`$(call get-host-alias-ip,$1,$2,$3)`" \ + --set k8gb.imageRepo=$4 $5 + + @echo "\n$(YELLOW)Deploy Ingress $(NC)" + helm repo add --force-update stable https://charts.helm.sh/stable + helm repo update + helm -n k8gb upgrade -i nginx-ingress stable/nginx-ingress \ + --version 1.41.1 -f deploy/ingress/nginx-ingress-values.yaml + + @echo "\n$(YELLOW)Deploy GSLB cr $(NC)" + kubectl apply -f deploy/crds/test-namespace.yaml + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml) + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml) + + @echo "\n$(YELLOW)Deploy test apps $(NC)" + $(call deploy-test-apps) + + @echo "\n$(YELLOW)Wait for GSLB, Ingress are ready $(NC)" + $(call wait) + + @echo "\n$(CYAN)$1 $(YELLOW)deployed! $(NC)" +endef + +define destroy-local-cluster + kind delete cluster --name $1 +endef + +define apply-cr + sed -i 's/cloud\.example\.com/$(GSLB_DOMAIN)/g' "$1" + kubectl apply -f "$1" + git checkout -- "$1" +endef + +define deploy-test-apps + kubectl apply -f deploy/test-apps + helm repo add podinfo https://stefanprodan.github.io/podinfo + helm upgrade --install frontend --namespace test-gslb -f deploy/test-apps/podinfo/podinfo-values.yaml \ + --set ui.message="`$(call get-cluster-geo-tag)`" \ + --set image.repository="$(PODINFO_IMAGE_REPO)" \ + podinfo/podinfo +endef + +define get-cluster-geo-tag + kubectl -n k8gb describe deploy k8gb | awk '/CLUSTER_GEO_TAG/ { printf $$2 }' +endef + +# get-host-alias-ip switch to second context ($2), search for IP and switch back to first context ($1) +# function returns one IP address +define get-host-alias-ip + kubectl config use-context kind-$2 > /dev/null && \ + kubectl get nodes $2-$3 -o custom-columns='IP:status.addresses[0].address' --no-headers && \ + kubectl config use-context kind-$1 > /dev/null +endef + +define hit-testapp-host + kubectl run -it --rm busybox --restart=Never --image=busybox -- sh -c \ + "echo 'nameserver `$(K8GB_COREDNS_IP)`' > /etc/resolv.conf && \ + wget -qO - $1" +endef + +define init-test-strategy + kubectl config use-context kind-test-gslb2 + kubectl apply -f $1 + kubectl config use-context kind-test-gslb1 + kubectl apply -f $1 + $(call testapp-set-replicas,2) +endef + define testapp-set-replicas kubectl scale deployment frontend-podinfo -n test-gslb --replicas=$1 endef @@ -314,27 +347,52 @@ define demo-host "`$(K8GB_COREDNS_IP)`" $1 endef -define init-test-strategy - kubectl config use-context kind-test-gslb2 - kubectl apply -f $1 - kubectl config use-context kind-test-gslb1 - kubectl apply -f $1 - $(call testapp-set-replicas,2) +# waits for NGINX, GSLB are ready +define wait + kubectl -n k8gb wait --for=condition=Ready pod -l app=nginx-ingress --timeout=600s + kubectl -n k8gb wait --for=condition=Ready pod -l app=etcd --timeout=600s endef -define get-cluster-geo-tag - kubectl -n k8gb describe deploy k8gb | awk '/CLUSTER_GEO_TAG/ { printf $$2 }' +define generate + $(call controller-gen,object:headerFile="hack/boilerplate.go.txt" paths="./...") endef -define apply-cr - sed -i 's/cloud\.example\.com/$(GSLB_DOMAIN)/g' "$1" - kubectl apply -f "$1" - git checkout -- "$1" +define manifest + $(call controller-gen,crd:trivialVersions=true rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases) +endef + +# function retrieves controller-gen path or installs controller-gen@v3.0.0 and retrieve new path in case it is not installed +define controller-gen + @$(if $(filter $(CONTROLLER_GEN_PATH),$(NO_VALUE)),$(call install-controller-gen),) + $(CONTROLLER_GEN_PATH) $1 +endef + +define install-controller-gen + GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 + $(eval CONTROLLER_GEN_PATH = $(GOBIN)/controller-gen) +endef + +# installs kustomize and sets KUSTOMIZE_PATH if is not specified +define install-kustomize-if-not-exists + @$(if $(filter $(KUSTOMIZE_PATH),$(NO_VALUE)),$(call install-kustomize),) +endef + +define install-kustomize + GO111MODULE=on go get sigs.k8s.io/kustomize/kustomize/v3@v3.8.6 + $(eval KUSTOMIZE_PATH = $(GOBIN)/kustomize) endef define docker-test-build-push - docker build . -t k8gb:$(COMMIT_HASH) - docker tag k8gb:$(COMMIT_HASH) $(REPO):v$(COMMIT_HASH) - docker push $(REPO):v$(COMMIT_HASH) - sed -i "s/$(VERSION)/$(COMMIT_HASH)/g" chart/k8gb/Chart.yaml + docker build . -t k8gb:$(COMMIT_HASH) + docker tag k8gb:$(COMMIT_HASH) $(REPO):v$(COMMIT_HASH) + docker push $(REPO):v$(COMMIT_HASH) + sed -i "s/$(VERSION)/$(COMMIT_HASH)/g" chart/k8gb/Chart.yaml +endef + +define debug + $(call manifest) + kubectl apply -f deploy/crds/test-namespace.yaml + kubectl apply -f ./deploy/crds/k8gb.absa.oss_gslbs_crd.yaml + kubectl apply -f ./deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml + dlv $1 endef diff --git a/deploy/full.sh b/deploy/full.sh deleted file mode 100755 index 6e78e04ae2..0000000000 --- a/deploy/full.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -o errexit - -# Workaround of Make being to smart on up-to-date PHONY targets -# If we execute all of them normal way, then targets from `deploy-second-k8gb` -# will never be executed as they contain the same underlying target as `deploy-first-k8gb` -# but with different variables - -NODE_ROLE=${NODE_ROLE:-worker} -ADDITIONAL_TARGETS=${ADDITIONAL_TARGETS:-} -TEST_CURRENT_COMMIT=${TEST_CURRENT_COMMIT:-} - -if [ "$TEST_CURRENT_COMMIT" == "yes" ] -then - ./deploy/registry.sh - commit_hash=$(git rev-parse --short HEAD) - docker build . -t k8gb:${commit_hash} - docker tag k8gb:${commit_hash} localhost:5000/k8gb:v${commit_hash} - docker push localhost:5000/k8gb:v${commit_hash} - export K8GB_IMAGE_REPO=localhost:5000/k8gb - sed -i "s/${VERSION}/${commit_hash}/g" chart/k8gb/Chart.yaml -fi - -make use-second-context -export HOST_ALIAS_IP1=$(kubectl get nodes test-gslb2-${NODE_ROLE} -o custom-columns='IP:status.addresses[0].address' --no-headers) -make use-first-context deploy-first-k8gb deploy-gslb-cr ${ADDITIONAL_TARGETS} -export HOST_ALIAS_IP2=$(kubectl get nodes test-gslb1-${NODE_ROLE} -o custom-columns='IP:status.addresses[0].address' --no-headers) -make use-second-context deploy-second-k8gb deploy-gslb-cr ${ADDITIONAL_TARGETS} - -make wait-for-nginx-ingress-ready -make wait-for-gslb-ready - -make use-first-context -make wait-for-nginx-ingress-ready -make wait-for-gslb-ready diff --git a/deploy/registry.sh b/deploy/registry.sh deleted file mode 100755 index 29ff704d88..0000000000 --- a/deploy/registry.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -set -o errexit - -DOCKER_REGISTRY_IMAGE=${DOCKER_REGISTRY_IMAGE:-'registry:2'} - -# create registry container unless it already exists -reg_name='kind-registry' -reg_port='5000' -running="$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" -if [ "${running}" != 'true' ]; then - docker run \ - -d --restart=always -p "${reg_port}:5000" --name "${reg_name}" \ - "${DOCKER_REGISTRY_IMAGE}" -fi diff --git a/docs/local.md b/docs/local.md index 33ae1991ab..5696e44b69 100644 --- a/docs/local.md +++ b/docs/local.md @@ -67,7 +67,7 @@ eed5a40bbfb6ee97, started, etcd-cluster-xsjmwdkdf8, http://etcd-cluster-xsjmwdkd Cluster [test-gslb1](https://github.com/AbsaOSS/k8gb/tree/master/deploy/kind/cluster.yaml) is exposing external DNS on default port `:5053` while [test-gslb2](https://github.com/AbsaOSS/k8gb/tree/master/deploy/kind/cluster2.yaml) on port `:5054`. ```shell script -dig @localhost localtargets.app3.cloud.example.com -p 5053 && dig -p 5054 @localhost localtargets.app3.cloud.example.com +dig @localhost localtargets-app3.cloud.example.com -p 5053 && dig -p 5054 @localhost localtargets-app3.cloud.example.com ``` As expected result you should see **six A records** divided between nodes of both clusters. ```shell script