switch host alias to real edgeDNS

This runs rfc2136 provider and remove all local hacks Signed-off-by: Dinar Valeev <dinar.valeev@absa.africa>
k8gb-io · Nov 23, 2021 · 0d3002b · 0d3002b
1 parent f2729ad
commit 0d3002b
Show file tree

Hide file tree

Showing 14 changed files with 242 additions and 47 deletions.
diff --git a/.github/workflows/terratest.yaml b/.github/workflows/terratest.yaml
@@ -34,6 +34,12 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Create edgeDNS k3s Cluster
+        uses: AbsaOSS/k3d-action@v1.5.0
+        with:
+          cluster-name: "edgedns"
+          args: -c k3d/edge-dns.yaml
+
       - name: Create 1st k3s Cluster
         uses: AbsaOSS/k3d-action@v1.5.0
         with:

diff --git a/.github/workflows/upgrade-testing.yaml b/.github/workflows/upgrade-testing.yaml
@@ -34,6 +34,12 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Create edgeDNS k3s Cluster
+        uses: AbsaOSS/k3d-action@v1.5.0
+        with:
+          cluster-name: "edgedns"
+          args: -c k3d/edge-dns.yaml
+
       - name: Create 1st k3s Cluster
         uses: AbsaOSS/k3d-action@v1.5.0
         with:

diff --git a/Makefile b/Makefile
@@ -117,6 +117,7 @@ demo: ## Execute end-to-end demo
 .PHONY: deploy-full-local-setup
 deploy-full-local-setup: ensure-cluster-size ## Deploy full local multicluster setup (k3d >= 4.2.0)
 	@echo "\n$(YELLOW)Creating $(CLUSTERS_NUMBER) k8s clusters$(NC)"
+	$(MAKE) create-local-cluster CLUSTER_NAME=edgedns
 	@for c in $(CLUSTER_IDS); do \
 		$(MAKE) create-local-cluster CLUSTER_NAME=$(CLUSTER_NAME)$$c ;\
 	done
@@ -125,6 +126,7 @@ deploy-full-local-setup: ensure-cluster-size ## Deploy full local multicluster s
 
 .PHONY: deploy-stable-version
 deploy-stable-version:
+	$(call deploy-edgedns)
 	@for c in $(CLUSTER_IDS); do \
 		$(MAKE) deploy-local-cluster CLUSTER_ID=$$c ;\
 	done
@@ -207,6 +209,7 @@ deploy-k8gb-with-helm:
 		--set k8gb.reconcileRequeueSeconds=10 \
 		--set k8gb.dnsZoneNegTTL=10 \
 		--set k8gb.imageTag=${VERSION:"stable"=""} \
+		--set externaldns.image=absaoss/external-dns:rfc-ns1 \
 		--set k8gb.log.format=$(LOG_FORMAT) \
 		--set k8gb.log.level=$(LOG_LEVEL) \
 		--wait --timeout=2m0s
@@ -222,6 +225,7 @@ deploy-gslb-operator: ## Deploy k8gb operator
 # destroy local test environment
 .PHONY: destroy-full-local-setup
 destroy-full-local-setup: ## Destroy full local multicluster setup
+	k3d cluster delete edgedns
 	@for c in $(CLUSTER_IDS); do \
 		k3d cluster delete $(CLUSTER_NAME)$$c ;\
 	done
@@ -447,6 +451,55 @@ help: ## Show this help
 #		FUNCTIONS
 ###############################
 
+define create-local-cluster
+	@echo "\n$(YELLOW)Deploy local cluster $(CYAN)$1 $(NC)"
+	k3d cluster create -c k3d/$1.yaml
+endef
+
+define deploy-k8gb-with-helm
+	# create rfc2136 secret
+	kubectl -n k8gb create secret generic rfc2136 --from-literal=secret=96Ah/a2g0/nLeFGK+d/0tzQcccf9hCEIy34PoXX2Qg8= || true
+	helm repo add --force-update k8gb https://www.k8gb.io
+	cd chart/k8gb && helm dependency update
+	helm -n k8gb upgrade -i k8gb $5 -f $(VALUES_YAML) \
+		--set k8gb.reconcileRequeueSeconds=10 \
+		--set k8gb.dnsZoneNegTTL=10 \
+		--set k8gb.imageTag=$3 $4 \
+		--set rfc2136.enabled=true \
+		--set k8gb.edgeDNSServers[0]=host.k3d.internal:1053 \
+		--set k8gb.log.format=$(LOG_FORMAT) \
+		--set k8gb.log.level=$(LOG_LEVEL) \
+		--wait --timeout=2m0s
+endef
+
+define deploy-edgedns
+	@echo "\n$(YELLOW)Deploying EdgeDNS $(NC)"
+	kubectl config use-context k3d-edgedns
+	kubectl apply -f deploy/edge/
+endef
+
+define deploy-local-cluster
+	@echo "\n$(YELLOW)Local cluster $(CYAN)$1 $(NC)"
+	kubectl config use-context k3d-$1
+
+	@echo "\n$(YELLOW)Create namespace $(NC)"
+	kubectl apply -f deploy/namespace.yaml
+
+	@echo "\n$(YELLOW)Deploy GSLB operator from ${3} $(NC)"
+	$(call deploy-k8gb-with-helm,$1,$2,${3:"stable"=""},$4,$5)
+
+	@echo "\n$(YELLOW)Deploy Ingress $(NC)"
+	helm repo add --force-update nginx-stable https://kubernetes.github.io/ingress-nginx
+	helm repo update
+	helm -n k8gb upgrade -i nginx-ingress nginx-stable/ingress-nginx \
+		--version 3.24.0 -f deploy/ingress/nginx-ingress-values.yaml
+
+	@echo "\n$(YELLOW)Wait until Ingress controller is ready $(NC)"
+	$(call wait-for-ingress)
+
+	@echo "\n$(CYAN)$1 $(YELLOW)deployed! $(NC)"
+endef
+
 define apply-cr
 	sed -i 's/cloud\.example\.com/$(GSLB_DOMAIN)/g' "$1"
 	kubectl apply -f "$1"
@@ -469,14 +522,6 @@ k8gb.clusterGeoTag='$(call nth-geo-tag,$1)' --set k8gb.extGslbClustersGeoTags='$
 $(foreach cl,$(shell seq $(CLUSTERS_NUMBER)), --set "k8gb.hostAliases[$$(( $(cl) - 1 ))].ip=$(shell $(call get-host-alias-ip,k3d-$(CLUSTER_NAME)$1,k3d-$(CLUSTER_NAME)$(cl)))" --set "k8gb.hostAliases[$$(( $(cl) - 1 ))].hostnames={gslb-ns-$(call nth-geo-tag,$(cl))-cloud.example.com}")
 endef
 
-# get-host-alias-ip switch to second context ($2), search for IP and switch back to first context ($1)
-# function returns one IP address
-define get-host-alias-ip
-kubectl config use-context $2 > /dev/null && \
-kubectl get nodes $2-agent-0 -o custom-columns='IP:status.addresses[0].address' --no-headers && \
-kubectl config use-context $1 > /dev/null
-endef
-
 define hit-testapp-host
 	kubectl run -it --rm busybox --restart=Never --image=busybox --command \
 	--overrides='{"spec": {"dnsConfig": {"nameservers": ["$(K8GB_COREDNS_IP)"]}, \"dnsPolicy\": \"None\"}}' \

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/chart/k8gb/templates/_helpers.tpl b/chart/k8gb/templates/_helpers.tpl
@@ -69,14 +69,16 @@ Create the name of the service account to use
 {{- if .Values.route53.enabled }}
 {{- print "aws" -}}
 {{- end -}}
+{{- if .Values.rfc2136.enabled }}
+{{- print "rfc2136" -}}
+{{- end -}}
 {{- end -}}
 
 {{- define "k8gb.extdnsOwnerID" -}}
-{{- if .Values.ns1.enabled -}}
-k8gb-{{ .Values.k8gb.dnsZone }}-{{ .Values.k8gb.clusterGeoTag }}
-{{- end -}}
 {{- if .Values.route53.enabled -}}
 k8gb-{{ .Values.route53.hostedZoneID }}-{{ .Values.k8gb.clusterGeoTag }}
+{{- else -}}
+k8gb-{{ .Values.k8gb.dnsZone }}-{{ .Values.k8gb.clusterGeoTag }}
 {{- end -}}
 {{- end -}}
 
@@ -87,3 +89,35 @@ k8gb-{{ .Values.route53.hostedZoneID }}-{{ .Values.k8gb.clusterGeoTag }}
 {{ join "," .Values.k8gb.edgeDNSServers }}
 {{- end -}}
 {{- end -}}
+
+{{- define "k8gb.extdnsProviderOpts" -}}
+{{- if .Values.ns1.enabled -}}
+{{- if .Values.ns1.endpoint -}}
+        - --ns1-endpoint={{ .Values.ns1.endpoint }}
+{{- end -}}
+{{- if .Values.ns1.ignoreSSL -}}
+        - --ns1-ignoressl
+{{- end -}}
+        env:
+        - name: NS1_APIKEY
+          valueFrom:
+            secretKeyRef:
+              name: ns1
+              key: apiKey
+{{- end }}
+{{- if .Values.rfc2136.enabled -}}
+        - --rfc2136-zone={{ .Values.k8gb.edgeDNSZone }}
+        - --rfc2136-tsig-axfr
+{{- range $k, $v := .Values.rfc2136.rfc2136Opts }}
+{{- range $kk, $vv := $v }}
+        - --rfc2136-{{ $kk }}={{ $vv }}
+{{- end -}}
+{{- end }}
+        env:
+        - name: EXTERNAL_DNS_RFC2136_TSIG_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: rfc2136
+              key: secret
+{{- end -}}
+{{- end -}}
diff --git a/chart/k8gb/templates/external-dns/external-dns.yaml b/chart/k8gb/templates/external-dns/external-dns.yaml
@@ -1,4 +1,4 @@
-{{- if or .Values.ns1.enabled .Values.route53.enabled }}
+{{- if or .Values.ns1.enabled .Values.route53.enabled .Values.rfc2136.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -31,20 +31,7 @@ spec:
         - --annotation-filter=k8gb.absa.oss/dnstype=extdns # filter out only relevant DNSEntrypoints
         - --txt-owner-id={{ include "k8gb.extdnsOwnerID" . }}
         - --provider={{ include "k8gb.extdnsProvider" . }}
-{{- if .Values.ns1.enabled -}}
-{{- if .Values.ns1.endpoint -}}
-        - --ns1-endpoint={{ .Values.ns1.endpoint }}
-{{- end -}}
-{{- if .Values.ns1.ignoreSSL -}}
-        - --ns1-ignoressl
-{{- end -}}
-        env:
-        - name: NS1_APIKEY
-          valueFrom:
-            secretKeyRef:
-              name: ns1
-              key: apiKey
-{{- end }}
+        {{ include "k8gb.extdnsProviderOpts" . }}
         resources:
           requests:
             memory: "32Mi"

diff --git a/chart/k8gb/templates/external-dns/rbac.yaml b/chart/k8gb/templates/external-dns/rbac.yaml
@@ -1,5 +1,5 @@
 ---
-{{- if or .Values.ns1.enabled .Values.route53.enabled }}
+{{- if or .Values.ns1.enabled .Values.route53.enabled .Values.rfc2136.enabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -11,9 +11,7 @@ rules:
 - apiGroups: ["externaldns.k8s.io"]
   resources: ["dnsendpoints/status"]
   verbs: ["*"]
-{{- end }}
 ---
-{{- if or .Values.ns1.enabled .Values.route53.enabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
@@ -26,9 +24,7 @@ subjects:
 - kind: ServiceAccount
   name: k8gb-external-dns
   namespace: {{ .Release.Namespace }}
-{{- end }}
 ---
-{{- if or .Values.ns1.enabled .Values.route53.enabled }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

diff --git a/chart/k8gb/templates/operator.yaml b/chart/k8gb/templates/operator.yaml
@@ -15,10 +15,6 @@ spec:
       labels:
         name: k8gb
     spec:
-      {{- if .Values.k8gb.hostAliases }}
-      hostAliases:
-        {{- toYaml .Values.k8gb.hostAliases | nindent 8 }}
-      {{- end }}
       serviceAccountName: k8gb
       containers:
         - name: k8gb
@@ -99,7 +95,7 @@ spec:
                   name: infoblox
                   key: INFOBLOX_WAPI_PASSWORD
             {{- end }}
-            {{- if or .Values.route53.enabled .Values.ns1.enabled }}
+            {{- if or .Values.route53.enabled .Values.ns1.enabled .Values.rfc2136.enabled }}
             - name: EXTDNS_ENABLED
               value: "true"
             {{- end }}

diff --git a/chart/k8gb/values.yaml b/chart/k8gb/values.yaml
@@ -23,17 +23,6 @@ k8gb:
   clusterGeoTag: "eu"
   # -- comma-separated list of external gslb geo tags to pair with
   extGslbClustersGeoTags: "us"
-  # -- use [/etc/hosts](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/)
-  # inside operator pod. Useful for advanced testing scenarios and to break dependency on EdgeDNS for cross k8gb collaboration
-  # example: `[{"ip": "172.17.0.1", "hostnames": ["gslb-ns-us-cloud.example.com"]}]`
-  hostAliases: []
-  #   - ip: "172.17.0.1"
-  #     hostnames: 
-  #      - "gslb-ns-us-cloud.example.com"
-  #   - ip: "172.17.0.2"
-  #     hostnames: 
-  #      - "gslb-ns-eu-cloud.example.com"
-
   # -- Reconcile time in seconds
   reconcileRequeueSeconds: 30
   log:
@@ -114,3 +103,11 @@ ns1:
 openshift:
   # -- Install OpenShift specific RBAC
   enabled: false
+
+rfc2136:
+  enabled: false
+  rfc2136Opts:
+    - host: host.k3d.internal
+    - port: 1053
+    - tsig-secret-alg: hmac-sha256
+    - tsig-keyname: externaldns-key
diff --git a/deploy/edge/ddns-secret.yaml b/deploy/edge/ddns-secret.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ddns-key
+type: Opaque
+stringData:
+  ddns.key: |
+    key "externaldns-key" {
+      algorithm hmac-sha256;
+      secret "96Ah/a2g0/nLeFGK+d/0tzQcccf9hCEIy34PoXX2Qg8=";
+    };
diff --git a/deploy/edge/deployment.yaml b/deploy/edge/deployment.yaml
@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: edge
+  labels:
+    app: edge
+spec:
+  selector:
+    matchLabels:
+      app: edge
+  template:
+    metadata:
+      labels:
+        app: edge
+    spec:
+      initContainers:
+      - name: copy-zone
+        image: gcr.io/google_containers/busybox
+        command: [ "cp", "/etc/bind/k8s.zone", "/var/lib/bind/zones/k8s.zone" ]
+        volumeMounts:
+          - mountPath: /etc/bind
+            name: config
+          - mountPath: /var/lib/bind/zones
+            name: zone-volume
+      containers:
+      - name: bind
+        image: internetsystemsconsortium/bind9:9.16
+        ports:
+        - containerPort: 1053
+          protocol: TCP
+        volumeMounts:
+          - mountPath: /etc/bind
+            name: config
+          - mountPath: /var/lib/bind/zones
+            name: zone-volume
+      volumes:
+      - name: zone-volume
+        emptyDir: {}
+      - name: config
+        projected:
+          sources:
+          - secret:
+              name: ddns-key
+          - configMap:
+              name: zone
diff --git a/deploy/edge/service.yaml b/deploy/edge/service.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: bind
+spec:
+  type: NodePort
+  selector:
+    app: edge
+  ports:
+    - protocol: TCP
+      name: tcp
+      port: 53
+      targetPort: 1053
+      nodePort: 30053
+    - protocol: UDP
+      name: udp
+      port: 53
+      targetPort: 1053
+      nodePort: 30053