operator: operator: Change the configurator to respect subdomain

If subdomain is not empty, then external listener is configured. The annotations in the headless service is used to register all pods in DNS provider. The externalDNS does not support AWS external IP, but there is open PR for addressing that. During the testing the code from kubernetes-sigs/external-dns#1391 was used. REF https://hub.docker.com/layers/wanderadock/external-dns/v0.7.6-pub-external-ips/images/sha256-dc59c17c70504bb86843bf9b71f649eb3220aa075751a7a11e46977cbdaedbcb?context=explore
joejulian · Mar 17, 2021 · d783ff2 · d783ff2
1 parent 98d5d2e
commit d783ff2
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 12 deletions.
diff --git a/cmd/configurator/main.go b/cmd/configurator/main.go
@@ -32,15 +32,16 @@ import (
 )
 
 const (
-	hostNameEnvVar             = "HOSTNAME"
-	svcFQDNEnvVar              = "SERVICE_FQDN"
-	configSourceDirEnvVar      = "CONFIG_SOURCE_DIR"
-	configDestinationEnvVar    = "CONFIG_DESTINATION"
-	redpandaRPCPortEnvVar      = "REDPANDA_RPC_PORT"
-	kafkaAPIEnvVar             = "KAFKA_API_PORT"
-	nodeNameEnvVar             = "NODE_NAME"
-	externalConnectivityEnvVar = "EXTERNAL_CONNECTIVITY"
-	hostPortEnvVar             = "HOST_PORT"
+	hostNameEnvVar                      = "HOSTNAME"
+	svcFQDNEnvVar                       = "SERVICE_FQDN"
+	configSourceDirEnvVar               = "CONFIG_SOURCE_DIR"
+	configDestinationEnvVar             = "CONFIG_DESTINATION"
+	redpandaRPCPortEnvVar               = "REDPANDA_RPC_PORT"
+	kafkaAPIEnvVar                      = "KAFKA_API_PORT"
+	nodeNameEnvVar                      = "NODE_NAME"
+	externalConnectivityEnvVar          = "EXTERNAL_CONNECTIVITY"
+	externalConnectivitySubDomainEnvVar = "EXTERNAL_CONNECTIVITY_SUBDOMAIN"
+	hostPortEnvVar                      = "HOST_PORT"
 )
 
 type configuratorConfig struct {
@@ -49,6 +50,7 @@ type configuratorConfig struct {
 	configSourceDir      string
 	configDestination    string
 	nodeName             string
+	subdomain            string
 	externalConnectivity bool
 	kafkaAPIPort         int
 	redpandaRPCPort      int
@@ -63,6 +65,7 @@ func (c *configuratorConfig) String() string {
 		"configDestination: %s\n"+
 		"nodeName: %s\n"+
 		"externalConnectivity: %t\n"+
+		"externalConnectivitySubdomain: %s\n"+
 		"kafkaAPIPort: %d\n"+
 		"redpandaRPCPort: %d\n"+
 		"hostPort: %d\n",
@@ -72,6 +75,7 @@ func (c *configuratorConfig) String() string {
 		c.configDestination,
 		c.nodeName,
 		c.externalConnectivity,
+		c.subdomain,
 		c.kafkaAPIPort,
 		c.redpandaRPCPort,
 		c.hostPort)
@@ -195,6 +199,17 @@ func registerAdvertisedKafkaAPI(
 		return nil
 	}
 
+	if len(c.subdomain) > 0 {
+		cfg.Redpanda.AdvertisedKafkaApi = append(cfg.Redpanda.AdvertisedKafkaApi, config.NamedSocketAddress{
+			SocketAddress: config.SocketAddress{
+				Address: fmt.Sprintf("%s.%s", c.hostName, c.subdomain),
+				Port:    c.hostPort,
+			},
+			Name: "External",
+		})
+		return nil
+	}
+
 	k8sconfig, err := rest.InClusterConfig()
 	if err != nil {
 		return fmt.Errorf("unable to create in cluster config: %w", err)
@@ -265,6 +280,10 @@ func checkEnvVars() (configuratorConfig, error) {
 			value: &c.nodeName,
 			name:  nodeNameEnvVar,
 		},
+		{
+			value: &c.subdomain,
+			name:  externalConnectivitySubDomainEnvVar,
+		},
 		{
 			value: &extCon,
 			name:  externalConnectivityEnvVar,

diff --git a/controllers/redpanda/cluster_controller.go b/controllers/redpanda/cluster_controller.go
@@ -221,6 +221,16 @@ func (r *ClusterReconciler) createExternalNodesList(
 			return []string{}, fmt.Errorf("failed to retrieve node %s: %w", pods[i].Spec.NodeName, err)
 		}
 
+		if len(pandaCluster.Spec.ExternalConnectivity.Subdomain) > 0 {
+			observedNodesExternal = append(observedNodesExternal,
+				fmt.Sprintf("%s.%s:%d",
+					pods[i].Spec.Hostname,
+					pandaCluster.Spec.ExternalConnectivity.Subdomain,
+					getNodePort(&nodePortSvc),
+				))
+			continue
+		}
+
 		observedNodesExternal = append(observedNodesExternal,
 			fmt.Sprintf("%s:%d",
 				getExternalIP(&node),

diff --git a/pkg/resources/headless_service.go b/pkg/resources/headless_service.go
@@ -27,6 +27,11 @@ import (
 
 var _ Resource = &HeadlessServiceResource{}
 
+const (
+	externalDNSHostname  = "external-dns.alpha.kubernetes.io/hostname"
+	externalDNSUseHostIP = "external-dns.alpha.kubernetes.io/use-external-host-ip"
+)
+
 // HeadlessServiceResource is part of the reconciliation of redpanda.vectorized.io CRD
 // focusing on the internal connectivity management of redpanda cluster
 type HeadlessServiceResource struct {
@@ -70,9 +75,10 @@ func (r *HeadlessServiceResource) obj() (k8sclient.Object, error) {
 	objLabels := labels.ForCluster(r.pandaCluster)
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: r.Key().Namespace,
-			Name:      r.Key().Name,
-			Labels:    objLabels,
+			Namespace:   r.Key().Namespace,
+			Name:        r.Key().Name,
+			Labels:      objLabels,
+			Annotations: r.getAnnotation(),
 		},
 		Spec: corev1.ServiceSpec{
 			Type:      corev1.ServiceTypeClusterIP,
@@ -118,3 +124,16 @@ func (r *HeadlessServiceResource) HeadlessServiceFQDN() string {
 		'.',
 		r.Key().Namespace)
 }
+
+func (r *HeadlessServiceResource) getAnnotation() map[string]string {
+	if !r.pandaCluster.Spec.ExternalConnectivity.Enabled && r.pandaCluster.Spec.ExternalConnectivity.Subdomain == "" {
+		return nil
+	}
+
+	return map[string]string{
+		externalDNSHostname: r.pandaCluster.Spec.ExternalConnectivity.Subdomain,
+		// This annotation comes from the not merged feature
+		// https://github.com/kubernetes-sigs/external-dns/pull/1391
+		externalDNSUseHostIP: "true",
+	}
+}
diff --git a/pkg/resources/statefulset.go b/pkg/resources/statefulset.go
@@ -291,6 +291,10 @@ func (r *StatefulSetResource) obj() (k8sclient.Object, error) {
 									Name:  "EXTERNAL_CONNECTIVITY",
 									Value: strconv.FormatBool(r.pandaCluster.Spec.ExternalConnectivity.Enabled),
 								},
+								{
+									Name:  "EXTERNAL_CONNECTIVITY_SUBDOMAIN",
+									Value: r.pandaCluster.Spec.ExternalConnectivity.Subdomain,
+								},
 								{
 									Name:  "HOST_PORT",
 									Value: r.getNodePort(),