Skip to content

Commit

Permalink
Added Extension Controller and StatsD containers to the ActiveGate po…
Browse files Browse the repository at this point in the history
…d on reconciliation

Added StatsD UDP port to the ActiveGate service

Hide StatsD and EEC containers under feature flag 'enable-statsd'
Added feature flag 'use-activegate-image-for-statsd' to control whether custom entry points are needed for StatsD/EEC containers
  • Loading branch information
toszr committed Oct 19, 2021
1 parent 076e957 commit d7b9c60
Show file tree
Hide file tree
Showing 16 changed files with 660 additions and 44 deletions.
2 changes: 1 addition & 1 deletion api/v1beta1/activegate_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ var (
RoutingCapability = ActiveGateCapability{
DisplayName: "routing",
ShortName: "routing",
ArgumentName: "MSGrouter",
ArgumentName: "MSGrouter,extension_controller",
}

KubeMonCapability = ActiveGateCapability{
Expand Down
14 changes: 14 additions & 0 deletions api/v1beta1/feature_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ const (
annotationFeatureIgnoreUnknownState = annotationFeaturePrefix + "ignore-unknown-state"
annotationFeatureCSITolerations = annotationFeaturePrefix + "csi-tolerations"
annotationFeatureIgnoredNamespaces = annotationFeaturePrefix + "ignored-namespaces"
annotationFeatureEnableStatsDIngest = annotationFeaturePrefix + "enable-statsd"
annotationFeatureUseActiveGateImageForStatsD = annotationFeaturePrefix + "use-activegate-image-for-statsd"
)

var (
Expand Down Expand Up @@ -118,3 +120,15 @@ func (dk *DynaKube) FeatureIgnoredNamespaces() []string {
}
return *ignoredNamespaces
}

// FeatureEnableStatsDIngest is a feature flag that makes the operator include 2 extra containers (Extension Controller and StatsD data source)
// in the ActiveGate pod, and defines an extra UDP port in the AG service for StatsD packets.
func (dk *DynaKube) FeatureEnableStatsDIngest() bool {
return dk.Annotations[annotationFeatureEnableStatsDIngest] == "true"
}

// FeatureUseActiveGateImageForStatsD is a feature flag that makes the operator use ActiveGate image when initializing Extension Controller and StatsD containers
// (using special predefined entry points).
func (dk *DynaKube) FeatureUseActiveGateImageForStatsD() bool {
return dk.Annotations[annotationFeatureUseActiveGateImageForStatsD] == "true"
}
5 changes: 5 additions & 0 deletions controllers/activegate/internal/consts/consts.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
package consts

const (
ServicePortName = "https"
ServicePort = 443
ServiceTargetPort = "ag-https"

StatsDIngestPortName = "statsd"
StatsDIngestPort = 18125
StatsDIngestTargetPort = "statsd-port"
)
12 changes: 11 additions & 1 deletion controllers/activegate/reconciler/capability/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,24 @@ func setReadinessProbePort() events.StatefulSetEvent {
}
}

func setCommunicationsPort(_ *dynatracev1beta1.DynaKube) events.StatefulSetEvent {
func setCommunicationsPort(dk *dynatracev1beta1.DynaKube) events.StatefulSetEvent {
return func(sts *appsv1.StatefulSet) {
sts.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{
{
Name: consts.ServiceTargetPort,
ContainerPort: containerPort,
},
}
if dk.FeatureEnableStatsDIngest() {
// TODO Refactor (access containers by name instead of index)
sts.Spec.Template.Spec.Containers[2].Ports = []corev1.ContainerPort{
{
Name: consts.StatsDIngestTargetPort,
ContainerPort: consts.StatsDIngestPort,
Protocol: corev1.ProtocolUDP,
},
}
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/Dynatrace/dynatrace-operator/logger"
"github.com/Dynatrace/dynatrace-operator/scheme"
"github.com/go-logr/logr"
testlogr "github.com/go-logr/logr/testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -178,7 +179,9 @@ func TestReconcile(t *testing.T) {

func TestSetReadinessProbePort(t *testing.T) {
r := createDefaultReconciler(t)
stsProps := rsfs.NewStatefulSetProperties(r.Instance, metricsCapability.Properties(), "", "", "", "", "", "", "", nil, nil, nil)
stsProps := rsfs.NewStatefulSetProperties(r.Instance, metricsCapability.Properties(), "", "", "", "", "", "", "",
nil, nil, nil, testlogr.TestLogger{T: t},
)
sts, err := rsfs.CreateStatefulSet(stsProps)

assert.NoError(t, err)
Expand Down
25 changes: 18 additions & 7 deletions controllers/activegate/reconciler/capability/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,23 @@ import (
)

func createService(instance *dynatracev1beta1.DynaKube, feature string) *corev1.Service {
ports := []corev1.ServicePort{
{
Name: consts.ServicePortName,
Protocol: corev1.ProtocolTCP,
Port: consts.ServicePort,
TargetPort: intstr.FromString(consts.ServiceTargetPort),
},
}
if instance.FeatureEnableStatsDIngest() {
ports = append(ports, corev1.ServicePort{
Name: consts.StatsDIngestPortName,
Protocol: corev1.ProtocolUDP,
Port: consts.StatsDIngestPort,
TargetPort: intstr.FromString(consts.StatsDIngestTargetPort),
})
}

return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: BuildServiceName(instance.Name, feature),
Expand All @@ -22,13 +39,7 @@ func createService(instance *dynatracev1beta1.DynaKube, feature string) *corev1.
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: statefulset.BuildLabelsFromInstance(instance, feature),
Ports: []corev1.ServicePort{
{
Protocol: corev1.ProtocolTCP,
Port: consts.ServicePort,
TargetPort: intstr.FromString(consts.ServiceTargetPort),
},
},
Ports: ports,
},
}
}
Expand Down
23 changes: 22 additions & 1 deletion controllers/activegate/reconciler/capability/service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@ const (

func TestCreateService(t *testing.T) {
instance := &dynatracev1beta1.DynaKube{
ObjectMeta: v1.ObjectMeta{Namespace: testNamespace, Name: testName},
ObjectMeta: v1.ObjectMeta{
Namespace: testNamespace, Name: testName,
},
Spec: dynatracev1beta1.DynaKubeSpec{
APIURL: "https://testing.dev.dynatracelabs.com/api",
},
}
service := createService(instance, testFeature)

Expand All @@ -37,10 +42,26 @@ func TestCreateService(t *testing.T) {

ports := serviceSpec.Ports
assert.Contains(t, ports, corev1.ServicePort{
Name: consts.ServicePortName,
Protocol: corev1.ProtocolTCP,
Port: consts.ServicePort,
TargetPort: intstr.FromString(consts.ServiceTargetPort),
})
if instance.FeatureEnableStatsDIngest() {
assert.Contains(t, ports, corev1.ServicePort{
Name: consts.StatsDIngestPortName,
Protocol: corev1.ProtocolUDP,
Port: consts.StatsDIngestPort,
TargetPort: intstr.FromString(consts.StatsDIngestTargetPort),
})
} else {
assert.NotContains(t, ports, corev1.ServicePort{
Name: consts.StatsDIngestPortName,
Protocol: corev1.ProtocolUDP,
Port: consts.StatsDIngestPort,
TargetPort: intstr.FromString(consts.StatsDIngestTargetPort),
})
}
}

func TestBuildServiceNameForDNSEntryPoint(t *testing.T) {
Expand Down
16 changes: 16 additions & 0 deletions controllers/activegate/reconciler/statefulset/container_builder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package statefulset

import corev1 "k8s.io/api/core/v1"

type ContainerBuilder interface {
BuildContainer() corev1.Container
BuildVolumes() []corev1.Volume
}

type GenericContainer struct {
StsProperties *statefulSetProperties
}

func NewGenericContainer(stsProperties *statefulSetProperties) *GenericContainer {
return &GenericContainer{StsProperties: stsProperties}
}
141 changes: 141 additions & 0 deletions controllers/activegate/reconciler/statefulset/container_eec.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package statefulset

import (
"fmt"
"net/url"
"strings"

dynatracev1beta1 "github.com/Dynatrace/dynatrace-operator/api/v1beta1"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)

const eecIngestPortName = "eec-http"
const eecIngestPort = 14599

const activeGateInternalCommunicationPort = 9999

var _ ContainerBuilder = (*ExtensionController)(nil)

type ExtensionController struct {
GenericContainer
}

func NewExtensionController(stsProperties *statefulSetProperties) *ExtensionController {
return &ExtensionController{
GenericContainer: *NewGenericContainer(stsProperties),
}
}

func (eec *ExtensionController) BuildContainer() corev1.Container {
return corev1.Container{
Name: fmt.Sprintf("%s-eec", dynatracev1beta1.OperatorName),
Image: eec.StsProperties.DynaKube.ActiveGateImage(),
ImagePullPolicy: corev1.PullAlways,
Env: eec.buildEnvs(),
VolumeMounts: eec.buildVolumeMounts(),
Command: eec.buildCommand(),
Ports: eec.buildPorts(),
ReadinessProbe: &corev1.Probe{
Handler: corev1.Handler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/readyz",
Port: intstr.IntOrString{IntVal: eecIngestPort},
},
},
InitialDelaySeconds: 30,
PeriodSeconds: 15,
FailureThreshold: 3,
SuccessThreshold: 1,
TimeoutSeconds: 1,
},
}
}

func (eec *ExtensionController) BuildVolumes() []corev1.Volume {
return []corev1.Volume{
{
Name: "auth-tokens",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "eec-ds-shared",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "dsauthtokendir",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
},
{
Name: "eec-config",
VolumeSource: corev1.VolumeSource{
// TODO
},
},
}
}

func (eec *ExtensionController) buildPorts() []corev1.ContainerPort {
return []corev1.ContainerPort{
{Name: eecIngestPortName, ContainerPort: eecIngestPort},
}
}

func (eec *ExtensionController) buildCommand() []string {
if eec.StsProperties.DynaKube.FeatureUseActiveGateImageForStatsD() {
return []string{
"/bin/bash", "/dt/eec/entrypoint.sh",
}
}
return nil
}

func (eec *ExtensionController) buildVolumeMounts() []corev1.VolumeMount {
return []corev1.VolumeMount{
{Name: "auth-tokens", MountPath: "/var/lib/dynatrace/gateway/config"},
{Name: "eec-ds-shared", MountPath: "/mnt/dsexecargs"},
{Name: "dsauthtokendir", MountPath: "/var/lib/dynatrace/remotepluginmodule/agent/runtime/datasources"},
{Name: "eec-config", MountPath: "/var/lib/dynatrace/remotepluginmodule/agent/conf/runtime"},
{Name: "ds-metadata", MountPath: "/opt/dynatrace/remotepluginmodule/agent/datasources/statsd", ReadOnly: true},
}
}

func (eec *ExtensionController) buildEnvs() []corev1.EnvVar {
tenantId, err := getTenantId(eec.StsProperties.Spec.APIURL)
if err != nil {
eec.StsProperties.log.Error(err, "Problem getting tenant id from api url")
}
return []corev1.EnvVar{
{Name: "TenantId", Value: tenantId},
{Name: "ServerUrl", Value: fmt.Sprintf("https://localhost:%d/communication", activeGateInternalCommunicationPort)},
{Name: "EecIngestPort", Value: fmt.Sprintf("%d", eecIngestPort)},
}
}

func getTenantId(apiUrl string) (string, error) {
if !strings.HasSuffix(apiUrl, "/api") {
return "", fmt.Errorf("api url %s does not end with /api", apiUrl)
}

parsedUrl, err := url.Parse(apiUrl)
if err != nil {
return "", errors.WithMessagef(err, "problem parsing tenant id from url %s", apiUrl)
}

fqdn := parsedUrl.Hostname()
hostnameWithDomains := strings.FieldsFunc(fqdn,
func(r rune) bool { return r == '.' },
)
if len(hostnameWithDomains) < 1 {
return "", fmt.Errorf("problem getting tenant id from fqdn '%s'", fqdn)
}

return hostnameWithDomains[0], nil
}
Loading

0 comments on commit d7b9c60

Please sign in to comment.