Skip to content

Commit

Permalink
OTel Agent Feature Support (#1559)
Browse files Browse the repository at this point in the history 
* OTel Agent Feature Support

* update test

* add port documentation

* remove flag support

* add examples

* fix

* fix examples

* fix

* fix test

* rename feature from otelAgent to otelCollector

* rename

* rename directory/ pkg

* undo change to InitConfig volumes

* remove unrelated changes

* add empty line

* add otel collector feature to Test_defaultFeatures

* add otel agent feature to factory test

* add annotation enabled/ feature disabled test

* Use Replace with i=1 instead of ReplaceAll as we know there is only 1 instance to replace

* fix examples

* update example ports to 0.0.0.0 (defaults to localhost) and add annotations example with remove note

* update o.ports rather than DDA directly

* add support for core config

* Move image to images.go per feedback

* fix test after merge of main

* add non default auth path volume mount, required for staging

* add vm logs

* change sync delay to 30s

---------

Co-authored-by: levan-m <116471169+levan-m@users.noreply.github.com>
  • Loading branch information
@mackjmr @levan-m
mackjmr and levan-m authored Dec 20, 2024
1 parent f517775 commit e2f08ba
Show file tree
Hide file tree
Showing 27 changed files with 1,867 additions and 40 deletions.
2 changes: 2 additions & 0 deletions api/datadoghq/v2alpha1/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ const (
DefaultDogstatsdPort = 8125
// DefaultDogstatsdPortName default dogstatsd port name
DefaultDogstatsdPortName = "dogstatsdport"
// DefaultOTelAgentConf default otel agent ConfigMap name
DefaultOTelAgentConf string = "otel-agent-config"
// DefaultKubeStateMetricsCoreConf default ksm core ConfigMap name
DefaultKubeStateMetricsCoreConf string = "kube-state-metrics-core-config"
// DefaultOrchestratorExplorerConf default orchestrator explorer ConfigMap name
Expand Down
7 changes: 7 additions & 0 deletions api/datadoghq/v2alpha1/datadogagent_default.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const (
defaultLogPodLogsPath string = "/var/log/pods"
defaultLogContainerSymlinksPath string = "/var/log/containers"

defaultOtelCollectorEnabled bool = false
defaultLiveProcessCollectionEnabled bool = false
defaultLiveContainerCollectionEnabled bool = true
defaultProcessDiscoveryEnabled bool = true
Expand Down Expand Up @@ -222,6 +223,12 @@ func defaultFeaturesConfig(ddaSpec *DatadogAgentSpec) {
}
apiutils.DefaultBooleanIfUnset(&ddaSpec.Features.LiveContainerCollection.Enabled, defaultLiveContainerCollectionEnabled)

// OTelCollector Feature
if ddaSpec.Features.OtelCollector == nil {
ddaSpec.Features.OtelCollector = &OtelCollectorFeatureConfig{}
}
apiutils.DefaultBooleanIfUnset(&ddaSpec.Features.OtelCollector.Enabled, defaultOtelCollectorEnabled)

// LiveProcessCollection Feature
if ddaSpec.Features.LiveProcessCollection == nil {
ddaSpec.Features.LiveProcessCollection = &LiveProcessCollectionFeatureConfig{}
Expand Down
185 changes: 185 additions & 0 deletions api/datadoghq/v2alpha1/datadogagent_default_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -332,6 +335,9 @@ func Test_defaultFeatures(t *testing.T) {
APM: &APMFeatureConfig{
Enabled: apiutils.NewBoolPointer(valueFalse),
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(valueFalse),
Expand Down Expand Up @@ -419,6 +425,9 @@ func Test_defaultFeatures(t *testing.T) {
APM: &APMFeatureConfig{
Enabled: apiutils.NewBoolPointer(valueFalse),
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(valueFalse),
Expand Down Expand Up @@ -554,6 +563,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -698,6 +710,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -837,6 +852,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -976,6 +994,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -1124,6 +1145,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -1263,6 +1287,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -1405,6 +1432,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -1542,6 +1572,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -1698,6 +1731,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -1788,6 +1824,149 @@ func Test_defaultFeatures(t *testing.T) {
},
},
},

{
name: "OTel Collector is enabled",
ddaSpec: &DatadogAgentSpec{
Features: &DatadogFeatures{
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(valueTrue),
},
},
},
want: &DatadogAgentSpec{
Features: &DatadogFeatures{
LogCollection: &LogCollectionFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultLogCollectionEnabled),
},
LiveProcessCollection: &LiveProcessCollectionFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultLiveProcessCollectionEnabled),
},
LiveContainerCollection: &LiveContainerCollectionFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultLiveContainerCollectionEnabled),
},
ProcessDiscovery: &ProcessDiscoveryFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultProcessDiscoveryEnabled),
},
OOMKill: &OOMKillFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOOMKillEnabled),
},
TCPQueueLength: &TCPQueueLengthFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultTCPQueueLengthEnabled),
},
EBPFCheck: &EBPFCheckFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultEBPFCheckEnabled),
},
ServiceDiscovery: &ServiceDiscoveryFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultServiceDiscoveryEnabled),
},
APM: &APMFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultAPMEnabled),
HostPortConfig: &HostPortConfig{
Port: apiutils.NewInt32Pointer(defaultAPMHostPort),
Enabled: apiutils.NewBoolPointer(defaultAPMHostPortEnabled),
},
UnixDomainSocketConfig: &UnixDomainSocketConfig{
Enabled: apiutils.NewBoolPointer(defaultAPMSocketEnabled),
Path: apiutils.NewStringPointer(defaultAPMSocketHostPath),
},
SingleStepInstrumentation: &SingleStepInstrumentation{
Enabled: apiutils.NewBoolPointer(defaultAPMSingleStepInstrEnabled),
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(valueTrue),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
},
SCA: &ASMSCAConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMSCAEnabled),
},
IAST: &ASMIASTConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMIASTEnabled),
},
},
CSPM: &CSPMFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultCSPMEnabled),
},
CWS: &CWSFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultCWSEnabled),
},
NPM: &NPMFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultNPMEnabled),
},
USM: &USMFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultUSMEnabled),
},
Dogstatsd: &DogstatsdFeatureConfig{
OriginDetectionEnabled: apiutils.NewBoolPointer(defaultDogstatsdOriginDetectionEnabled),
HostPortConfig: &HostPortConfig{Enabled: apiutils.NewBoolPointer(defaultDogstatsdHostPortEnabled)},
UnixDomainSocketConfig: &UnixDomainSocketConfig{
Enabled: apiutils.NewBoolPointer(defaultDogstatsdSocketEnabled),
Path: apiutils.NewStringPointer(defaultDogstatsdHostSocketPath),
},
},
OTLP: &OTLPFeatureConfig{Receiver: OTLPReceiverConfig{Protocols: OTLPProtocolsConfig{
GRPC: &OTLPGRPCConfig{
Enabled: apiutils.NewBoolPointer(defaultOTLPGRPCEnabled),
HostPortConfig: nil,
Endpoint: apiutils.NewStringPointer(defaultOTLPGRPCEndpoint),
},
HTTP: &OTLPHTTPConfig{
Enabled: apiutils.NewBoolPointer(defaultOTLPHTTPEnabled),
HostPortConfig: nil,
Endpoint: apiutils.NewStringPointer(defaultOTLPHTTPEndpoint),
},
}}},
RemoteConfiguration: &RemoteConfigurationFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultRemoteConfigurationEnabled),
},
EventCollection: &EventCollectionFeatureConfig{
CollectKubernetesEvents: apiutils.NewBoolPointer(defaultCollectKubernetesEvents),
},
OrchestratorExplorer: &OrchestratorExplorerFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOrchestratorExplorerEnabled),
ScrubContainers: apiutils.NewBoolPointer(defaultOrchestratorExplorerScrubContainers),
},
ExternalMetricsServer: &ExternalMetricsServerFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultExternalMetricsServerEnabled),
},
KubeStateMetricsCore: &KubeStateMetricsCoreFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultKubeStateMetricsCoreEnabled),
},
ClusterChecks: &ClusterChecksFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultClusterChecksEnabled),
UseClusterChecksRunners: apiutils.NewBoolPointer(defaultUseClusterChecksRunners),
},
AdmissionController: &AdmissionControllerFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionControllerEnabled),
Validation: &AdmissionControllerValidationConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionControllerValidationEnabled),
},
Mutation: &AdmissionControllerMutationConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionControllerMutationEnabled),
},
MutateUnlabelled: apiutils.NewBoolPointer(defaultAdmissionControllerMutateUnlabelled),
ServiceName: apiutils.NewStringPointer(defaultAdmissionServiceName),
CWSInstrumentation: &CWSInstrumentationConfig{
Enabled: apiutils.NewBoolPointer(DefaultAdmissionControllerCWSInstrumentationEnabled),
},
KubernetesAdmissionEvents: &KubernetesAdmissionEventsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionControllerKubernetesAdmissionEventsEnabled),
},
},
PrometheusScrape: &PrometheusScrapeFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultPrometheusScrapeEnabled),
},
HelmCheck: &HelmCheckFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultHelmCheckEnabled),
},
},
},
},
{
// This test sets same defaults as the one with `Features: nil`; and leaves other configs as empty structs.
name: "all feature configs are empty structs, configures defaults where applicable, leaves others empty",
Expand Down Expand Up @@ -1860,6 +2039,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down Expand Up @@ -2002,6 +2184,9 @@ func Test_defaultFeatures(t *testing.T) {
LanguageDetection: &LanguageDetectionConfig{Enabled: apiutils.NewBoolPointer(defaultLanguageDetectionEnabled)},
},
},
OtelCollector: &OtelCollectorFeatureConfig{
Enabled: apiutils.NewBoolPointer(defaultOtelCollectorEnabled),
},
ASM: &ASMFeatureConfig{
Threats: &ASMThreatsConfig{
Enabled: apiutils.NewBoolPointer(defaultAdmissionASMThreatsEnabled),
Expand Down
47 changes: 47 additions & 0 deletions api/datadoghq/v2alpha1/datadogagent_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ type DatadogAgentSpec struct {
type DatadogFeatures struct {
// Application-level features

// OtelCollector configuration.
OtelCollector *OtelCollectorFeatureConfig `json:"otelCollector,omitempty"`
// LogCollection configuration.
LogCollection *LogCollectionFeatureConfig `json:"logCollection,omitempty"`
// LiveProcessCollection configuration.
Expand Down Expand Up @@ -682,6 +684,51 @@ type KubeStateMetricsCoreFeatureConfig struct {
Conf *CustomConfig `json:"conf,omitempty"`
}

// OtelCollectorFeatureConfig contains the configuration for the otel-agent.
// +k8s:openapi-gen=true
type OtelCollectorFeatureConfig struct {
// Enabled enables the OTel Agent.
// Default: true
// +optional
Enabled *bool `json:"enabled,omitempty"`

// Conf overrides the configuration for the default Kubernetes State Metrics Core check.
// This must point to a ConfigMap containing a valid cluster check configuration.
// When passing a configmap, file name *must* be otel-config.yaml.
// +optional
Conf *CustomConfig `json:"conf,omitempty"`

// Ports contains the ports for the otel-agent.
// Defaults: otel-grpc:4317 / otel-http:4318. Note: setting 4317
// or 4318 manually is *only* supported if name match default names (otel-grpc, otel-http).
// If not, this will lead to a port conflict.
// This limitation will be lifted once annotations support is removed.
// +optional
Ports []*corev1.ContainerPort `json:"ports,omitempty"`

// OTelCollector Config Relevant to the Core agent
// +optional
CoreConfig *CoreConfig `json:"coreConfig,omitempty"`
}

// CoreConfig exposes the otel collector configs relevant to the core agent.
// +k8s:openapi-gen=true
type CoreConfig struct {
// Enabled marks otelcollector as enabled in core agent.
// +optional
Enabled *bool `json:"enabled,omitempty"`

// +optional
// Extension URL provides the URL of the ddflareextension to
// the core agent.
ExtensionURL *string `json:"extension_url,omitempty"`

// +optional
// Extension URL provides the timout of the ddflareextension to
// the core agent.
ExtensionTimeout *int `json:"extension_timeout,omitempty"`
}

// AdmissionControllerFeatureConfig contains the Admission Controller feature configuration.
// The Admission Controller runs in the Cluster Agent.
type AdmissionControllerFeatureConfig struct {
Expand Down
4 changes: 4 additions & 0 deletions api/datadoghq/v2alpha1/envvar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ const (
DDSystemProbeExternal = "DD_SYSTEM_PROBE_EXTERNAL"
DDSystemProbeSocket = "DD_SYSPROBE_SOCKET"
DDTags = "DD_TAGS"
// otelcollector core agent configs
DDOtelCollectorCoreConfigEnabled = "DD_OTELCOLLECTOR_ENABLED"
DDOtelCollectorCoreConfigExtensionURL = "DD_OTELCOLLECTOR_EXTENSION_URL"
DDOtelCollectorCoreConfigExtensionTimeout = "DD_OTELCOLLECTOR_EXTENSION_TIMEOUT"

DockerHost = "DOCKER_HOST"
// KubernetesEnvvarName Env var used by the Datadog Agent container entrypoint
Expand Down
Loading

0 comments on commit e2f08ba

Please sign in to comment.