[Elastic Agent] Enable configuring monitoring namespace (#26439) (#26555

) [Elastic Agent] Enable configuring monitoring namespace (#26439)
elastic · Jun 29, 2021 · 8e0c1a1 · 8e0c1a1
1 parent 6b970c7
commit 8e0c1a1
Show file tree

Hide file tree

Showing 16 changed files with 594 additions and 16 deletions.
diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc
@@ -116,4 +116,5 @@
 - Use `filestream` input for internal log collection. {pull}25660[25660]
 - Enable agent to send custom headers to kibana/ES {pull}26275[26275]
 - Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394]
+- Enable configuring monitoring namespace {issue}26439[26439]
 - Communicate with Fleet Server over HTTP2. {pull}26474[26474]
diff --git a/...ck/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go b/...ck/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go
@@ -49,6 +49,7 @@ func InjectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *tr
 			transpiler.NewKey("logs", transpiler.NewBoolVal(true)),
 			transpiler.NewKey("metrics", transpiler.NewBoolVal(true)),
 			transpiler.NewKey("use_output", transpiler.NewStrVal("default")),
+			transpiler.NewKey("namespace", transpiler.NewStrVal("default")),
 		})
 
 		transpiler.Insert(rootAst, transpiler.NewKey("monitoring", monitoringNode), "settings")

diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go
@@ -161,11 +161,12 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in
 	var steps []configrequest.Step
 	watchLogs := o.monitor.WatchLogs()
 	watchMetrics := o.monitor.WatchMetrics()
+	monitoringNamespace := o.monitor.MonitoringNamespace()
 
 	// generate only when monitoring is running (for config refresh) or
 	// state changes (turning on/off)
 	if watchLogs != o.isMonitoringLogs() || watchLogs {
-		fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output)
+		fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output, monitoringNamespace)
 		stepID := configrequest.StepRun
 		if !watchLogs || !any {
 			stepID = configrequest.StepRemove
@@ -182,7 +183,7 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in
 		steps = append(steps, filebeatStep)
 	}
 	if watchMetrics != o.isMonitoringMetrics() || watchMetrics {
-		mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output)
+		mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output, monitoringNamespace)
 		stepID := configrequest.StepRun
 		if !watchMetrics || !any {
 			stepID = configrequest.StepRemove
@@ -215,12 +216,12 @@ func loadSpecFromSupported(processName string) program.Spec {
 	}
 }
 
-func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) {
+func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) {
 	inputs := []interface{}{
 		map[string]interface{}{
 			"type": "filestream",
 			"parsers": []map[string]interface{}{
-				map[string]interface{}{
+				{
 					"ndjson": map[string]interface{}{
 						"overwrite_keys": true,
 						"message_key":    "message",
@@ -233,15 +234,15 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa
 				filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"),
 				filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"),
 			},
-			"index": "logs-elastic_agent-default",
+			"index": fmt.Sprintf("logs-elastic_agent-%s", monitoringNamespace),
 			"processors": []map[string]interface{}{
 				{
 					"add_fields": map[string]interface{}{
 						"target": "data_stream",
 						"fields": map[string]interface{}{
 							"type":      "logs",
 							"dataset":   "elastic_agent",
-							"namespace": "default",
+							"namespace": monitoringNamespace,
 						},
 					},
 				},
@@ -280,23 +281,23 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa
 			inputs = append(inputs, map[string]interface{}{
 				"type": "filestream",
 				"parsers": []map[string]interface{}{
-					map[string]interface{}{
+					{
 						"ndjson": map[string]interface{}{
 							"overwrite_keys": true,
 							"message_key":    "message",
 						},
 					},
 				},
 				"paths": paths,
-				"index": fmt.Sprintf("logs-elastic_agent.%s-default", name),
+				"index": fmt.Sprintf("logs-elastic_agent.%s-%s", name, monitoringNamespace),
 				"processors": []map[string]interface{}{
 					{
 						"add_fields": map[string]interface{}{
 							"target": "data_stream",
 							"fields": map[string]interface{}{
 								"type":      "logs",
 								"dataset":   fmt.Sprintf("elastic_agent.%s", name),
-								"namespace": "default",
+								"namespace": monitoringNamespace,
 							},
 						},
 					},
@@ -345,7 +346,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa
 	return result, true
 }
 
-func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) {
+func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) {
 	hosts := o.getMetricbeatEndpoints()
 	if len(hosts) == 0 {
 		return nil, false
@@ -359,15 +360,15 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter
 			"metricsets": []string{"stats", "state"},
 			"period":     "10s",
 			"hosts":      endpoints,
-			"index":      fmt.Sprintf("metrics-elastic_agent.%s-default", name),
+			"index":      fmt.Sprintf("metrics-elastic_agent.%s-%s", name, monitoringNamespace),
 			"processors": []map[string]interface{}{
 				{
 					"add_fields": map[string]interface{}{
 						"target": "data_stream",
 						"fields": map[string]interface{}{
 							"type":      "metrics",
 							"dataset":   fmt.Sprintf("elastic_agent.%s", name),
-							"namespace": "default",
+							"namespace": monitoringNamespace,
 						},
 					},
 				},
@@ -397,15 +398,15 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter
 			"period":     "10s",
 			"path":       "/stats",
 			"hosts":      endpoints,
-			"index":      fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName),
+			"index":      fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace),
 			"processors": []map[string]interface{}{
 				{
 					"add_fields": map[string]interface{}{
 						"target": "data_stream",
 						"fields": map[string]interface{}{
 							"type":      "metrics",
 							"dataset":   fmt.Sprintf("elastic_agent.%s", fixedAgentName),
-							"namespace": "default",
+							"namespace": monitoringNamespace,
 						},
 					},
 				},
@@ -480,15 +481,15 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter
 		"period":     "10s",
 		"path":       "/stats",
 		"hosts":      []string{beats.AgentPrefixedMonitoringEndpoint(o.config.DownloadConfig.OS(), o.config.MonitoringConfig.HTTP)},
-		"index":      fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName),
+		"index":      fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace),
 		"processors": []map[string]interface{}{
 			{
 				"add_fields": map[string]interface{}{
 					"target": "data_stream",
 					"fields": map[string]interface{}{
 						"type":      "metrics",
 						"dataset":   fmt.Sprintf("elastic_agent.%s", fixedAgentName),
-						"namespace": "default",
+						"namespace": monitoringNamespace,
 					},
 				},
 			},

diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go
@@ -215,6 +215,9 @@ func (b *testMonitor) Reload(cfg *config.Config) error { return nil }
 // IsMonitoringEnabled returns true if monitoring is configured.
 func (b *testMonitor) IsMonitoringEnabled() bool { return b.monitorLogs || b.monitorMetrics }
 
+// MonitoringNamespace returns monitoring namespace configured.
+func (b *testMonitor) MonitoringNamespace() string { return "default" }
+
 // WatchLogs return true if monitoring is configured and monitoring logs is enabled.
 func (b *testMonitor) WatchLogs() bool { return b.monitorLogs }
 

diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go
@@ -383,6 +383,10 @@ func TestConfiguration(t *testing.T) {
 		empty    bool
 		err      bool
 	}{
+		"namespace": {
+			programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"},
+			expected: 6,
+		},
 		"single_config": {
 			programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"},
 			expected: 6,

diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml
@@ -0,0 +1,114 @@
+fleet:
+  enabled: true
+  access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw
+  protocol: https
+  hosts: [ localhost:5601 ]
+  timeout: 30s
+  agent:
+    id: fleet-agent-id
+    logging.level: error
+  host:
+    id: host-agent-id
+
+output:
+  elasticsearch:
+    hosts:
+      - "127.0.0.1:9200"
+      - "127.0.0.1:9300"
+    namespace: test_namespace
+    username: elastic
+    password: changeme
+    api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA
+    ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y=
+
+inputs:
+- id: endpoint-id
+  type: endpoint
+  name: endpoint-1
+  enabled: true
+  package:
+    name: endpoint
+    version: 0.3.0
+  data_stream:
+    namespace: default
+  artifact_manifest:
+    schema_version: v22
+    manifest_version: v21
+    artifacts:
+      - endpoint-allowlist-windows:
+          sha256: 1234
+          size: 2
+          url: /relative/path/to/endpoint-allowlist-windows
+      - endpoint-allowlist-macos:
+          sha256: 1234
+          size: 2
+          url: /relative/path/to/endpoint-allowlist-macos
+      - endpoint-allowlist-linux:
+          sha256: 1234
+          size: 2
+          url: /relative/path/to/endpoint-allowlist-linux
+  policy:
+    linux:
+      advanced:
+        free-form: free-form-value
+      indices:
+        network: logs-endpoint.events.network-default
+        file: logs-endpoint.events.file-default
+        process: logs-endpoint.events.process-default
+        metadata: metrics-endpoint.metadata-default
+        policy: metrics-endpoint.policy-default
+        telemetry: metrics-endpoint.telemetry-default
+      logging:
+        file: info
+        stdout: debug
+      events:
+        process: true
+        file: true
+        network: true
+    windows:
+      malware:
+        mode: prevent
+      advanced:
+        free-form: free-form-value
+      indices:
+        network: logs-endpoint.events.network-default
+        file: logs-endpoint.events.file-default
+        registry: logs-endpoint.events.registry-default
+        process: logs-endpoint.events.process-default
+        driver: logs-endpoint.events.driver-default
+        library: logs-endpoint.events.library-default
+        alerts: logs-endpoint.alerts-default
+        metadata: metrics-endpoint.metadata-default
+        policy: metrics-endpoint.policy-default
+        telemetry: metrics-endpoint.telemetry-default
+      logging:
+        file: info
+        stdout: debug
+      events:
+        registry: true
+        process: true
+        security: true
+        file: true
+        dns: false
+        dll_and_driver_load: false
+        network: true
+    mac:
+      malware:
+        mode: prevent
+      advanced:
+        free-form: free-form-value
+      indices:
+        network: logs-endpoint.events.network-default
+        file: logs-endpoint.events.file-default
+        process: logs-endpoint.events.process-default
+        alerts: logs-endpoint.alerts-default
+        metadata: metrics-endpoint.metadata-default
+        policy: metrics-endpoint.policy-default
+        telemetry: metrics-endpoint.telemetry-default
+      logging:
+        file: info
+        stdout: debug
+      events:
+        process: true
+        file: true
+        network: true
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml
@@ -0,0 +1,68 @@
+filebeat:
+  inputs:
+  - type: log
+    paths:
+      - /var/log/hello1.log
+      - /var/log/hello2.log
+    index: logs-generic-default
+    vars:
+      var: value
+    processors:
+      - add_fields:
+          target: "data_stream"
+          fields:
+            type: logs
+            dataset: generic
+            namespace: default
+      - add_fields:
+          target: "event"
+          fields:
+            dataset: generic
+      - add_fields:
+          target: "elastic_agent"
+          fields:
+            id: agent-id
+            version: 8.0.0
+            snapshot: false
+      - add_fields:
+          target: "agent"
+          fields:
+            id: agent-id
+  - type: log
+    paths:
+      - /var/log/hello3.log
+      - /var/log/hello4.log
+    index: testtype-generic-default
+    vars:
+      var: value
+    processors:
+      - add_fields:
+          target: "data_stream"
+          fields:
+            type: testtype
+            dataset: generic
+            namespace: default
+      - add_fields:
+          target: "event"
+          fields:
+            dataset: generic
+      - add_fields:
+          target: "elastic_agent"
+          fields:
+            id: agent-id
+            version: 8.0.0
+            snapshot: false
+      - add_fields:
+          target: "agent"
+          fields:
+            id: agent-id
+output:
+  elasticsearch:
+    hosts:
+      - 127.0.0.1:9200
+      - 127.0.0.1:9300
+    namespace: test_namespace
+    username: elastic
+    password: changeme
+    api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA
+    ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y=
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml
@@ -0,0 +1,16 @@
+fleet:
+  agent:
+    id: fleet-agent-id
+    logging.level: error
+  host:
+    id: host-agent-id
+
+output:
+  elasticsearch:
+    hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ]
+    username: fleet
+    password: fleetpassword
+
+inputs:
+  - id: fleet-server-id
+    type: fleet-server