Add clusterName, taskName, cloud.region and dashboard to aws fargate (#…

…22941) (#23026) (cherry picked from commit 7cbb1a2)
elastic · Dec 9, 2020 · a6fb530 · a6fb530
1 parent 492da29
commit a6fb530
Show file tree

Hide file tree

Showing 15 changed files with 1,369 additions and 25 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -569,6 +569,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347]
 - Add io.ops in fields exported by system.diskio. {pull}22066[22066]
 - Adjust the Apache status fields in the fleet mode. {pull}22821[22821]
+- Add AWS Fargate overview dashboard. {pull}22941[22941]
 - Add process.state, process.cpu.pct, process.cpu.start_time and process.memory.pct. {pull}22845[22845]
 
 *Packetbeat*

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -4604,6 +4604,16 @@ type: double
 
 
 
+*`task_stats.identifier`*::
++
+--
+Container identifier across tasks and clusters, which equals to container.name + '/' + container.id.
+
+
+type: keyword
+
+--
+
 [float]
 === cpu
 

diff --git a/metricbeat/docs/images/metricbeat-awsfargate-overview.png b/metricbeat/docs/images/metricbeat-awsfargate-overview.png
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -32,7 +32,7 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-aws-transitgateway,transitgateway>> beta[]  
 |<<metricbeat-metricset-aws-usage,usage>> beta[]  
 |<<metricbeat-metricset-aws-vpn,vpn>> beta[]  
-|<<metricbeat-module-awsfargate,AWS Fargate>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
+|<<metricbeat-module-awsfargate,AWS Fargate>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-awsfargate-task_stats,task_stats>> beta[]  
 |<<metricbeat-module-azure,Azure>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .11+| .11+|  |<<metricbeat-metricset-azure-app_insights,app_insights>> beta[]  

diff --git a/...metricbeat/module/awsfargate/_meta/kibana/7/dashboard/Metricbeat-awsfargate-overview.json b/...metricbeat/module/awsfargate/_meta/kibana/7/dashboard/Metricbeat-awsfargate-overview.json
diff --git a/x-pack/metricbeat/module/awsfargate/awsfargate.go b/x-pack/metricbeat/module/awsfargate/awsfargate.go
@@ -38,7 +38,7 @@ func newModule(base mb.BaseModule) (mb.Module, error) {
 	return &base, nil
 }
 
-// NewMetricSet creates a base metricset for awsfargate metricsets
+// NewMetricSet creates a base metricset for awsfargate metricset
 func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 	var config Config
 	err := base.Module().UnpackConfig(&config)

diff --git a/x-pack/metricbeat/module/awsfargate/cloudformation.yml b/x-pack/metricbeat/module/awsfargate/cloudformation.yml
@@ -2,22 +2,36 @@ AWSTemplateFormatVersion: "2010-09-09"
 Parameters:
   SubnetID:
     Type: String
+  CloudIDArn:
+    Type: String
+  CloudAuthArn:
+    Type: String
+  ClusterName:
+    Type: String
+  RoleName:
+    Type: String
+  TaskName:
+    Type: String
+  ServiceName:
+    Type: String
+  LogGroupName:
+    Type: String
 Resources:
   Cluster:
     Type: AWS::ECS::Cluster
     Properties:
-      ClusterName: metricbeat-cloudformation-fargate
+      ClusterName: !Ref ClusterName
       ClusterSettings:
         - Name: containerInsights
           Value: enabled
   LogGroup:
     Type: AWS::Logs::LogGroup
     Properties:
-      LogGroupName: metricbeat-fargate-log-group
+      LogGroupName: !Ref LogGroupName
   ExecutionRole:
     Type: AWS::IAM::Role
     Properties:
-      RoleName: ecsFargateTaskExecutionRole
+      RoleName: !Ref RoleName
       AssumeRolePolicyDocument:
         Statement:
           - Effect: Allow
@@ -35,24 +49,24 @@ Resources:
                 Action:
                   - secretsmanager:GetSecretValue
                 Resource:
-                  - <ELASTIC_CLOUD_ID_ARN>
-                  - <ELASTIC_CLOUD_AUTH_ARN>
+                  - !Ref CloudIDArn
+                  - !Ref CloudAuthArn
   TaskDefinition:
     Type: AWS::ECS::TaskDefinition
     Properties:
-      Family: deployment-task-metricbeat
+      Family: !Ref TaskName
       Cpu: 256
       Memory: 512
       NetworkMode: awsvpc
       ExecutionRoleArn: !Ref ExecutionRole
       ContainerDefinitions:
-        - Name: deployment-task-metricbeat-container
-          Image: kaiyansheng/metricbeat-awsfargate:v1
+        - Name: metricbeat-container
+          Image: docker.elastic.co/beats/metricbeat:7.11.0-SNAPSHOT
           Secrets:
             - Name: ELASTIC_CLOUD_ID
-              ValueFrom: <ELASTIC_CLOUD_ID_ARN>
+              ValueFrom: !Ref CloudIDArn
             - Name: ELASTIC_CLOUD_AUTH
-              ValueFrom: <ELASTIC_CLOUD_AUTH_ARN>
+              ValueFrom: !Ref CloudAuthArn
           LogConfiguration:
             LogDriver: awslogs
             Options:
@@ -63,14 +77,25 @@ Resources:
             - sh
             - -c
           Command:
-            - ./metricbeat setup && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH
+            - ./metricbeat setup -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH
+        - Name: stress-test
+          Image: containerstack/alpine-stress
+          Essential: false
+          DependsOn:
+            - ContainerName: metricbeat-container
+              Condition: START
+          EntryPoint:
+            - sh
+            - -c
+          Command:
+            - stress --cpu 8 --io 4 --vm 2 --vm-bytes 128M --timeout 6000s
       RequiresCompatibilities:
         - EC2
         - FARGATE
   Service:
     Type: AWS::ECS::Service
     Properties:
-      ServiceName: deployment-metricbeat-service
+      ServiceName: !Ref ServiceName
       Cluster: !Ref Cluster
       TaskDefinition: !Ref TaskDefinition
       DesiredCount: 1

diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go
diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json
@@ -2,6 +2,7 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "awsfargate": {
         "task_stats": {
+            "cluster_name": "default",
             "cpu": {
                 "core": null,
                 "kernel": {
@@ -61,6 +62,7 @@
                 },
                 "writes": 0
             },
+            "identifier": "query-metadata/1234",
             "memory": {
                 "fail": {
                     "count": 0
@@ -125,11 +127,15 @@
                         "packets": 25857
                     }
                 }
-            }
+            },
+            "task_name": "query-metadata"
         }
     },
+    "cloud": {
+        "region": "us-west-2"
+    },
     "container": {
-        "id": "query-metadata-1",
+        "id": "1234",
         "image": {
             "name": "mreferre/eksutils"
         },

diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml
@@ -4,6 +4,10 @@
     `task_stats` contains the metrics that were scraped from AWS fargate task stats ${ECS_CONTAINER_METADATA_URI_V4}/task/stats metadata endpoint.
   release: beta
   fields:
+    - name: identifier
+      type: keyword
+      description: >
+        Container identifier across tasks and clusters, which equals to container.name + '/' + container.id.
     - name: cpu
       type: group
       description: Runtime CPU metrics.

diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json
@@ -4,7 +4,7 @@
     "Family": "query-metadata-1",
     "Revision": "7",
     "Containers": [{
-        "DockerId": "query-metadata-1",
+        "DockerId": "1234",
         "Name": "query-metadata",
         "Image": "mreferre/eksutils",
         "Labels": {

diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json
@@ -1,5 +1,5 @@
 {
-    "query-metadata-1": {
+    "1234": {
         "blkio_stats": {
             "io_service_bytes_recursive": [
                 {"major": 202, "minor": 26368, "op": "Read", "value": 3452928},
@@ -36,7 +36,7 @@
                 "throttled_time": 0
             }
         },
-        "id": "query-metadata-1",
+        "id": "1234",
         "memory_stats": {
             "limit": 3937787904,
             "max_usage": 15294464,

diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go
@@ -5,33 +5,74 @@
 package task_stats
 
 import (
+	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws/arn"
+
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
+var (
+	clusterLabel = "com_amazonaws_ecs_cluster"
+	taskLabel    = "com_amazonaws_ecs_task-definition-family"
+)
+
 func eventsMapping(r mb.ReporterV2, statsList []Stats) {
 	for _, stats := range statsList {
 		r.Event(createEvent(&stats))
 	}
 }
 
 func createEvent(stats *Stats) mb.Event {
-	return mb.Event{
-		Timestamp:  time.Time(stats.Time),
-		RootFields: createContainerFields(stats),
+	e := mb.Event{
+		Timestamp: time.Time(stats.Time),
 		MetricSetFields: common.MapStr{
 			"cpu":     createCPUFields(stats),
 			"memory":  createMemoryFields(stats),
 			"network": createNetworkFields(stats),
 			"diskio":  createDiskIOFields(stats),
 		},
 	}
+
+	regionName, clusterName := getRegionAndClusterName(stats.Container.Labels)
+	e.RootFields = createRootFields(stats, regionName)
+	if clusterName != "" {
+		e.MetricSetFields.Put("cluster_name", clusterName)
+	}
+
+	taskName := stats.Container.Labels[taskLabel]
+	if taskName != "" {
+		e.MetricSetFields.Put("task_name", taskName)
+	}
+
+	e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId))
+	return e
 }
 
-func createContainerFields(stats *Stats) common.MapStr {
-	return common.MapStr{
+func generateIdentifier(containerName string, containerID string) string {
+	return containerName + "/" + containerID
+}
+
+func getRegionAndClusterName(labels map[string]string) (regionName string, clusterName string) {
+	if v, ok := labels[clusterLabel]; ok {
+		vSplit := strings.Split(v, "cluster/")
+		if len(vSplit) == 2 {
+			clusterName = vSplit[1]
+		}
+
+		arnParsed, err := arn.Parse(v)
+		if err == nil {
+			regionName = arnParsed.Region
+		}
+		return
+	}
+	return
+}
+
+func createRootFields(stats *Stats, regionName string) common.MapStr {
+	rootFields := common.MapStr{
 		"container": common.MapStr{
 			"id": stats.Container.DockerId,
 			"image": common.MapStr{
@@ -41,6 +82,15 @@ func createContainerFields(stats *Stats) common.MapStr {
 			"labels": stats.Container.Labels,
 		},
 	}
+
+	// add cloud.region
+	if regionName != "" {
+		cloud := common.MapStr{
+			"region": regionName,
+		}
+		rootFields.Put("cloud", cloud)
+	}
+	return rootFields
 }
 
 func createCPUFields(stats *Stats) common.MapStr {

diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go
@@ -30,6 +30,7 @@ func TestData(t *testing.T) {
 		"metricsets": []string{"task_stats"},
 	}
 
+	os.Setenv("ECS_CONTAINER_METADATA_URI_V4", "1.2.3.4")
 	m := mbtest.NewFetcher(t, config)
 
 	taskStatsFile, err := os.Open("./_meta/testdata/task_stats.json")

diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go
@@ -122,3 +122,15 @@ func TestGetCPUStats(t *testing.T) {
 	assert.Equal(t, 0.4, cpuStats.TotalUsage)
 	assert.Equal(t, 0.2, cpuStats.TotalUsageNormalized)
 }
+
+func TestGetRegionAndClusterName(t *testing.T) {
+	labels := map[string]string{}
+	labels["com_amazonaws_ecs_cluster"] = "arn:aws:ecs:us-east-1:1234:cluster/metricbeat-fargate-1"
+	regionName, clusterName := getRegionAndClusterName(labels)
+	assert.Equal(t, "us-east-1", regionName)
+	assert.Equal(t, "metricbeat-fargate-1", clusterName)
+}
+
+func TestGenerateIdentifier(t *testing.T) {
+	assert.Equal(t, "container-name/123", generateIdentifier("container-name", "123"))
+}