Move s3_daily_storage and s3_request metricsets to use cloudwatch inp…

…ut (elastic#21703)
leweafan · Oct 14, 2020 · 28ea348 · 28ea348
1 parent c51c70c
commit 28ea348
Show file tree

Hide file tree

Showing 18 changed files with 209 additions and 581 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -756,6 +756,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move Prometheus query & remote_write to GA. {pull}21507[21507]
 - Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]
 - Map cloud data filed `cloud.account.id` to azure subscription.  {pull}21483[21483] {issue}21381[21381]
+- Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703]
 
 *Packetbeat*
 

diff --git a/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc b/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc
@@ -8,6 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 include::../../../../x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc[]
 
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
 
 ==== Fields
 

diff --git a/metricbeat/docs/modules/aws/s3_request.asciidoc b/metricbeat/docs/modules/aws/s3_request.asciidoc
@@ -8,6 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 include::../../../../x-pack/metricbeat/module/aws/s3_request/_meta/docs.asciidoc[]
 
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
 
 ==== Fields
 

diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go
diff --git a/x-pack/metricbeat/module/aws/module.yml b/x-pack/metricbeat/module/aws/module.yml
@@ -4,6 +4,8 @@ metricsets:
   - ebs
   - usage
   - sns
+  - s3_daily_storage
+  - s3_request
   - lambda
   - dynamodb
   - vpn

diff --git a/x-pack/metricbeat/module/aws/mtest/integration.go b/x-pack/metricbeat/module/aws/mtest/integration.go
@@ -39,6 +39,7 @@ func GetConfigForTest(t *testing.T, metricSetName string, period string) map[str
 			"access_key_id":     accessKeyID,
 			"secret_access_key": secretAccessKey,
 			"default_region":    defaultRegion,
+			"latency":           "5m",
 			// You can specify which region to run test on by using regions variable
 			// "regions":           []string{"us-east-1"},
 		}

diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json
@@ -1,27 +1,28 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "aws": {
-        "s3": {
-            "bucket": {
-                "name": "test-s3-ks-2"
-            }
+        "cloudwatch": {
+            "namespace": "AWS/S3"
+        },
+        "dimensions": {
+            "BucketName": "filebeat-aws-elb-test",
+            "StorageType": "AllStorageTypes"
         },
-        "s3_daily_storage": {
-            "bucket": {
-                "size": {
-                    "bytes": 207372
+        "s3": {
+            "metrics": {
+                "NumberOfObjects": {
+                    "avg": 57828
                 }
-            },
-            "number_of_objects": 128
+            }
         }
     },
     "cloud": {
         "account": {
-            "id": "627959692251",
-            "name": "elastic-test"
+            "id": "428152502467",
+            "name": "elastic-beats"
         },
         "provider": "aws",
-        "region": "ap-southeast-1"
+        "region": "eu-central-1"
     },
     "event": {
         "dataset": "aws.s3_daily_storage",

diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/data.go b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go
diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml
@@ -0,0 +1,22 @@
+default: true
+input:
+  module: aws
+  metricset: cloudwatch
+  defaults:
+    metrics:
+      - namespace: AWS/S3
+        statistic: ["Average"]
+        name:
+          - BucketSizeBytes
+          - NumberOfObjects
+
+processors:
+  - rename:
+      ignore_missing: true
+      fields:
+        - from: "aws.s3.metrics.NumberOfObjects.avg"
+          to: "aws.s3_daily_storage.number_of_objects"
+        - from: "aws.s3.metrics.BucketSizeBytes.avg"
+          to: "aws.s3_daily_storage.bucket.size.bytes"
+        - from: "aws.dimensions.BucketName"
+          to: "aws.s3.bucket.name"
diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go
diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go
@@ -28,22 +28,18 @@ func TestFetch(t *testing.T) {
 	assert.NotEmpty(t, events)
 
 	for _, event := range events {
-		// RootField
-		mtest.CheckEventField("service.name", "string", event, t)
 		mtest.CheckEventField("cloud.region", "string", event, t)
-
-		// MetricSetField
-		mtest.CheckEventField("bucket.name", "string", event, t)
-		mtest.CheckEventField("bucket.size.bytes", "float", event, t)
-		mtest.CheckEventField("number_of_objects", "float", event, t)
+		mtest.CheckEventField("aws.dimensions.BucketName", "string", event, t)
+		mtest.CheckEventField("aws.dimensions.StorageType", "string", event, t)
+		mtest.CheckEventField("aws.s3.metrics.BucketSizeBytes.avg", "float", event, t)
+		mtest.CheckEventField("aws.s3.metrics.NumberOfObjects.avg", "float", event, t)
+		break
 	}
 }
 
 func TestData(t *testing.T) {
 	config := mtest.GetConfigForTest(t, "s3_daily_storage", "86400s")
 
-	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
-	if err := mbtest.WriteEventsReporterV2Error(metricSet, t, "/"); err != nil {
-		t.Fatal("write", err)
-	}
+	metricSet := mbtest.NewFetcher(t, config)
+	metricSet.WriteEvents(t, "/")
 }