Cherry-pick #23726 to 7.x: Fix ec2 metricset fields.yml and add integ…

…ration test (#24067)

(cherry picked from commit 685d85c)

CHANGELOG.next.asciidoc

@@ -313,6 +313,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
 - Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
+- Fix ec2 metricset fields.yml and the integration test {pull}23726[23726]
 - Unskip s3_request integration test. {pull}23887[23887]
 - Add system.hostfs configuration option for system module. {pull}23831[23831]
 

metricbeat/docs/fields.asciidoc

@@ -2326,7 +2326,7 @@ type: scaled_float
 
 --
 
-*`aws.ec2.diskio.read.ops`*::
+*`aws.ec2.diskio.read.count`*::
 +
 --
 Total completed read operations from all instance store volumes available to the instance in collection period.
@@ -2336,7 +2336,7 @@ type: long
 
 --
 
-*`aws.ec2.diskio.read.ops_per_sec`*::
+*`aws.ec2.diskio.read.count_per_sec`*::
 +
 --
 Completed read operations per second from all instance store volumes available to the instance in a specified period of time.
@@ -2346,7 +2346,7 @@ type: long
 
 --
 
-*`aws.ec2.diskio.write.ops`*::
+*`aws.ec2.diskio.write.count`*::
 +
 --
 Total completed write operations to all instance store volumes available to the instance in collection period.
@@ -2356,7 +2356,7 @@ type: long
 
 --
 
-*`aws.ec2.diskio.write.ops_per_sec`*::
+*`aws.ec2.diskio.write.count_per_sec`*::
 +
 --
 Completed write operations per second to all instance store volumes available to the instance in a specified period of time.

x-pack/metricbeat/module/aws/ec2/_meta/fields.yml

@@ -76,19 +76,19 @@
       type: scaled_float
       description: >
         Bytes written per second to all instance store volumes available to the instance.
-    - name: diskio.read.ops
+    - name: diskio.read.count
       type: long
       description: >
         Total completed read operations from all instance store volumes available to the instance in collection period.
-    - name: diskio.read.ops_per_sec
+    - name: diskio.read.count_per_sec
       type: long
       description: >
         Completed read operations per second from all instance store volumes available to the instance in a specified period of time.
-    - name: diskio.write.ops
+    - name: diskio.write.count
       type: long
       description: >
         Total completed write operations to all instance store volumes available to the instance in collection period.
-    - name: diskio.write.ops_per_sec
+    - name: diskio.write.count_per_sec
       type: long
       description: >
         Completed write operations per second to all instance store volumes available to the instance in a specified period of time.

x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go

@@ -17,7 +17,6 @@ import (
 )
 
 func TestFetch(t *testing.T) {
-	t.Skip("flaky test: https://github.com/elastic/beats/issues/20951")
 	config := mtest.GetConfigForTest(t, "ec2", "300s")
 
 	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
@@ -27,41 +26,7 @@ func TestFetch(t *testing.T) {
 	}
 
 	assert.NotEmpty(t, events)
-
-	for _, event := range events {
-		// RootField
-		mtest.CheckEventField("service.name", "string", event, t)
-		mtest.CheckEventField("cloud.availability_zone", "string", event, t)
-		mtest.CheckEventField("cloud.provider", "string", event, t)
-		mtest.CheckEventField("cloud.instance.id", "string", event, t)
-		mtest.CheckEventField("cloud.machine.type", "string", event, t)
-		mtest.CheckEventField("cloud.provider", "string", event, t)
-		mtest.CheckEventField("cloud.region", "string", event, t)
-		mtest.CheckEventField("instance.image.id", "string", event, t)
-		mtest.CheckEventField("instance.state.name", "string", event, t)
-		mtest.CheckEventField("instance.state.code", "int", event, t)
-		mtest.CheckEventField("instance.monitoring.state", "string", event, t)
-		mtest.CheckEventField("instance.core.count", "int", event, t)
-		mtest.CheckEventField("instance.threads_per_core", "int", event, t)
-
-		// MetricSetField
-		mtest.CheckEventField("cpu.total.pct", "float", event, t)
-		mtest.CheckEventField("cpu.credit_usage", "float", event, t)
-		mtest.CheckEventField("cpu.credit_balance", "float", event, t)
-		mtest.CheckEventField("cpu.surplus_credit_balance", "float", event, t)
-		mtest.CheckEventField("cpu.surplus_credits_charged", "float", event, t)
-		mtest.CheckEventField("network.in.packets", "float", event, t)
-		mtest.CheckEventField("network.out.packets", "float", event, t)
-		mtest.CheckEventField("network.in.bytes", "float", event, t)
-		mtest.CheckEventField("network.out.bytes", "float", event, t)
-		mtest.CheckEventField("diskio.read.bytes", "float", event, t)
-		mtest.CheckEventField("diskio.write.bytes", "float", event, t)
-		mtest.CheckEventField("diskio.read.ops", "float", event, t)
-		mtest.CheckEventField("diskio.write.ops", "float", event, t)
-		mtest.CheckEventField("status.check_failed", "int", event, t)
-		mtest.CheckEventField("status.check_failed_system", "int", event, t)
-		mtest.CheckEventField("status.check_failed_instance", "int", event, t)
-	}
+	mbtest.TestMetricsetFieldsDocumented(t, metricSet, events)
 }
 
 func TestData(t *testing.T) {

x-pack/metricbeat/module/aws/terraform.tf

@@ -46,3 +46,23 @@ resource "aws_s3_bucket_object" "test" {
   bucket  = aws_s3_bucket.test.id
   content = "something"
 }
+
+resource "aws_instance" "test" {
+  ami           = data.aws_ami.latest-amzn.id
+  monitoring    = true
+  instance_type = "t1.micro"
+  tags = {
+    Name = "metricbeat-test"
+  }
+}
+
+data "aws_ami" "latest-amzn" {
+  most_recent = true
+  owners      = ["amazon"]
+  filter {
+    name   = "name"
+    values = [
+      "amzn2-ami-hvm-*",
+    ]
+  }
+}