Cherry-pick #17319 to 7.x: Move Metricbeat azure module to GA and add…

… final tests (#17353) * Move Metricbeat azure module to GA and add final tests (#17319) * temp * add tests and move to GA * changelog * fix (cherry picked from commit 2bb6fc9) * fix changelog
elastic · Mar 31, 2020 · 070c380 · 070c380
1 parent b129cb6
commit 070c380
Show file tree

Hide file tree

Showing 31 changed files with 99 additions and 40 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -356,6 +356,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Storage metricsets to GCP module {pull}15598[15598]
 - Release vsphere module as GA. {issue}15798[15798] {pull}17119[17119]
 - Add PubSub metricset to Google Cloud Platform module {pull}15536[15536]
+- Add final tests and move label to GA for the azure module in metricbeat. {pull}17319[17319]
 
 *Packetbeat*
 

diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc
@@ -6,8 +6,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [role="xpack"]
 == azure module
 
-beta[]
-
 This is the azure module.
 
 The Azure Monitor feature collects and aggregates logs and metrics from a variety of sources into a common data platform where it can be used for analysis, visualization, and alerting.

diff --git a/metricbeat/docs/modules/azure/compute_vm.asciidoc b/metricbeat/docs/modules/azure/compute_vm.asciidoc
@@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-metricset-azure-compute_vm]]
 === azure compute_vm metricset
 
-beta[]
-
 include::../../../../x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc[]
 
 

diff --git a/metricbeat/docs/modules/azure/compute_vm_scaleset.asciidoc b/metricbeat/docs/modules/azure/compute_vm_scaleset.asciidoc
@@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-metricset-azure-compute_vm_scaleset]]
 === azure compute_vm_scaleset metricset
 
-beta[]
-
 include::../../../../x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/docs.asciidoc[]
 
 

diff --git a/metricbeat/docs/modules/azure/container_instance.asciidoc b/metricbeat/docs/modules/azure/container_instance.asciidoc
@@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-metricset-azure-container_instance]]
 === azure container_instance metricset
 
-beta[]
-
 include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc[]
 
 This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.

diff --git a/metricbeat/docs/modules/azure/container_registry.asciidoc b/metricbeat/docs/modules/azure/container_registry.asciidoc
@@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-metricset-azure-container_registry]]
 === azure container_registry metricset
 
-beta[]
-
 include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc[]
 
 This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.

diff --git a/metricbeat/docs/modules/azure/container_service.asciidoc b/metricbeat/docs/modules/azure/container_service.asciidoc
@@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-metricset-azure-container_service]]
 === azure container_service metricset
 
-beta[]
-
 include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc[]
 
 This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.

diff --git a/metricbeat/docs/modules/azure/database_account.asciidoc b/metricbeat/docs/modules/azure/database_account.asciidoc
@@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-metricset-azure-database_account]]
 === azure database_account metricset
 
-beta[]
-
 include::../../../../x-pack/metricbeat/module/azure/database_account/_meta/docs.asciidoc[]
 
 This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.

diff --git a/metricbeat/docs/modules/azure/monitor.asciidoc b/metricbeat/docs/modules/azure/monitor.asciidoc
@@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-metricset-azure-monitor]]
 === azure monitor metricset
 
-beta[]
-
 include::../../../../x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc[]
 
 This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.

diff --git a/metricbeat/docs/modules/azure/storage.asciidoc b/metricbeat/docs/modules/azure/storage.asciidoc
@@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go
 [[metricbeat-metricset-azure-storage]]
 === azure storage metricset
 
-beta[]
-
 include::../../../../x-pack/metricbeat/module/azure/storage/_meta/docs.asciidoc[]
 
 

diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -32,15 +32,15 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-aws-transitgateway,transitgateway>> beta[]  
 |<<metricbeat-metricset-aws-usage,usage>> beta[]  
 |<<metricbeat-metricset-aws-vpn,vpn>> beta[]  
-|<<metricbeat-module-azure,azure>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.8+| .8+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
-|<<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>> beta[]  
-|<<metricbeat-metricset-azure-container_instance,container_instance>> beta[]  
-|<<metricbeat-metricset-azure-container_registry,container_registry>> beta[]  
-|<<metricbeat-metricset-azure-container_service,container_service>> beta[]  
-|<<metricbeat-metricset-azure-database_account,database_account>> beta[]  
-|<<metricbeat-metricset-azure-monitor,monitor>> beta[]  
-|<<metricbeat-metricset-azure-storage,storage>> beta[]  
+|<<metricbeat-module-azure,azure>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
+.8+| .8+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>>   
+|<<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>>   
+|<<metricbeat-metricset-azure-container_instance,container_instance>>   
+|<<metricbeat-metricset-azure-container_registry,container_registry>>   
+|<<metricbeat-metricset-azure-container_service,container_service>>   
+|<<metricbeat-metricset-azure-database_account,database_account>>   
+|<<metricbeat-metricset-azure-monitor,monitor>>   
+|<<metricbeat-metricset-azure-storage,storage>>   
 |<<metricbeat-module-beat,Beat>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .2+| .2+|  |<<metricbeat-metricset-beat-state,state>>   
 |<<metricbeat-metricset-beat-stats,stats>>   

diff --git a/x-pack/metricbeat/module/azure/_meta/fields.yml b/x-pack/metricbeat/module/azure/_meta/fields.yml
@@ -2,7 +2,7 @@
   title: "azure"
   description: >
     azure module
-  release: beta
+  release: ga
   fields:
     - name: azure
       type: group

diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go
@@ -8,8 +8,6 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
-
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/metricbeat/mb"
@@ -82,7 +80,6 @@ type MetricSet struct {
 // NewMetricSet will instantiate a new azure metricset
 func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 	metricsetName := base.Name()
-	cfgwarn.Beta("The azure %s metricset is beta.", metricsetName)
 	var config Config
 	err := base.Module().UnpackConfig(&config)
 	if err != nil {

diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/fields.yml b/x-pack/metricbeat/module/azure/compute_vm/_meta/fields.yml
@@ -1,5 +1,5 @@
 - name: compute_vm.*.*
-  release: beta
+  release: ga
   type: object
   object_type: float
   object_type_mapping_type: "*"

diff --git a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go
@@ -25,6 +25,8 @@ func TestFetchMetricset(t *testing.T) {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}
 	assert.NotEmpty(t, events)
+	test.TestFieldsDocumentation(t, events)
+
 }
 
 func TestData(t *testing.T) {

diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/fields.yml b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/fields.yml
@@ -1,5 +1,5 @@
 - name: compute_vm_scaleset.*.*
-  release: beta
+  release: ga
   type: object
   object_type: float
   object_type_mapping_type: "*"

diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go
@@ -25,6 +25,7 @@ func TestFetchMetricset(t *testing.T) {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}
 	assert.NotEmpty(t, events)
+	test.TestFieldsDocumentation(t, events)
 }
 
 func TestData(t *testing.T) {

diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml
@@ -1,5 +1,5 @@
 - name: container_instance.*.*
-  release: beta
+  release: ga
   type: object
   object_type: float
   object_type_mapping_type: "*"

diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go
@@ -28,6 +28,7 @@ func TestFetchMetricset(t *testing.T) {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}
 	assert.NotEmpty(t, events)
+	test.TestFieldsDocumentation(t, events)
 }
 
 func TestData(t *testing.T) {

diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml
@@ -1,5 +1,5 @@
 - name: container_registry.*.*
-  release: beta
+  release: ga
   type: object
   object_type: float
   object_type_mapping_type: "*"

diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go
@@ -28,6 +28,7 @@ func TestFetchMetricset(t *testing.T) {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}
 	assert.NotEmpty(t, events)
+	test.TestFieldsDocumentation(t, events)
 }
 
 func TestData(t *testing.T) {

diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_service/_meta/fields.yml
@@ -1,5 +1,5 @@
 - name: container_service.*.*
-  release: beta
+  release: ga
   type: object
   object_type: float
   object_type_mapping_type: "*"

diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go
@@ -28,6 +28,7 @@ func TestFetchMetricset(t *testing.T) {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}
 	assert.NotEmpty(t, events)
+	test.TestFieldsDocumentation(t, events)
 }
 
 func TestData(t *testing.T) {

diff --git a/x-pack/metricbeat/module/azure/database_account/_meta/fields.yml b/x-pack/metricbeat/module/azure/database_account/_meta/fields.yml
@@ -1,5 +1,5 @@
 - name: database_account.*.*
-  release: beta
+  release: ga
   type: object
   object_type: float
   object_type_mapping_type: "*"

diff --git a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go
@@ -25,6 +25,7 @@ func TestFetchMetricset(t *testing.T) {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}
 	assert.NotEmpty(t, events)
+	test.TestFieldsDocumentation(t, events)
 }
 
 func TestData(t *testing.T) {

diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go
diff --git a/x-pack/metricbeat/module/azure/monitor/_meta/fields.yml b/x-pack/metricbeat/module/azure/monitor/_meta/fields.yml
@@ -1,6 +1,6 @@
 - name: monitor
   type: group
-  release: beta
+  release: ga
   description: >
     monitor
   fields:

diff --git a/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go b/x-pack/metricbeat/module/azure/monitor/monitor_integration_test.go
@@ -25,6 +25,7 @@ func TestFetchMetricset(t *testing.T) {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}
 	assert.NotEmpty(t, events)
+	test.TestFieldsDocumentation(t, events)
 }
 
 func TestData(t *testing.T) {

diff --git a/x-pack/metricbeat/module/azure/storage/_meta/fields.yml b/x-pack/metricbeat/module/azure/storage/_meta/fields.yml
@@ -1,5 +1,5 @@
 - name: storage.*.*
-  release: beta
+  release: ga
   type: object
   object_type: float
   object_type_mapping_type: "*"

diff --git a/x-pack/metricbeat/module/azure/storage/storage_integration_test.go b/x-pack/metricbeat/module/azure/storage/storage_integration_test.go
@@ -25,6 +25,7 @@ func TestFetchMetricset(t *testing.T) {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
 	}
 	assert.NotEmpty(t, events)
+	test.TestFieldsDocumentation(t, events)
 }
 
 func TestData(t *testing.T) {

diff --git a/x-pack/metricbeat/module/azure/test/integration.go b/x-pack/metricbeat/module/azure/test/integration.go
@@ -5,8 +5,13 @@
 package test
 
 import (
+	"errors"
 	"os"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
 // GetConfig function gets azure credentials for integration tests.
@@ -40,3 +45,68 @@ func GetConfig(t *testing.T, metricSetName string) map[string]interface{} {
 		"subscription_id":       subId,
 	}
 }
+
+// TestFieldsDocumentation func checks if all the documented fields have the expected type
+func TestFieldsDocumentation(t *testing.T, events []mb.Event) {
+	for _, event := range events {
+		// RootField
+		checkIsDocumented("service.name", "string", event, t)
+		checkIsDocumented("cloud.provider", "string", event, t)
+		checkIsDocumented("cloud.region", "string", event, t)
+		checkIsDocumented("cloud.instance.name", "string", event, t)
+		checkIsDocumented("cloud.instance.id", "string", event, t)
+
+		// MetricSetField
+		checkIsDocumented("azure.timegrain", "string", event, t)
+		checkIsDocumented("azure.subscription_id", "string", event, t)
+		checkIsDocumented("azure.namespace", "string", event, t)
+		checkIsDocumented("azure.resource.type", "string", event, t)
+		checkIsDocumented("azure.resource.group", "string", event, t)
+	}
+}
+
+// checkIsDocumented function checks a given field type and compares it with the expected type for integration tests.
+// this implementation is only temporary, will be replaced by issue https://github.com/elastic/beats/issues/17315
+func checkIsDocumented(metricName string, expectedType string, event mb.Event, t *testing.T) {
+	t.Helper()
+
+	ok1, err1 := event.MetricSetFields.HasKey(metricName)
+	ok2, err2 := event.RootFields.HasKey(metricName)
+	if ok1 || ok2 {
+		if ok1 {
+			assert.NoError(t, err1)
+			metricValue, err := event.MetricSetFields.GetValue(metricName)
+			assert.NoError(t, err)
+			err = compareType(metricValue, expectedType, metricName)
+			assert.NoError(t, err)
+			t.Log("Succeed: Field " + metricName + " matches type " + expectedType)
+		} else if ok2 {
+			assert.NoError(t, err2)
+			rootValue, err := event.RootFields.GetValue(metricName)
+			assert.NoError(t, err)
+			err = compareType(rootValue, expectedType, metricName)
+			assert.NoError(t, err)
+			t.Log("Succeed: Field " + metricName + " matches type " + expectedType)
+		}
+	} else {
+		t.Log("Field " + metricName + " does not exist in metric set fields")
+	}
+}
+
+func compareType(metricValue interface{}, expectedType string, metricName string) (err error) {
+	switch metricValue.(type) {
+	case float64:
+		if expectedType != "float" {
+			err = errors.New("Failed: Field " + metricName + " is not in type " + expectedType)
+		}
+	case string:
+		if expectedType != "string" {
+			err = errors.New("Failed: Field " + metricName + " is not in type " + expectedType)
+		}
+	case int64:
+		if expectedType != "int" {
+			err = errors.New("Failed: Field " + metricName + " is not in type " + expectedType)
+		}
+	}
+	return
+}