Move the compute_vm_scalset to a light metricset and map the cloud metadata

CHANGELOG.next.asciidoc

@@ -674,6 +674,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Request prometheus endpoints to be gzipped by default {pull}20766[20766]
 - Release all kubernetes `state` metricsets as GA {pull}20901[20901]
 - Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738]
+- Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985]
 - Sanitize `event.host`. {pull}21022[21022]
 
 *Packetbeat*

metricbeat/docs/modules/azure/compute_vm.asciidoc

@@ -8,7 +8,6 @@ This file is generated! See scripts/mage/docs_collector.go
 
 include::../../../../x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc[]
 
-This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
 
 ==== Fields
 

x-pack/metricbeat/module/azure/add_metadata.go

@@ -35,16 +35,16 @@ func addHostMetadata(event *mb.Event, metricList common.MapStr) {
 	}
 }
 
-func addCloudVMMetadata(event *mb.Event, resource Resource) {
-	event.RootFields.Put("cloud.instance.name", resource.Name)
-	event.RootFields.Put("host.name", resource.Name)
-	if resource.Vm != (VmResource{}) {
-		if resource.Vm.Id != "" {
-			event.RootFields.Put("cloud.instance.id", resource.Vm.Id)
-			event.RootFields.Put("host.id", resource.Vm.Id)
-		}
-		if resource.Vm.Size != "" {
-			event.RootFields.Put("cloud.machine.type", resource.Vm.Size)
-		}
+func addCloudVMMetadata(event *mb.Event, vm VmResource) {
+	if vm.Name != "" {
+		event.RootFields.Put("cloud.instance.name", vm.Name)
+		event.RootFields.Put("host.name", vm.Name)
+	}
+	if vm.Id != "" {
+		event.RootFields.Put("cloud.instance.id", vm.Id)
+		event.RootFields.Put("host.id", vm.Id)
+	}
+	if vm.Size != "" {
+		event.RootFields.Put("cloud.machine.type", vm.Size)
 	}
 }

x-pack/metricbeat/module/azure/client.go

@@ -185,39 +185,72 @@ func (client *Client) MapMetricByPrimaryAggregation(metrics []insights.MetricDef
 	return clientMetrics
 }
 
-// GetResourceForData will retrieve resource details for the selected metric configuration
-func (client *Client) GetResourceForData(resourceId string) Resource {
-	for i, res := range client.Resources {
-		if res.Id == resourceId {
-			var vmSize string
-			var vmId string
-			if client.Config.AddCloudMetadata && res.Vm == (VmResource{}) {
-				expandedResource, err := client.AzureMonitorService.GetResourceDefinitionById(res.Id)
-				if err != nil {
-					client.Log.Error(err, "could not retrieve the resource details by resource ID %s", res.Id)
-					return Resource{}
+// GetVMForMetaData func will retrieve the vm details in order to fill in the cloud metadata and also update the client resources
+func (client *Client) GetVMForMetaData(resource *Resource, metricValues []MetricValue) VmResource {
+	var vm VmResource
+	resourceName := resource.Name
+	resourceId := resource.Id
+	// check first if this is a vm scaleset and the instance name is stored in the dimension value
+	if dimension, ok := getDimension("VMName", metricValues[0].dimensions); ok {
+		instanceId := getInstanceId(dimension.Value)
+		if instanceId != "" {
+			resourceId += fmt.Sprintf("/virtualMachines/%s", instanceId)
+			resourceName = dimension.Value
+		}
+	}
+	// if vm has been already added to the resource then it should be returned
+	if existingVM, ok := getVM(resourceName, resource.Vms); ok {
+		return existingVM
+	}
+	// an additional call is necessary in order to retrieve the vm specific details
+	expandedResource, err := client.AzureMonitorService.GetResourceDefinitionById(resourceId)
+	if err != nil {
+		client.Log.Error(err, "could not retrieve the resource details by resource ID %s", resourceId)
+		return VmResource{}
+	}
+	vm.Name = *expandedResource.Name
+	if expandedResource.Properties != nil {
+		if properties, ok := expandedResource.Properties.(map[string]interface{}); ok {
+			if hardware, ok := properties["hardwareProfile"]; ok {
+				if vmSz, ok := hardware.(map[string]interface{})["vmSize"]; ok {
+					vm.Size = vmSz.(string)
 				}
-				if expandedResource.Properties != nil {
-					if properties, ok := expandedResource.Properties.(map[string]interface{}); ok {
-						if hardware, ok := properties["hardwareProfile"]; ok {
-							if vmSz, ok := hardware.(map[string]interface{})["vmSize"]; ok {
-								vmSize = vmSz.(string)
-							}
-							if vmID, ok := properties["vmId"]; ok {
-								vmId = vmID.(string)
-							}
-						}
-					}
+				if vmID, ok := properties["vmId"]; ok {
+					vm.Id = vmID.(string)
 				}
-				client.Resources[i].Vm = VmResource{Size: vmSize, Id: vmId}
-				return client.Resources[i]
 			}
+		}
+	}
+	if len(vm.Size) == 0 && expandedResource.Sku != nil && expandedResource.Sku.Name != nil {
+		vm.Size = *expandedResource.Sku.Name
+	}
+	// the client resource and selected resources are being updated in order to avoid additional calls
+	client.AddVmToResource(resource.Id, vm)
+	resource.Vms = append(resource.Vms, vm)
+	return vm
+}
+
+// GetResourceForMetaData will retrieve resource details for the selected metric configuration
+func (client *Client) GetResourceForMetaData(grouped Metric) Resource {
+	for _, res := range client.Resources {
+		if res.Id == grouped.ResourceId {
 			return res
 		}
 	}
 	return Resource{}
 }
 
+// AddVmToResource will add the vm details to the resource
+func (client *Client) AddVmToResource(resourceId string, vm VmResource) {
+	if len(vm.Id) > 0 && len(vm.Name) > 0 {
+		for i, res := range client.Resources {
+			if res.Id == resourceId {
+				client.Resources[i].Vms = append(client.Resources[i].Vms, vm)
+			}
+		}
+	}
+}
+
 // NewMockClient instantiates a new client with the mock azure service
 func NewMockClient() *Client {
 	azureMockService := new(MockService)

x-pack/metricbeat/module/azure/client_utils.go

@@ -6,6 +6,7 @@ package azure
 
 import (
 	"reflect"
+	"regexp"
 	"strings"
 	"time"
 
@@ -15,6 +16,8 @@ import (
 // DefaultTimeGrain is set as default timegrain for the azure metrics
 const DefaultTimeGrain = "PT5M"
 
+var instanceIdRegex = regexp.MustCompile(`.*?(\d+)$`)
+
 // mapMetricValues should map the metric values
 func mapMetricValues(metrics []insights.Metric, previousMetrics []MetricValue, startTime time.Time, endTime time.Time) []MetricValue {
 	var currentMetrics []MetricValue
@@ -167,21 +170,41 @@ func groupMetricsByResource(metrics []Metric) map[string][]Metric {
 	return grouped
 }
 
-// ContainsDimension will check if the dimension value is found in the list
-func ContainsDimension(dimension string, dimensions []insights.LocalizableString) bool {
+// getDimension will check if the dimension value is found in the list
+func getDimension(dimension string, dimensions []Dimension) (Dimension, bool) {
 	for _, dim := range dimensions {
-		if *dim.Value == dimension {
-			return true
+		if strings.ToLower(dim.Name) == strings.ToLower(dimension) {
+			return dim, true
 		}
 	}
-	return false
+	return Dimension{}, false
 }
 
-func containsResource(resId string, resources []Resource) bool {
+func containsResource(resourceId string, resources []Resource) bool {
 	for _, res := range resources {
-		if res.Id == resId {
+		if res.Id == resourceId {
 			return true
 		}
 	}
 	return false
 }
+
+func getInstanceId(dimensionValue string) string {
+	matches := instanceIdRegex.FindStringSubmatch(dimensionValue)
+	if len(matches) == 2 {
+		return matches[1]
+	}
+	return ""
+}
+
+func getVM(vmName string, vms []VmResource) (VmResource, bool) {
+	if len(vms) == 0 {
+		return VmResource{}, false
+	}
+	for _, vm := range vms {
+		if vm.Name == vmName {
+			return vm, true
+		}
+	}
+	return VmResource{}, false
+}

x-pack/metricbeat/module/azure/client_utils_test.go

@@ -131,28 +131,94 @@ func TestCompareMetricValues(t *testing.T) {
 	assert.True(t, result)
 }
 
-func TestContainsDimension(t *testing.T) {
+func TestGetDimension(t *testing.T) {
 	dimension := "VMName"
 	dim1 := "SlotID"
 	dim2 := "VNU"
 	dim3 := "VMName"
-	dimensionList := []insights.LocalizableString{
+	dimensionList := []Dimension{
 		{
-			Value:          &dim1,
-			LocalizedValue: &dim1,
+			Name:  dim1,
+			Value: dim1,
 		},
 		{
-			Value:          &dim2,
-			LocalizedValue: &dim2,
+			Name:  dim2,
+			Value: dim2,
 		},
 		{
-			Value:          &dim3,
-			LocalizedValue: &dim3,
+			Name:  dim3,
+			Value: dim3,
 		},
 	}
-	result := ContainsDimension(dimension, dimensionList)
-	assert.True(t, result)
+	result, ok := getDimension(dimension, dimensionList)
+	assert.True(t, ok)
+	assert.Equal(t, result.Name, dim3)
+	assert.Equal(t, result.Value, dim3)
 	dimension = "VirtualMachine"
-	result = ContainsDimension(dimension, dimensionList)
-	assert.False(t, result)
+	result, ok = getDimension(dimension, dimensionList)
+	assert.False(t, ok)
+	assert.Equal(t, result.Name, "")
+	assert.Equal(t, result.Value, "")
+}
+
+func TestContainsResource(t *testing.T) {
+	resourceId := "resId"
+	resourceList := []Resource{
+		{
+			Name: "resource name",
+			Id:   "resId",
+		},
+		{
+			Name: "resource name1",
+			Id:   "resId1",
+		},
+		{
+			Name: "resource name2",
+			Id:   "resId2",
+		},
+	}
+	ok := containsResource(resourceId, resourceList)
+	assert.True(t, ok)
+	resourceId = "ressId"
+	ok = containsResource(resourceId, resourceList)
+	assert.False(t, ok)
+}
+
+func TestGetVM(t *testing.T) {
+	vmName := "resource name1"
+	vmResourceList := []VmResource{
+		{
+			Name: "resource name",
+			Id:   "resId",
+		},
+		{
+			Name: "resource name1",
+			Id:   "resId1",
+		},
+		{
+			Name: "resource name2",
+			Id:   "resId2",
+		},
+	}
+	vm, ok := getVM(vmName, vmResourceList)
+	assert.True(t, ok)
+	assert.Equal(t, vm.Name, vmName)
+	assert.Equal(t, vm.Id, "resId1")
+	vmName = "resource name3"
+	vm, ok = getVM(vmName, vmResourceList)
+	assert.False(t, ok)
+	assert.Equal(t, vm.Name, "")
+	assert.Equal(t, vm.Id, "")
+}
+
+func TestGetInstanceId(t *testing.T) {
+	dimensionValue := "sfjsfjghhbsjsjskjkf"
+	result := getInstanceId(dimensionValue)
+	assert.Empty(t, result)
+	dimensionValue = "fjsfhfhsjhjsfs_34"
+	result = getInstanceId(dimensionValue)
+	assert.Equal(t, result, "34")
+	dimensionValue = "fjsfhfhsjhjsfs_34sjsjfhsfsjjsjf_242"
+	result = getInstanceId(dimensionValue)
+	assert.Equal(t, result, "242")
 }

x-pack/metricbeat/module/azure/compute_vm/manifest.yml

@@ -1,4 +1,4 @@
-default: true
+default: false
 input:
   module: azure
   metricset: monitor