diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2499d2fd866..c3477f3b6e7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -674,6 +674,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Request prometheus endpoints to be gzipped by default {pull}20766[20766] - Release all kubernetes `state` metricsets as GA {pull}20901[20901] - Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738] +- Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985] - Sanitize `event.host`. {pull}21022[21022] *Packetbeat* diff --git a/metricbeat/docs/modules/azure/compute_vm.asciidoc b/metricbeat/docs/modules/azure/compute_vm.asciidoc index 8715cf5c58e..fdac6f7d06a 100644 --- a/metricbeat/docs/modules/azure/compute_vm.asciidoc +++ b/metricbeat/docs/modules/azure/compute_vm.asciidoc @@ -8,7 +8,6 @@ This file is generated! See scripts/mage/docs_collector.go include::../../../../x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc[] -This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index ea6ab5697b0..6ff00da3b3c 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -21,7 +21,6 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/app_insights" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/billing" - _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/compute_vm_scaleset" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/storage" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/cloudfoundry" diff --git a/x-pack/metricbeat/module/azure/add_metadata.go b/x-pack/metricbeat/module/azure/add_metadata.go index c8621e77692..ba8f35c7db6 100644 --- a/x-pack/metricbeat/module/azure/add_metadata.go +++ b/x-pack/metricbeat/module/azure/add_metadata.go @@ -35,16 +35,16 @@ func addHostMetadata(event *mb.Event, metricList common.MapStr) { } } -func addCloudVMMetadata(event *mb.Event, resource Resource) { - event.RootFields.Put("cloud.instance.name", resource.Name) - event.RootFields.Put("host.name", resource.Name) - if resource.Vm != (VmResource{}) { - if resource.Vm.Id != "" { - event.RootFields.Put("cloud.instance.id", resource.Vm.Id) - event.RootFields.Put("host.id", resource.Vm.Id) - } - if resource.Vm.Size != "" { - event.RootFields.Put("cloud.machine.type", resource.Vm.Size) - } +func addCloudVMMetadata(event *mb.Event, vm VmResource) { + if vm.Name != "" { + event.RootFields.Put("cloud.instance.name", vm.Name) + event.RootFields.Put("host.name", vm.Name) + } + if vm.Id != "" { + event.RootFields.Put("cloud.instance.id", vm.Id) + event.RootFields.Put("host.id", vm.Id) + } + if vm.Size != "" { + event.RootFields.Put("cloud.machine.type", vm.Size) } } diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 5bfbbacca67..e488fab98b6 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -185,39 +185,72 @@ func (client *Client) MapMetricByPrimaryAggregation(metrics []insights.MetricDef return clientMetrics } -// GetResourceForData will retrieve resource details for the selected metric configuration -func (client *Client) GetResourceForData(resourceId string) Resource { - for i, res := range client.Resources { - if res.Id == resourceId { - var vmSize string - var vmId string - if client.Config.AddCloudMetadata && res.Vm == (VmResource{}) { - expandedResource, err := client.AzureMonitorService.GetResourceDefinitionById(res.Id) - if err != nil { - client.Log.Error(err, "could not retrieve the resource details by resource ID %s", res.Id) - return Resource{} +// GetVMForMetaData func will retrieve the vm details in order to fill in the cloud metadata and also update the client resources +func (client *Client) GetVMForMetaData(resource *Resource, metricValues []MetricValue) VmResource { + var vm VmResource + resourceName := resource.Name + resourceId := resource.Id + // check first if this is a vm scaleset and the instance name is stored in the dimension value + if dimension, ok := getDimension("VMName", metricValues[0].dimensions); ok { + instanceId := getInstanceId(dimension.Value) + if instanceId != "" { + resourceId += fmt.Sprintf("/virtualMachines/%s", instanceId) + resourceName = dimension.Value + } + } + // if vm has been already added to the resource then it should be returned + if existingVM, ok := getVM(resourceName, resource.Vms); ok { + return existingVM + } + // an additional call is necessary in order to retrieve the vm specific details + expandedResource, err := client.AzureMonitorService.GetResourceDefinitionById(resourceId) + if err != nil { + client.Log.Error(err, "could not retrieve the resource details by resource ID %s", resourceId) + return VmResource{} + } + vm.Name = *expandedResource.Name + if expandedResource.Properties != nil { + if properties, ok := expandedResource.Properties.(map[string]interface{}); ok { + if hardware, ok := properties["hardwareProfile"]; ok { + if vmSz, ok := hardware.(map[string]interface{})["vmSize"]; ok { + vm.Size = vmSz.(string) } - if expandedResource.Properties != nil { - if properties, ok := expandedResource.Properties.(map[string]interface{}); ok { - if hardware, ok := properties["hardwareProfile"]; ok { - if vmSz, ok := hardware.(map[string]interface{})["vmSize"]; ok { - vmSize = vmSz.(string) - } - if vmID, ok := properties["vmId"]; ok { - vmId = vmID.(string) - } - } - } + if vmID, ok := properties["vmId"]; ok { + vm.Id = vmID.(string) } - client.Resources[i].Vm = VmResource{Size: vmSize, Id: vmId} - return client.Resources[i] } + } + } + if len(vm.Size) == 0 && expandedResource.Sku != nil && expandedResource.Sku.Name != nil { + vm.Size = *expandedResource.Sku.Name + } + // the client resource and selected resources are being updated in order to avoid additional calls + client.AddVmToResource(resource.Id, vm) + resource.Vms = append(resource.Vms, vm) + return vm +} + +// GetResourceForMetaData will retrieve resource details for the selected metric configuration +func (client *Client) GetResourceForMetaData(grouped Metric) Resource { + for _, res := range client.Resources { + if res.Id == grouped.ResourceId { return res } } return Resource{} } +// AddVmToResource will add the vm details to the resource +func (client *Client) AddVmToResource(resourceId string, vm VmResource) { + if len(vm.Id) > 0 && len(vm.Name) > 0 { + for i, res := range client.Resources { + if res.Id == resourceId { + client.Resources[i].Vms = append(client.Resources[i].Vms, vm) + } + } + } +} + // NewMockClient instantiates a new client with the mock azure service func NewMockClient() *Client { azureMockService := new(MockService) diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index c0cd02e589f..abfccfa75ec 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -6,6 +6,7 @@ package azure import ( "reflect" + "regexp" "strings" "time" @@ -15,6 +16,8 @@ import ( // DefaultTimeGrain is set as default timegrain for the azure metrics const DefaultTimeGrain = "PT5M" +var instanceIdRegex = regexp.MustCompile(`.*?(\d+)$`) + // mapMetricValues should map the metric values func mapMetricValues(metrics []insights.Metric, previousMetrics []MetricValue, startTime time.Time, endTime time.Time) []MetricValue { var currentMetrics []MetricValue @@ -167,21 +170,41 @@ func groupMetricsByResource(metrics []Metric) map[string][]Metric { return grouped } -// ContainsDimension will check if the dimension value is found in the list -func ContainsDimension(dimension string, dimensions []insights.LocalizableString) bool { +// getDimension will check if the dimension value is found in the list +func getDimension(dimension string, dimensions []Dimension) (Dimension, bool) { for _, dim := range dimensions { - if *dim.Value == dimension { - return true + if strings.ToLower(dim.Name) == strings.ToLower(dimension) { + return dim, true } } - return false + return Dimension{}, false } -func containsResource(resId string, resources []Resource) bool { +func containsResource(resourceId string, resources []Resource) bool { for _, res := range resources { - if res.Id == resId { + if res.Id == resourceId { return true } } return false } + +func getInstanceId(dimensionValue string) string { + matches := instanceIdRegex.FindStringSubmatch(dimensionValue) + if len(matches) == 2 { + return matches[1] + } + return "" +} + +func getVM(vmName string, vms []VmResource) (VmResource, bool) { + if len(vms) == 0 { + return VmResource{}, false + } + for _, vm := range vms { + if vm.Name == vmName { + return vm, true + } + } + return VmResource{}, false +} diff --git a/x-pack/metricbeat/module/azure/client_utils_test.go b/x-pack/metricbeat/module/azure/client_utils_test.go index ffa09d4faea..37528540f70 100644 --- a/x-pack/metricbeat/module/azure/client_utils_test.go +++ b/x-pack/metricbeat/module/azure/client_utils_test.go @@ -131,28 +131,94 @@ func TestCompareMetricValues(t *testing.T) { assert.True(t, result) } -func TestContainsDimension(t *testing.T) { +func TestGetDimension(t *testing.T) { dimension := "VMName" dim1 := "SlotID" dim2 := "VNU" dim3 := "VMName" - dimensionList := []insights.LocalizableString{ + dimensionList := []Dimension{ { - Value: &dim1, - LocalizedValue: &dim1, + Name: dim1, + Value: dim1, }, { - Value: &dim2, - LocalizedValue: &dim2, + Name: dim2, + Value: dim2, }, { - Value: &dim3, - LocalizedValue: &dim3, + Name: dim3, + Value: dim3, }, } - result := ContainsDimension(dimension, dimensionList) - assert.True(t, result) + result, ok := getDimension(dimension, dimensionList) + assert.True(t, ok) + assert.Equal(t, result.Name, dim3) + assert.Equal(t, result.Value, dim3) dimension = "VirtualMachine" - result = ContainsDimension(dimension, dimensionList) - assert.False(t, result) + result, ok = getDimension(dimension, dimensionList) + assert.False(t, ok) + assert.Equal(t, result.Name, "") + assert.Equal(t, result.Value, "") +} + +func TestContainsResource(t *testing.T) { + resourceId := "resId" + resourceList := []Resource{ + { + Name: "resource name", + Id: "resId", + }, + { + Name: "resource name1", + Id: "resId1", + }, + { + Name: "resource name2", + Id: "resId2", + }, + } + ok := containsResource(resourceId, resourceList) + assert.True(t, ok) + resourceId = "ressId" + ok = containsResource(resourceId, resourceList) + assert.False(t, ok) +} + +func TestGetVM(t *testing.T) { + vmName := "resource name1" + vmResourceList := []VmResource{ + { + Name: "resource name", + Id: "resId", + }, + { + Name: "resource name1", + Id: "resId1", + }, + { + Name: "resource name2", + Id: "resId2", + }, + } + vm, ok := getVM(vmName, vmResourceList) + assert.True(t, ok) + assert.Equal(t, vm.Name, vmName) + assert.Equal(t, vm.Id, "resId1") + vmName = "resource name3" + vm, ok = getVM(vmName, vmResourceList) + assert.False(t, ok) + assert.Equal(t, vm.Name, "") + assert.Equal(t, vm.Id, "") +} + +func TestGetInstanceId(t *testing.T) { + dimensionValue := "sfjsfjghhbsjsjskjkf" + result := getInstanceId(dimensionValue) + assert.Empty(t, result) + dimensionValue = "fjsfhfhsjhjsfs_34" + result = getInstanceId(dimensionValue) + assert.Equal(t, result, "34") + dimensionValue = "fjsfhfhsjhjsfs_34sjsjfhsfsjjsjf_242" + result = getInstanceId(dimensionValue) + assert.Equal(t, result, "242") } diff --git a/x-pack/metricbeat/module/azure/compute_vm/manifest.yml b/x-pack/metricbeat/module/azure/compute_vm/manifest.yml index 4cfcad00cc7..95421a31ef2 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/manifest.yml +++ b/x-pack/metricbeat/module/azure/compute_vm/manifest.yml @@ -1,4 +1,4 @@ -default: true +default: false input: module: azure metricset: monitor diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index 6baa47d8e8c..e8f59859d8b 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -1,84 +1,46 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { - "dimensions": { - "vmname": "obslinuxvmss_0" - }, - "metrics": { - "disk_read_bytes": { - "total": 0 - }, - "disk_read_operations_per_sec": { - "avg": 0 - }, - "disk_write_bytes": { - "total": 1032189.54 - }, - "disk_write_operations_per_sec": { - "avg": 0.254 - }, - "inbound_flows": { - "avg": 58.2 - }, - "inbound_flows_maximum_creation_rate": { - "avg": 8.4 - }, - "network_in": { - "total": 253142 - }, - "network_in_total": { - "total": 646125 - }, - "network_out": { - "total": 874827 - }, - "network_out_total": { - "total": 1140051 - }, - "os_disk_bandwidth_consumed_percentage": { - "avg": 0 - }, - "os_disk_iops_consumed_percentage": { + "compute_vm_scaleset": { + "os_per_disk_qd": { "avg": 0 }, - "os_disk_queue_depth": { + "os_per_disk_read_bytes_per_sec": { "avg": 0 }, - "os_disk_read_bytes_per_sec": { + "os_per_disk_read_operations_per_sec": { "avg": 0 }, - "os_disk_read_operations_per_sec": { - "avg": 0 - }, - "os_disk_write_bytes_per_sec": { - "avg": 3440.626 - }, - "os_disk_write_operations_per_sec": { - "avg": 0.508 - }, - "outbound_flows": { - "avg": 58.2 + "os_per_disk_write_bytes_per_sec": { + "avg": 1872.1200000000001 }, - "outbound_flows_maximum_creation_rate": { - "avg": 11.2 - }, - "percentage_cpu": { - "avg": 0.966 + "os_per_disk_write_operations_per_sec": { + "avg": 0.296 } }, "namespace": "Microsoft.Compute/virtualMachineScaleSets", "resource": { - "group": "obs-infrastructure", - "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", - "name": "obslinuxvmss", + "group": "testgroup", + "id": "/subscriptions/70bd6e23-e3er3-4835-6785-db77b8eef364/resourceGroups/testgroup/providers/Microsoft.Compute/virtualMachineScaleSets/vmscaleset", + "name": "vmscaleset", + "tags": { + "environment": "staging", + "role": "allocator" + }, "type": "Microsoft.Compute/virtualMachineScaleSets" }, - "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", + "subscription_id": "70bd6e23-e3er3-4835-6785-db77b8eef364", "timegrain": "PT5M" }, "cloud": { + "instance": { + "name": "vmscaleset" + }, + "machine": { + "type": "Standard_D4s_v3" + }, "provider": "azure", - "region": "westeurope" + "region": "eastus2" }, "event": { "dataset": "azure.compute_vm_scaleset", @@ -86,27 +48,7 @@ "module": "azure" }, "host": { - "cpu": { - "pct": 0.00966 - }, - "disk": { - "read": { - "bytes": 0 - }, - "write": { - "bytes": 1032189.54 - } - }, - "network": { - "in": { - "bytes": 646125, - "packets": 253142 - }, - "out": { - "bytes": 1140051, - "packets": 874827 - } - } + "name": "vmscaleset" }, "metricset": { "name": "compute_vm_scaleset", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go deleted file mode 100644 index ecabff74741..00000000000 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package compute_vm_scaleset - -import ( - "strings" - - "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" -) - -const ( - defaultVMDimension = "VMName" - customVMDimension = "VirtualMachine" - defaultSlotIDDimension = "SlotId" -) - -// mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters -func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { - var metrics []azure.Metric - for _, resource := range resources { - // return resource size - for _, metric := range resourceConfig.Metrics { - metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) - if err != nil { - return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s", *resource.ID, metric.Namespace) - } - if len(*metricDefinitions.Value) == 0 && metric.Namespace != customVMNamespace { - return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) - } - var supportedMetricNames []insights.MetricDefinition - if strings.Contains(strings.Join(metric.Name, " "), "*") { - for _, definition := range *metricDefinitions.Value { - supportedMetricNames = append(supportedMetricNames, definition) - } - } else { - // verify if configured metric names are valid, return log error event for the invalid ones, map only the valid metric names - for _, name := range metric.Name { - for _, metricDefinition := range *metricDefinitions.Value { - if name == *metricDefinition.Name.Value { - supportedMetricNames = append(supportedMetricNames, metricDefinition) - } - } - } - } - if len(supportedMetricNames) == 0 { - continue - } - groupedMetrics := make(map[string][]insights.MetricDefinition) - var vmdim string - if metric.Namespace == defaultVMScalesetNamespace { - vmdim = defaultVMDimension - } else if metric.Namespace == customVMNamespace { - vmdim = customVMDimension - } - for _, metricName := range supportedMetricNames { - if metricName.Dimensions == nil || len(*metricName.Dimensions) == 0 { - groupedMetrics[azure.NoDimension] = append(groupedMetrics[azure.NoDimension], metricName) - } else if azure.ContainsDimension(vmdim, *metricName.Dimensions) { - groupedMetrics[vmdim] = append(groupedMetrics[vmdim], metricName) - } else if azure.ContainsDimension(defaultSlotIDDimension, *metricName.Dimensions) { - groupedMetrics[defaultSlotIDDimension] = append(groupedMetrics[defaultSlotIDDimension], metricName) - } - } - for key, metricGroup := range groupedMetrics { - var metricNameList []string - for _, metricName := range metricGroup { - metricNameList = append(metricNameList, *metricName.Name.Value) - } - var dimensions []azure.Dimension - if key != azure.NoDimension { - dimensions = []azure.Dimension{{Name: key, Value: "*"}} - } - metrics = append(metrics, client.MapMetricByPrimaryAggregation(metricGroup, *resource.ID, "", metric.Namespace, dimensions, azure.DefaultTimeGrain)...) - } - } - } - return metrics, nil -} - -// mapResourceSize func will try to map if existing the resource size, for the vmss it seems that SKU is populated and resource size is mapped in the name -func mapResourceSize(resource resources.GenericResource) string { - if resource.Sku != nil && resource.Sku.Name != nil { - return *resource.Sku.Name - } - return "" -} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go deleted file mode 100644 index 764cef2962b..00000000000 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package compute_vm_scaleset - -import ( - "testing" - - "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" -) - -func MockResource() resources.GenericResource { - id := "123" - name := "resourceName" - location := "resourceLocation" - rType := "resourceType" - skuName := "standard" - sku := resources.Sku{ - Name: &skuName, - } - return resources.GenericResource{ - ID: &id, - Name: &name, - Location: &location, - Type: &rType, - Sku: &sku, - } -} - -func MockMetricDefinitions() *[]insights.MetricDefinition { - metric1 := "TotalRequests" - metric2 := "Capacity" - metric3 := "BytesRead" - defs := []insights.MetricDefinition{ - { - Name: &insights.LocalizableString{Value: &metric1}, - PrimaryAggregationType: insights.Average, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Maximum, insights.Count, insights.Total, insights.Average}, - }, - { - Name: &insights.LocalizableString{Value: &metric2}, - PrimaryAggregationType: insights.Average, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, - }, - { - Name: &insights.LocalizableString{Value: &metric3}, - PrimaryAggregationType: insights.Minimum, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, - }, - } - return &defs -} - -func TestMapMetric(t *testing.T) { - resource := MockResource() - metricDefinitions := insights.MetricDefinitionCollection{ - Value: MockMetricDefinitions(), - } - var emptyList []insights.MetricDefinition - emptyMetricDefinitions := insights.MetricDefinitionCollection{ - Value: &emptyList, - } - metricConfig := azure.MetricConfig{Name: []string{"*"}, Namespace: "namespace"} - var resourceConfig = azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} - client := azure.NewMockClient() - t.Run("return error when the metric metric definition api call returns an error", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, errors.New("invalid resource ID")) - client.AzureMonitorService = m - metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.Error(t, err) - assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace namespace: invalid resource ID") - assert.Equal(t, metric, []azure.Metric(nil)) - m.AssertExpectations(t) - }) - t.Run("return error when no metric definitions were found", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) - client.AzureMonitorService = m - metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.Error(t, err) - assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace namespace.") - assert.Equal(t, metric, []azure.Metric(nil)) - m.AssertExpectations(t) - }) - t.Run("return mapped metrics correctly", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) - client.AzureMonitorService = m - metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.NoError(t, err) - assert.Equal(t, len(metrics), 2) - - assert.Equal(t, metrics[0].ResourceId, "123") - assert.Equal(t, metrics[0].Namespace, "namespace") - assert.Equal(t, metrics[1].ResourceId, "123") - assert.Equal(t, metrics[1].Namespace, "namespace") - assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) - assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) - - //order of elements can be different when running the test - if metrics[0].Aggregations == "Average" { - assert.Equal(t, metrics[0].Names, []string{"TotalRequests", "Capacity"}) - } else { - assert.Equal(t, metrics[0].Names, []string{"BytesRead"}) - assert.Equal(t, metrics[0].Aggregations, "Minimum") - } - m.AssertExpectations(t) - }) -} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset.go deleted file mode 100644 index b2755eeabf0..00000000000 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package compute_vm_scaleset - -import ( - "fmt" - - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" -) - -// init registers the MetricSet with the central registry as soon as the program -// starts. The New function will be called later to instantiate an instance of -// the MetricSet for each host defined in the module's configuration. After the -// MetricSet has been created then Fetch will begin to be called periodically. -func init() { - mb.Registry.MustAddMetricSet("azure", "compute_vm_scaleset", New) -} - -// MetricSet holds any configuration or state information. It must implement -// the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet -// interface methods except for Fetch. -type MetricSet struct { - *azure.MetricSet -} - -const ( - defaultVMScalesetNamespace = "Microsoft.Compute/virtualMachineScaleSets" - customVMNamespace = "Azure.VM.Windows.GuestMetrics" -) - -var memoryMetrics = []string{"Memory\\Commit Limit", "Memory\\Committed Bytes", "Memory\\% Committed Bytes In Use", "Memory\\Available Bytes"} - -// New creates a new instance of the MetricSet. New is responsible for unpacking -// any MetricSet specific configuration options if there are any. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - ms, err := azure.NewMetricSet(base) - if err != nil { - return nil, err - } - // if no options are entered we will retrieve all the vm's from the entire subscription - if len(ms.Client.Config.Resources) == 0 { - ms.Client.Config.Resources = []azure.ResourceConfig{ - { - Query: fmt.Sprintf("resourceType eq '%s'", defaultVMScalesetNamespace), - }, - } - } - for index := range ms.Client.Config.Resources { - // add the default vm scaleset type if groups are defined - if len(ms.Client.Config.Resources[index].Group) > 0 { - ms.Client.Config.Resources[index].Type = defaultVMScalesetNamespace - } - // add the default metrics for each resource option - ms.Client.Config.Resources[index].Metrics = []azure.MetricConfig{ - { - Name: []string{"*"}, - Namespace: defaultVMScalesetNamespace, - }, - { - Name: memoryMetrics, - Namespace: customVMNamespace, - }, - } - } - ms.MapMetrics = mapMetrics - return &MetricSet{ - MetricSet: ms, - }, nil -} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go index 7403203ad12..0a4a8df5e6b 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_integration_test.go @@ -15,6 +15,9 @@ import ( "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_test.go index d82ea953140..2c5c7f04c8f 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/compute_vm_scaleset_test.go @@ -5,69 +5,13 @@ package compute_vm_scaleset import ( - "fmt" - "testing" + "os" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" ) -var ( - missingResourcesConfig = common.MapStr{ - "module": "azure", - "period": "60s", - "metricsets": []string{"compute_vm_scaleset"}, - "client_secret": "unique identifier", - "client_id": "unique identifier", - "subscription_id": "unique identifier", - "tenant_id": "unique identifier", - } - - resourceConfig = common.MapStr{ - "module": "azure", - "period": "60s", - "metricsets": []string{"compute_vm_scaleset"}, - "client_secret": "unique identifier", - "client_id": "unique identifier", - "subscription_id": "unique identifier", - "tenant_id": "unique identifier", - "resources": []common.MapStr{ - { - "resource_id": "test", - "metrics": []map[string]interface{}{ - { - "name": []string{"*"}, - }}, - }}, - } -) - -func TestFetch(t *testing.T) { - c, err := common.NewConfigFrom(missingResourcesConfig) - if err != nil { - t.Fatal(err) - } - module, metricsets, err := mb.NewModule(c, mb.Registry) - assert.NotNil(t, module) - assert.NotNil(t, metricsets) - assert.NoError(t, err) - ms, ok := metricsets[0].(*MetricSet) - assert.Equal(t, len(ms.Client.Config.Resources), 1) - assert.Equal(t, ms.Client.Config.Resources[0].Query, fmt.Sprintf("resourceType eq '%s'", defaultVMScalesetNamespace)) - c, err = common.NewConfigFrom(resourceConfig) - if err != nil { - t.Fatal(err) - } - module, metricsets, err = mb.NewModule(c, mb.Registry) - if err != nil { - t.Fatal(err) - } - assert.NotNil(t, module) - assert.NotNil(t, metricsets) - ms, ok = metricsets[0].(*MetricSet) - require.True(t, ok, "metricset must be MetricSet") - assert.NotNil(t, ms) +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) } diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml b/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml new file mode 100644 index 00000000000..9369a36b79e --- /dev/null +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml @@ -0,0 +1,66 @@ +default: false +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.Compute/virtualMachineScaleSets" + add_cloud_metadata: true + resources: + - resource_group: "" + resource_type: "Microsoft.Compute/virtualMachineScaleSets" + metrics: + - name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"] + namespace: "Microsoft.Compute/virtualMachineScaleSets" + timegrain: "PT5M" + - name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"] + namespace: "Microsoft.Compute/virtualMachineScaleSets" + timegrain: "PT5M" + dimensions: + - name: "SlotId" + value: "*" + - name: ["Network In", "Percentage CPU", "Network Out", "Disk Read Bytes", "Disk Write Bytes", "Disk Read Operations/Sec", "Disk Write Operations/Sec", "Data Disk Read Bytes/sec", "Data Disk Write Bytes/sec", "Network Out Total", + "Network In Total", "VM Uncached IOPS Consumed Percentange", "VM Uncached Bandwidth Consumed Percentange", "VM Cached IOPS Consumed Percentange", "VM Cached Bandwidth Consumed Percentange", "Premium OS Disk Cache Read Miss", + "Premium OS Disk Cache Read Hit", "Premium Data Disk Cache Read Miss", "Data Disk Read Operations/Sec", "Data Disk Write Operations/Sec", "Data Disk Queue Depth", "Data Disk Bandwidth Consumed Percentage", + "Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage", + "OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"] + namespace: "Microsoft.Compute/virtualMachineScaleSets" + timegrain: "PT5M" + dimensions: + - name: "VMName" + value: "*" + - name: ["Memory\\Commit Limit", "Memory\\Committed Bytes", "Memory\\% Committed Bytes In Use", "Memory\\Available Bytes"] + namespace: "Azure.VM.Windows.GuestMetrics" + timegrain: "PT5M" + dimensions: + - name: "VirtualMachine" + value: "*" + ignore_unsupported: true + - resource_id: "" + timegrain: "PT5M" + metrics: + - name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"] + namespace: "Microsoft.Compute/virtualMachineScaleSets" + timegrain: "PT5M" + - name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"] + namespace: "Microsoft.Compute/virtualMachineScaleSets" + timegrain: "PT5M" + dimensions: + - name: "SlotId" + value: "*" + - name: ["Network In", "Percentage CPU", "Network Out", "Disk Read Bytes", "Disk Write Bytes", "Disk Read Operations/Sec", "Disk Write Operations/Sec", "Data Disk Read Bytes/sec", "Data Disk Write Bytes/sec", "Network Out Total", + "Network In Total", "VM Uncached IOPS Consumed Percentange", "VM Uncached Bandwidth Consumed Percentange", "VM Cached IOPS Consumed Percentange", "VM Cached Bandwidth Consumed Percentange", "Premium OS Disk Cache Read Miss", + "Premium OS Disk Cache Read Hit", "Premium Data Disk Cache Read Miss", "Data Disk Read Operations/Sec", "Data Disk Write Operations/Sec", "Data Disk Queue Depth", "Data Disk Bandwidth Consumed Percentage", + "Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage", + "OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"] + namespace: "Microsoft.Compute/virtualMachineScaleSets" + timegrain: "PT5M" + dimensions: + - name: "VMName" + value: "*" + - name: ["Memory\\Commit Limit", "Memory\\Committed Bytes", "Memory\\% Committed Bytes In Use", "Memory\\Available Bytes"] + namespace: "Azure.VM.Windows.GuestMetrics" + timegrain: "PT5M" + dimensions: + - name: "VirtualMachine" + value: "*" + ignore_unsupported: true diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index d0ba18fdc36..bf77f657416 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -58,7 +58,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error // grouping metric values by timestamp and creating events (for each metric the REST api can retrieve multiple metric values for same aggregation but different timeframes) for _, grouped := range groupByDimensions { defaultMetric := grouped[0] - resource := client.GetResourceForData(defaultMetric.ResourceId) + resource := client.GetResourceForMetaData(defaultMetric) groupByTimeMetrics := make(map[time.Time][]MetricValue) for _, metric := range grouped { for _, m := range metric.Values { @@ -68,6 +68,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error for timestamp, groupTimeValues := range groupByTimeMetrics { var event mb.Event var metricList common.MapStr + var vm VmResource // group events by dimension values exists, validDimensions := returnAllDimensions(defaultMetric.Dimensions) if exists { @@ -79,13 +80,18 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error } for _, groupDimValues := range groupByDimensions { event, metricList = createEvent(timestamp, defaultMetric, resource, groupDimValues) + if client.Config.AddCloudMetadata { + vm = client.GetVMForMetaData(&resource, groupDimValues) + addCloudVMMetadata(&event, vm) + } } } } else { event, metricList = createEvent(timestamp, defaultMetric, resource, groupTimeValues) - } - if client.Config.AddCloudMetadata { - addCloudVMMetadata(&event, resource) + if client.Config.AddCloudMetadata { + vm = client.GetVMForMetaData(&resource, groupTimeValues) + addCloudVMMetadata(&event, vm) + } } if client.Config.DefaultResourceType == "" { event.ModuleFields.Put("metrics", metricList) @@ -208,6 +214,7 @@ func createEvent(timestamp time.Time, metric Metric, resource Resource, metricVa } } addHostMetadata(&event, metricList) + return event, metricList } diff --git a/x-pack/metricbeat/module/azure/module.yml b/x-pack/metricbeat/module/azure/module.yml index d7e4831e203..a51b202612b 100644 --- a/x-pack/metricbeat/module/azure/module.yml +++ b/x-pack/metricbeat/module/azure/module.yml @@ -5,3 +5,4 @@ metricsets: - container_service - database_account - compute_vm + - compute_vm_scaleset diff --git a/x-pack/metricbeat/module/azure/resources.go b/x-pack/metricbeat/module/azure/resources.go index ba2c93b4618..0a723c82bd5 100644 --- a/x-pack/metricbeat/module/azure/resources.go +++ b/x-pack/metricbeat/module/azure/resources.go @@ -18,12 +18,15 @@ type Resource struct { Tags map[string]string Subscription string Type string - Vm VmResource + // will be filled if cloud data is necessary, atm only in case of compute_vm and compute_vm_scaleset + Vms []VmResource } +// VmResource contains details specific to a vm type of resource type VmResource struct { Size string Id string + Name string } // Metric will contain the main azure metric details