Ignore unsupported metrics in the azure module

CHANGELOG.next.asciidoc

@@ -731,6 +731,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
 - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
 - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
+- Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]
 - Map cloud data filed `cloud.account.id` to azure subscription.  {pull}21483[21483] {issue}21381[21381]
 
 *Packetbeat*

x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml

@@ -12,8 +12,10 @@ input:
       - name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"]
         namespace: "Microsoft.Compute/virtualMachineScaleSets"
         timegrain: "PT5M"
+        ignore_unsupported: true
       - name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"]
         namespace: "Microsoft.Compute/virtualMachineScaleSets"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
           - name: "SlotId"
@@ -24,6 +26,7 @@ input:
       "Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage",
       "OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"]
         namespace: "Microsoft.Compute/virtualMachineScaleSets"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
           - name: "VMName"
@@ -40,9 +43,11 @@ input:
       metrics:
       - name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"]
         namespace: "Microsoft.Compute/virtualMachineScaleSets"
+        ignore_unsupported: true
         timegrain: "PT5M"
       - name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"]
         namespace: "Microsoft.Compute/virtualMachineScaleSets"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
           - name: "SlotId"
@@ -53,6 +58,7 @@ input:
                  "Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage",
                  "OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"]
         namespace: "Microsoft.Compute/virtualMachineScaleSets"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
           - name: "VMName"

x-pack/metricbeat/module/azure/config.go

@@ -41,7 +41,7 @@ type MetricConfig struct {
 	Dimensions   []DimensionConfig `config:"dimensions"`
 	Timegrain    string            `config:"timegrain"`
 	// namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if namespace is unsupported
-	// info messages will be logged instead
+	// info messages will be logged instead. Same situation with metrics, some are being removed from the API, we would like to make sure that does not affect the module
 	IgnoreUnsupported bool `config:"ignore_unsupported"`
 }
 

x-pack/metricbeat/module/azure/container_instance/manifest.yml

@@ -10,21 +10,25 @@ input:
       metrics:
       - name: ["CpuUsage", "MemoryUsage"]
         namespace: "Microsoft.ContainerInstance/containerGroups"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "containerName"
           value: "*"
       - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"]
         namespace: "Microsoft.ContainerInstance/containerGroups"
+        ignore_unsupported: true
         timegrain: "PT5M"
     - resource_id: ""
       metrics:
       - name: ["CpuUsage", "MemoryUsage"]
         namespace: "Microsoft.ContainerInstance/containerGroups"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "containerName"
           value: "*"
       - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"]
         namespace: "Microsoft.ContainerInstance/containerGroups"
+        ignore_unsupported: true
         timegrain: "PT5M"

x-pack/metricbeat/module/azure/container_service/manifest.yml

@@ -10,6 +10,7 @@ input:
       metrics:
       - name: ["kube_node_status_condition"]
         namespace: "Microsoft.ContainerService/managedClusters"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "node"
@@ -20,9 +21,11 @@ input:
           value: "*"
       - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes"]
         namespace: "Microsoft.ContainerService/managedClusters"
+        ignore_unsupported: true
         timegrain: "PT5M"
       - name: ["kube_pod_status_ready", "kube_pod_status_phase"]
         namespace: "Microsoft.ContainerService/managedClusters"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "pod"
@@ -31,6 +34,7 @@ input:
       metrics:
       - name: ["kube_node_status_condition"]
         namespace: "Microsoft.ContainerService/managedClusters"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "node"
@@ -41,9 +45,11 @@ input:
           value: "*"
       - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes"]
         namespace: "Microsoft.ContainerService/managedClusters"
+        ignore_unsupported: true
         timegrain: "PT5M"
       - name: ["kube_pod_status_ready", "kube_pod_status_phase"]
         namespace: "Microsoft.ContainerService/managedClusters"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "pod"

x-pack/metricbeat/module/azure/database_account/manifest.yml

@@ -14,12 +14,14 @@ input:
       - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount",
                "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"]
         namespace: "Microsoft.DocumentDb/databaseAccounts"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "DatabaseName"
           value: "*"
       - name: ["TotalRequestUnits", "TotalRequests"]
         namespace: "Microsoft.DocumentDb/databaseAccounts"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "DatabaseName"
@@ -28,6 +30,7 @@ input:
           value: "*"
       - name: ["CassandraRequestCharges", "CassandraRequests"]
         namespace: "Microsoft.DocumentDb/databaseAccounts"
+        ignore_unsupported: true
         timegrain: "PT1M"
         dimensions:
         - name: "DatabaseName"
@@ -38,6 +41,7 @@ input:
         "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete",
                 "TableTableThroughputUpdate","TableTableUpdate"]
         namespace: "Microsoft.DocumentDb/databaseAccounts"
+        ignore_unsupported: true
         dimensions:
         - name: "ResourceName"
           value: "*"
@@ -49,12 +53,14 @@ input:
       - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount",
                "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"]
         namespace: "Microsoft.DocumentDb/databaseAccounts"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "DatabaseName"
           value: "*"
       - name: ["TotalRequestUnits", "TotalRequests"]
         namespace: "Microsoft.DocumentDb/databaseAccounts"
+        ignore_unsupported: true
         timegrain: "PT5M"
         dimensions:
         - name: "DatabaseName"
@@ -63,6 +69,7 @@ input:
           value: "*"
       - name: ["CassandraRequestCharges", "CassandraRequests"]
         namespace: "Microsoft.DocumentDb/databaseAccounts"
+        ignore_unsupported: true
         timegrain: "PT1M"
         dimensions:
         - name: "DatabaseName"
@@ -73,6 +80,7 @@ input:
                 "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete",
                 "TableTableThroughputUpdate","TableTableUpdate"]
         namespace: "Microsoft.DocumentDb/databaseAccounts"
+        ignore_unsupported: true
         dimensions:
         - name: "ResourceName"
           value: "*"

x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc

@@ -50,6 +50,9 @@ Metrics with dimensions are exported as flattened single dimensional metrics, ag
 `name`:: Dimension key
 `value`:: Dimension value. (Users can select * to return metric values for each dimension)
 
+`ignore_unsupported`:: (_bool_) Namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if the namespace is unsupported.
+The same will go for the metrics configured, some can be removed from Azure Monitor and it should not affect the state of the module.
+
 Users can select the options to retrieve all metrics from a specific namespace using the following:
 
 ["source","yaml"]

x-pack/metricbeat/module/azure/monitor/client_helper.go

@@ -78,20 +78,20 @@ func filterMetricNames(resourceId string, metricConfig azure.MetricConfig, metri
 		}
 	} else {
 		// verify if configured metric names are valid, return log error event for the invalid ones, map only  the valid metric names
-		supportedMetricNames, unsupportedMetricNames = filterSConfiguredMetrics(metricConfig.Name, metricDefinitions)
-		if len(unsupportedMetricNames) > 0 {
+		supportedMetricNames, unsupportedMetricNames = filterConfiguredMetrics(metricConfig.Name, metricDefinitions)
+		if len(unsupportedMetricNames) > 0 && !metricConfig.IgnoreUnsupported {
 			return nil, errors.Errorf("the metric names configured  %s are not supported for the resource %s and namespace %s",
 				strings.Join(unsupportedMetricNames, ","), resourceId, metricConfig.Namespace)
 		}
 	}
-	if len(supportedMetricNames) == 0 {
+	if len(supportedMetricNames) == 0 && !metricConfig.IgnoreUnsupported {
 		return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceId, metricConfig.Namespace)
 	}
 	return supportedMetricNames, nil
 }
 
-// filterSConfiguredMetrics will filter out any unsupported metrics based on the namespace selected
-func filterSConfiguredMetrics(selectedRange []string, allRange []insights.MetricDefinition) ([]string, []string) {
+// filterConfiguredMetrics will filter out any unsupported metrics based on the namespace selected
+func filterConfiguredMetrics(selectedRange []string, allRange []insights.MetricDefinition) ([]string, []string) {
 	var inRange []string
 	var notInRange []string
 	var allMetrics string

x-pack/metricbeat/module/azure/monitor/client_helper_test.go

@@ -108,7 +108,7 @@ func TestMapMetric(t *testing.T) {
 
 func TestFilterSConfiguredMetrics(t *testing.T) {
 	selectedRange := []string{"TotalRequests", "Capacity", "CPUUsage"}
-	intersection, difference := filterSConfiguredMetrics(selectedRange, *MockMetricDefinitions())
+	intersection, difference := filterConfiguredMetrics(selectedRange, *MockMetricDefinitions())
 	assert.Equal(t, intersection, []string{"TotalRequests", "Capacity"})
 	assert.Equal(t, difference, []string{"CPUUsage"})
 }