diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 01630976e2f..bf7c19ae98e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -171,6 +171,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added new disk states and raid level to the system/raid metricset. {pull}11613[11613] - Added `path_name` and `start_name` to service metricset on windows module {issue}8364[8364] {pull}11877[11877] - Add check on object name in the counter path if the instance name is missing {issue}6528[6528] {pull}11878[11878] +- Add AWS cloudwatch metricset. {pull}11798[11798] {issue}11734[11734] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c27b1c3000d..fad706b26b2 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -830,6 +830,43 @@ Total. +[float] +== cloudwatch fields + +`cloudwatch` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by different namespaces. + + + +*`aws.cloudwatch.namespace`*:: ++ +-- +type: keyword + +The namespace specified when query cloudwatch api. + + +-- + +*`aws.cloudwatch.metrics.*`*:: ++ +-- +type: object + +Metrics that returned from Cloudwatch api query. + + +-- + +*`aws.cloudwatch.dimensions.*`*:: ++ +-- +type: object + +Cloudwatch metric dimensions. + + +-- + [float] == ec2 fields diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index ea919d6ee6a..20a20a7077d 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -9,7 +9,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for AWS services. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`. +The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch`. [float] === Module-specific configuration notes @@ -50,8 +50,8 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID}' secret_access_key: '${AWS_SECRET_ACCESS_KEY}' session_token: '${AWS_SESSION_TOKEN}' @@ -102,6 +102,11 @@ https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[Ho Configure Request Metrics for S3] for instructions on how to enable request metrics for each S3 bucket. +[float] +=== `cloudwatch` +This metricset gives users the freedom to query metrics from AWS Cloudwatch with +any given namespaces or specific instance with a given period. + [float] === Example configuration @@ -115,8 +120,8 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -124,12 +129,27 @@ metricbeat.modules: - module: aws period: 86400s metricsets: - - "s3_request" - - "s3_daily_storage" + - s3_request + - s3_daily_storage access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS ---- [float] @@ -137,6 +157,8 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> * <> @@ -145,6 +167,8 @@ The following metricsets are available: * <> +include::aws/cloudwatch.asciidoc[] + include::aws/ec2.asciidoc[] include::aws/s3_daily_storage.asciidoc[] diff --git a/metricbeat/docs/modules/aws/cloudwatch.asciidoc b/metricbeat/docs/modules/aws/cloudwatch.asciidoc new file mode 100644 index 00000000000..68df2f75b4c --- /dev/null +++ b/metricbeat/docs/modules/aws/cloudwatch.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-aws-cloudwatch]] +=== aws cloudwatch metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index ae042951010..ee2e22de58c 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,7 +10,8 @@ This file is generated! See scripts/docs_collector.py |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.4+| .4+| |<> +.5+| .5+| |<> beta[] +|<> |<> beta[] |<> beta[] |<> beta[] diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 761052d0df7..635c594e97e 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -9,6 +9,7 @@ package include import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/cloudwatch" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_daily_storage" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_request" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index b59e33b5cdf..07edfe8968c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -160,8 +160,8 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -169,12 +169,27 @@ metricbeat.modules: - module: aws period: 86400s metricsets: - - "s3_request" - - "s3_daily_storage" + - s3_request + - s3_daily_storage access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS #--------------------------------- Ceph Module --------------------------------- - module: ceph diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 18a720f84fd..d1e541fb07d 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -1,8 +1,8 @@ - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -10,9 +10,24 @@ - module: aws period: 86400s metricsets: - - "s3_request" - - "s3_daily_storage" + - s3_request + - s3_daily_storage access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index d4a39dc7cf4..243aeb0ecc0 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -2,7 +2,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for AWS services. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`. +The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch`. [float] === Module-specific configuration notes @@ -43,8 +43,8 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID}' secret_access_key: '${AWS_SECRET_ACCESS_KEY}' session_token: '${AWS_SESSION_TOKEN}' @@ -94,3 +94,8 @@ always adjust this to the granularity they want. Request metrics are not enabled https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[How to Configure Request Metrics for S3] for instructions on how to enable request metrics for each S3 bucket. + +[float] +=== `cloudwatch` +This metricset gives users the freedom to query metrics from AWS Cloudwatch with +any given namespaces or specific instance with a given period. diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index eb40e81c6a1..ce0f636ea7f 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -5,7 +5,7 @@ package aws import ( - "strconv" + "time" "github.com/elastic/beats/libbeat/common" @@ -20,11 +20,11 @@ import ( // Config defines all required and optional parameters for aws metricsets type Config struct { - Period string `config:"period"` - AccessKeyID string `config:"access_key_id"` - SecretAccessKey string `config:"secret_access_key"` - SessionToken string `config:"session_token"` - DefaultRegion string `config:"default_region"` + Period time.Duration `config:"period" validate:"nonzero,required"` + AccessKeyID string `config:"access_key_id"` + SecretAccessKey string `config:"secret_access_key"` + SessionToken string `config:"session_token"` + DefaultRegion string `config:"default_region"` } // MetricSet is the base metricset for all aws metricsets @@ -82,13 +82,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return nil, err } - // Calculate duration based on period - if config.Period == "" { - err = errors.New("period is not set in AWS module config") - return nil, err - } - - durationString, periodSec, err := convertPeriodToDuration(config.Period) + durationString, periodSec := convertPeriodToDuration(config.Period) if err != nil { return nil, err } @@ -118,28 +112,12 @@ func getRegions(svc ec2iface.EC2API) (regionsList []string, err error) { return } -func convertPeriodToDuration(period string) (string, int, error) { +func convertPeriodToDuration(period time.Duration) (string, int) { // Set starttime double the default frequency earlier than the endtime in order to make sure // GetMetricDataRequest gets the latest data point for each metric. - numberPeriod, err := strconv.Atoi(period[0 : len(period)-1]) - if err != nil { - return "", 0, err - } - - unitPeriod := period[len(period)-1:] - switch unitPeriod { - case "s": - duration := "-" + strconv.Itoa(numberPeriod*2) + unitPeriod - return duration, numberPeriod, nil - case "m": - duration := "-" + strconv.Itoa(numberPeriod*2) + unitPeriod - periodInSec := numberPeriod * 60 - return duration, periodInSec, nil - default: - err = errors.New("invalid period in config. Please reset period in config") - duration := "-" + strconv.Itoa(numberPeriod*2) + "s" - return duration, numberPeriod, err - } + duration := "-" + (period * 2).String() + numberPeriod := int(period.Seconds()) + return duration, numberPeriod } // StringInSlice checks if a string is already exists in list @@ -156,8 +134,8 @@ func StringInSlice(str string, list []string) bool { func InitEvent(metricsetName string, regionName string) mb.Event { event := mb.Event{} event.Service = metricsetName - event.RootFields = common.MapStr{} event.MetricSetFields = common.MapStr{} + event.RootFields = common.MapStr{} event.RootFields.Put("service.name", metricsetName) event.RootFields.Put("cloud.provider", "aws") if regionName != "" { diff --git a/x-pack/metricbeat/module/aws/aws_test.go b/x-pack/metricbeat/module/aws/aws_test.go index b503690464a..5f6c884f93f 100644 --- a/x-pack/metricbeat/module/aws/aws_test.go +++ b/x-pack/metricbeat/module/aws/aws_test.go @@ -9,6 +9,7 @@ package aws import ( "fmt" "testing" + "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -49,33 +50,36 @@ func TestGetRegions(t *testing.T) { } func TestConvertPeriodToDuration(t *testing.T) { - period1 := "300s" - duration1, periodSec1, err := convertPeriodToDuration(period1) - assert.NoError(t, nil, err) - assert.Equal(t, "-600s", duration1) - assert.Equal(t, 300, periodSec1) - - period2 := "30ss" - duration2, periodSec2, err := convertPeriodToDuration(period2) - assert.Error(t, err) - assert.Equal(t, "", duration2) - assert.Equal(t, 0, periodSec2) - - period3 := "10m" - duration3, periodSec3, err := convertPeriodToDuration(period3) - assert.NoError(t, nil, err) - assert.Equal(t, "-20m", duration3) - assert.Equal(t, 600, periodSec3) - - period4 := "30s" - duration4, periodSec4, err := convertPeriodToDuration(period4) - assert.NoError(t, nil, err) - assert.Equal(t, "-60s", duration4) - assert.Equal(t, 30, periodSec4) + cases := []struct { + period time.Duration + expectedDuration string + expectedPeriodNumber int + }{ + { + period: time.Duration(300) * time.Second, + expectedDuration: "-10m0s", + expectedPeriodNumber: 300, + }, + { + period: time.Duration(10) * time.Minute, + expectedDuration: "-20m0s", + expectedPeriodNumber: 600, + }, + { + period: time.Duration(30) * time.Second, + expectedDuration: "-1m0s", + expectedPeriodNumber: 30, + }, + { + period: time.Duration(60) * time.Second, + expectedDuration: "-2m0s", + expectedPeriodNumber: 60, + }, + } - period5 := "60s" - duration5, periodSec5, err := convertPeriodToDuration(period5) - assert.NoError(t, nil, err) - assert.Equal(t, "-120s", duration5) - assert.Equal(t, 60, periodSec5) + for _, c := range cases { + duration, periodSec := convertPeriodToDuration(c.period) + assert.Equal(t, c.expectedDuration, duration) + assert.Equal(t, c.expectedPeriodNumber, periodSec) + } } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json new file mode 100644 index 00000000000..6110e7a127a --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json @@ -0,0 +1,63 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "aws": { + "cloudwatch": { + "dimensions": { + "DBClusterIdentifier": "test1-cluster", + "Role": "WRITER" + }, + "metrics": { + "ActiveTransactions": 0, + "AuroraBinlogReplicaLag": 0, + "AuroraReplicaLagMaximum": 19.964599990844725, + "AuroraReplicaLagMinimum": 19.964599990844725, + "BinLogDiskUsage": 0, + "BlockedTransactions": 0, + "BufferCacheHitRatio": 100, + "CPUUtilization": 3, + "CommitLatency": 7.4758933333333335, + "CommitThroughput": 0.4999718865659406, + "DDLLatency": 0, + "DDLThroughput": 0, + "DMLLatency": 0.16068, + "DMLThroughput": 0.4999718865659406, + "DatabaseConnections": 0, + "Deadlocks": 0, + "DeleteLatency": 0, + "DeleteThroughput": 0, + "EngineUptime": 1906152, + "FreeLocalStorage": 32210518016, + "FreeableMemory": 4688029286.4, + "InsertLatency": 0.16068, + "InsertThroughput": 0.4999718865659406, + "LoginFailures": 0, + "NetworkReceiveThroughput": 0.7000140072335472, + "NetworkThroughput": 1.4000280144670945, + "NetworkTransmitThroughput": 0.7000140072335472, + "Queries": 8.796634525053722, + "ResultSetCacheHitRatio": 0, + "SelectLatency": 0.17715786436043143, + "SelectThroughput": 3.086495583173847, + "UpdateLatency": 0, + "UpdateThroughput": 0 + }, + "namespace": "AWS/RDS" + } + }, + "cloud": { + "provider": "aws", + "region": "us-east-2" + }, + "event": { + "dataset": "aws.cloudwatch", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "cloudwatch" + }, + "service": { + "name": "cloudwatch", + "type": "cloudwatch" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc new file mode 100644 index 00000000000..5032cb91697 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc @@ -0,0 +1,12 @@ +The cloudwatch metricset of aws module allows you to monitor various services on +AWS. `cloudwatch` metricset fetches metrics from given namespace periodically +by calling `GetMetricData` api. + +[float] +=== AWS Permissions +Some specific AWS permissions are required for IAM user to collect AWS SQS metrics. +---- +ec2:DescribeRegions +cloudwatch:GetMetricData +cloudwatch:ListMetrics +---- diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml new file mode 100644 index 00000000000..875b0aac111 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml @@ -0,0 +1,22 @@ +- name: cloudwatch + type: group + description: > + `cloudwatch` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by different namespaces. + release: beta + fields: + - name: namespace + type: keyword + description: > + The namespace specified when query cloudwatch api. + - name: metrics.* + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Metrics that returned from Cloudwatch api query. + - name: dimensions.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Cloudwatch metric dimensions. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go new file mode 100644 index 00000000000..486b01e8cc3 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -0,0 +1,334 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cloudwatch + +import ( + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/aws" +) + +var ( + metricsetName = "cloudwatch" + metricNameIdx = 0 + namespaceIdx = 1 + identifierNameIdx = 2 + identifierValueIdx = 3 +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(aws.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *aws.MetricSet + CloudwatchConfigs []Config `config:"cloudwatch_metrics" validate:"nonzero,required"` +} + +// Dimension holds name and value for cloudwatch metricset dimension config. +type Dimension struct { + Name string `config:"name" validate:"nonzero"` + Value string `config:"value" validate:"nonzero"` +} + +// Config holds a configuration specific for cloudwatch metricset. +type Config struct { + Namespace string `config:"namespace" validate:"nonzero,required"` + MetricName string `config:"metricname"` + Dimensions []Dimension `config:"dimensions"` +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The aws cloudwatch metricset is beta.") + metricSet, err := aws.NewMetricSet(base) + if err != nil { + return nil, errors.Wrap(err, "error creating aws metricset") + } + + config := struct { + CloudwatchMetrics []Config `config:"cloudwatch_metrics" validate:"nonzero,required"` + }{} + + err = base.Module().UnpackConfig(&config) + if err != nil { + return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig") + } + + if len(config.CloudwatchMetrics) == 0 { + return nil, errors.New("cloudwatch_metrics in config is missing") + } + + return &MetricSet{ + MetricSet: metricSet, + CloudwatchConfigs: config.CloudwatchMetrics, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + // Get startTime and endTime + startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) + if err != nil { + return errors.Wrap(err, "error GetStartTimeEndTime") + } + + // Get listMetrics and namespacesTotal from configuration + listMetrics, namespacesTotal := readCloudwatchConfig(m.CloudwatchConfigs) + for _, regionName := range m.MetricSet.RegionsList { + awsConfig := m.MetricSet.AwsConfig.Copy() + awsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(awsConfig) + err := createEvents(svcCloudwatch, listMetrics, regionName, m.PeriodInSec, startTime, endTime, report) + if err != nil { + return errors.Wrap(err, "createEvents failed") + } + } + + // Use namespaces from config + for _, namespace := range namespacesTotal { + for _, regionName := range m.MetricSet.RegionsList { + awsConfig := m.MetricSet.AwsConfig.Copy() + awsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(awsConfig) + listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) + if err != nil { + m.Logger().Info(err.Error()) + continue + } + + if listMetricsOutput == nil || len(listMetricsOutput) == 0 { + continue + } + + err = createEvents(svcCloudwatch, listMetricsOutput, regionName, m.PeriodInSec, startTime, endTime, report) + if err != nil { + return errors.Wrap(err, "createEvents failed for region "+regionName) + } + } + } + + return nil +} + +func readCloudwatchConfig(cloudwatchConfigs []Config) ([]cloudwatch.Metric, []string) { + var listMetrics []cloudwatch.Metric + var namespacesTotal []string + + for _, cloudwatchConfig := range cloudwatchConfigs { + namespace := cloudwatchConfig.Namespace + if cloudwatchConfig.MetricName != "" { + listMetricsOutput := convertConfigToListMetrics(cloudwatchConfig, namespace) + listMetrics = append(listMetrics, listMetricsOutput) + } else { + namespacesTotal = append(namespacesTotal, namespace) + } + } + return listMetrics, namespacesTotal +} + +func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery { + var metricDataQueries []cloudwatch.MetricDataQuery + for i, listMetric := range listMetricsOutput { + metricDataQuery := createMetricDataQuery(listMetric, i, period) + metricDataQueries = append(metricDataQueries, metricDataQuery) + } + return metricDataQueries +} + +func constructLabel(metric cloudwatch.Metric) string { + // label = metricName + namespace + dimensionKey1 + dimensionValue1 + + // dimensionKey2 + dimensionValue2 + ... + label := *metric.MetricName + " " + *metric.Namespace + dimNames := "" + dimValues := "" + for i, dim := range metric.Dimensions { + dimNames += *dim.Name + dimValues += *dim.Value + if i != len(metric.Dimensions)-1 { + dimNames += "," + dimValues += "," + } + } + + if dimNames != "" && dimValues != "" { + label += " " + dimNames + label += " " + dimValues + } + return label +} + +func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (metricDataQuery cloudwatch.MetricDataQuery) { + statistic := "Average" + id := "cw" + strconv.Itoa(index) + label := constructLabel(metric) + + metricDataQuery = cloudwatch.MetricDataQuery{ + Id: &id, + MetricStat: &cloudwatch.MetricStat{ + Period: &period, + Stat: &statistic, + Metric: &metric, + }, + Label: &label, + } + return +} + +func getIdentifiers(listMetricsOutputs []cloudwatch.Metric) map[string][]string { + if len(listMetricsOutputs) == 0 { + return nil + } + + identifiers := map[string][]string{} + for _, listMetrics := range listMetricsOutputs { + identifierName := "" + identifierValue := "" + if len(listMetrics.Dimensions) == 0 { + continue + } + + for i, dim := range listMetrics.Dimensions { + identifierName += *dim.Name + identifierValue += *dim.Value + if i != len(listMetrics.Dimensions)-1 { + identifierName += "," + identifierValue += "," + } + } + + if identifiers[identifierName] != nil { + if !aws.StringInSlice(identifierValue, identifiers[identifierName]) { + identifiers[identifierName] = append(identifiers[identifierName], identifierValue) + } + } else { + identifiers[identifierName] = []string{identifierValue} + } + } + + return identifiers +} + +func insertMetricSetFields(event mb.Event, metricValue float64, labels []string) mb.Event { + event.MetricSetFields.Put("metrics."+labels[metricNameIdx], metricValue) + event.MetricSetFields.Put("namespace", labels[namespaceIdx]) + if len(labels) == 2 { + return event + } + + dimNames := strings.Split(labels[identifierNameIdx], ",") + dimValues := strings.Split(labels[identifierValueIdx], ",") + for i := 0; i < len(dimNames); i++ { + event.MetricSetFields.Put("dimensions."+dimNames[i], dimValues[i]) + } + return event +} + +func convertConfigToListMetrics(cloudwatchConfig Config, namespace string) cloudwatch.Metric { + // convert config input to []cloudwatch.Metric + var cloudwatchDimensions []cloudwatch.Dimension + for _, dim := range cloudwatchConfig.Dimensions { + name := dim.Name + value := dim.Value + cloudwatchDimensions = append(cloudwatchDimensions, cloudwatch.Dimension{ + Name: &name, + Value: &value, + }) + } + + listMetricsOutput := cloudwatch.Metric{ + Namespace: &namespace, + MetricName: &cloudwatchConfig.MetricName, + Dimensions: cloudwatchDimensions, + } + return listMetricsOutput +} + +func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwatch.Metric, regionName string, period int, startTime time.Time, endTime time.Time, report mb.ReporterV2) error { + identifiers := getIdentifiers(listMetricsTotal) + // Initialize events map per region, which stores one event per identifierValue + events := map[string]mb.Event{} + for _, values := range identifiers { + for _, v := range values { + events[v] = aws.InitEvent(metricsetName, regionName) + } + } + // Initialize events for the ones without identifiers. + var eventsNoIdentifier []mb.Event + + // Construct metricDataQueries + metricDataQueries := constructMetricQueries(listMetricsTotal, int64(period)) + if len(metricDataQueries) == 0 { + return nil + } + + // Use metricDataQueries to make GetMetricData API calls + metricDataResults, err := aws.GetMetricDataResults(metricDataQueries, svc, startTime, endTime) + if err != nil { + return errors.Wrap(err, "GetMetricDataResults failed") + } + + // Find a timestamp for all metrics in output + timestamp := aws.FindTimestamp(metricDataResults) + if !timestamp.IsZero() { + for _, output := range metricDataResults { + if len(output.Values) == 0 { + continue + } + + exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) + if exists { + labels := strings.Split(*output.Label, " ") + if len(labels) == 4 { + identifierValue := labels[identifierValueIdx] + events[identifierValue] = insertMetricSetFields(events[identifierValue], output.Values[timestampIdx], labels) + } else { + eventNew := aws.InitEvent(metricsetName, regionName) + eventNew = insertMetricSetFields(eventNew, output.Values[timestampIdx], labels) + eventsNoIdentifier = append(eventsNoIdentifier, eventNew) + } + } + } + } + + for _, event := range events { + if len(event.MetricSetFields) != 0 { + if reported := report.Event(event); !reported { + return nil + } + } + } + + for _, event := range eventsNoIdentifier { + if len(event.MetricSetFields) != 0 { + if reported := report.Event(event); !reported { + return nil + } + } + } + + return nil +} diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go new file mode 100644 index 00000000000..c76808d6de7 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go @@ -0,0 +1,54 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package cloudwatch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" +) + +func TestFetch(t *testing.T) { + config, info := mtest.GetConfigForTest("cloudwatch", "300s") + if info != "" { + t.Skip("Skipping TestFetch: " + info) + } + + config = addCloudwatchMetricsToConfig(config) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, info := mtest.GetConfigForTest("cloudwatch", "300s") + if info != "" { + t.Skip("Skipping TestData: " + info) + } + + config = addCloudwatchMetricsToConfig(config) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + if err := mbtest.WriteEventsReporterV2Error(metricSet, t, "/"); err != nil { + t.Fatal("write", err) + } +} + +func addCloudwatchMetricsToConfig(config map[string]interface{}) map[string]interface{} { + cloudwatchMetricsConfig := []map[string]interface{}{} + cloudwatchMetric := map[string]interface{}{} + cloudwatchMetric["namespace"] = "AWS/RDS" + cloudwatchMetricsConfig = append(cloudwatchMetricsConfig, cloudwatchMetric) + config["cloudwatch_metrics"] = cloudwatchMetricsConfig + return config +} diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go new file mode 100644 index 00000000000..deda6bcc9fd --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -0,0 +1,223 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package cloudwatch + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/stretchr/testify/assert" +) + +var ( + instanceID1 = "i-1" + instanceID2 = "i-2" + namespace = "AWS/EC2" + dimName = "InstanceId" + metricName1 = "CPUUtilization" + metricName2 = "StatusCheckFailed" + metricName3 = "StatusCheckFailed_System" + metricName4 = "StatusCheckFailed_Instance" + metricName5 = "CommitThroughput" + namespaceRDS = "AWS/RDS" + listMetric1 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName, + Value: &instanceID1, + }}, + MetricName: &metricName1, + Namespace: &namespace, + } + + listMetric2 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName, + Value: &instanceID1, + }}, + MetricName: &metricName2, + Namespace: &namespace, + } + + listMetric3 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName, + Value: &instanceID2, + }}, + MetricName: &metricName3, + Namespace: &namespace, + } + + listMetric4 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName, + Value: &instanceID2, + }}, + MetricName: &metricName4, + Namespace: &namespace, + } + + listMetric5 = cloudwatch.Metric{ + MetricName: &metricName1, + Namespace: &namespace, + } + + dimName1 = "DBClusterIdentifier" + dimValue1 = "test1-cluster" + dimName2 = "Role" + dimValue2 = "READER" + listMetric6 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName1, + Value: &dimValue1, + }, + { + Name: &dimName2, + Value: &dimValue2, + }}, + MetricName: &metricName5, + Namespace: &namespaceRDS, + } + + listMetric7 = cloudwatch.Metric{ + MetricName: &metricName1, + Namespace: &namespace, + } +) + +func TestGetIdentifiers(t *testing.T) { + listMetricsOutput := []cloudwatch.Metric{listMetric1, listMetric2, listMetric3, listMetric4} + identifiers := getIdentifiers(listMetricsOutput) + assert.Equal(t, []string{instanceID1, instanceID2}, identifiers["InstanceId"]) +} + +func TestConstructLabel(t *testing.T) { + cases := []struct { + listMetric cloudwatch.Metric + expectedLabel string + }{ + { + listMetric1, + "CPUUtilization AWS/EC2 InstanceId i-1", + }, + { + listMetric2, + "StatusCheckFailed AWS/EC2 InstanceId i-1", + }, + { + listMetric3, + "StatusCheckFailed_System AWS/EC2 InstanceId i-2", + }, + { + listMetric4, + "StatusCheckFailed_Instance AWS/EC2 InstanceId i-2", + }, + { + listMetric5, + "CPUUtilization AWS/EC2", + }, + } + + for _, c := range cases { + label := constructLabel(c.listMetric) + assert.Equal(t, c.expectedLabel, label) + } +} + +func TestReadCloudwatchConfig(t *testing.T) { + cases := []struct { + cloudwatchMetricsConfig []Config + expectedListMetric []cloudwatch.Metric + expectedNamespace []string + }{ + { + []Config{ + { + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: []Dimension{ + { + Name: "InstanceId", + Value: instanceID1, + }, + }, + }, + }, + []cloudwatch.Metric{listMetric1}, + nil, + }, + { + []Config{ + { + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: []Dimension{ + { + Name: "InstanceId", + Value: instanceID1, + }, + }, + }, + { + Namespace: "AWS/EBS", + }, + }, + []cloudwatch.Metric{listMetric1}, + []string{"AWS/EBS"}, + }, + { + []Config{ + { + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: []Dimension{ + { + Name: "InstanceId", + Value: instanceID1, + }, + }, + }, + { + Namespace: "AWS/EBS", + }, + { + Namespace: "AWS/RDS", + MetricName: "CommitThroughput", + Dimensions: []Dimension{ + { + Name: "DBClusterIdentifier", + Value: "test1-cluster", + }, + { + Name: "Role", + Value: "READER", + }, + }, + }, + }, + []cloudwatch.Metric{listMetric1, listMetric6}, + []string{"AWS/EBS"}, + }, + { + []Config{ + { + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + }, + { + Namespace: "AWS/EBS", + }, + }, + []cloudwatch.Metric{listMetric7}, + []string{"AWS/EBS"}, + }, + } + for _, c := range cases { + listMetrics, namespaces := readCloudwatchConfig(c.cloudwatchMetricsConfig) + assert.Equal(t, c.expectedListMetric, listMetrics) + assert.Equal(t, c.expectedNamespace, namespaces) + } +} diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index 8b3bdae475d..802b957afc0 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -123,7 +123,7 @@ func TestGetInstanceIDs(t *testing.T) { func TestCreateCloudWatchEvents(t *testing.T) { mockModuleConfig := aws.Config{ - Period: "300s", + Period: 300 * time.Second, DefaultRegion: regionName, } diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 5d5ff0faf27..2bb0e5771ba 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsWktvIzcSvs+vKOSUAOPGZmayBx8WmHiMjYFs4kQe5NhTIktqrtlkDx+SZcyPXxT7oYdbsix3K5fVwTDEFuv7ih+LVcW+gHtaXQIu/RuAoIKmS/gOl/67NwCSvHCqCsqaS/jXGwCAL7j0X6C0MmoCYbUmETx8/GsCpTUqWKfMHEoKTgkPM2fLNHalbZRLDKLI3gA40oSeLmGObwBmirT0l2n2CzBYUouGP2FV8YPOxqr5pgfU9iSbE5F4133XN9neCevPFxLvvoCwJqAyHkJBHbdQYIAlOQIvHFYkd9j+xWxhWShRrCfo8ZEnE2C6Sj+8vnqXbdjf9lP72aW6SVdUMQs2oM4qEbaeaMl7gZpkPtMWdx844Af+3BUEFTlBJuCcwM4AtbYCA0kGDsKWVQwE0ajQuAcdgYjOkQl6BcpA9ATWJD8q4wMaQdleIsKRVCGPHuc0AhcTyyk55nF1+xlqYx581azHJkaYWZeeikFp9Yg87bO4p6j5t6MiJ3SG5BaB2vFmjb1ADyiEiyTBK/5GBViiB43RiIIkWAc+oAsk95Py0VU6+vyM5BqT28wKXBBMicx6pdBANFqVipXY0V4WZIB/dnX7+SrN8HONGRaoI4Hy8EjOHsvY56JANyc5LuXEqZc47yVjA1SoJEi7NEz96fq/BTSyCTuhiB6UEdGxj1BKxShQQ02ln7qhsLTuPlMmq1DcU/CjMm5sgCNBasFiNBxXWhigTCA3Q0F+d1Mehm9jOCv+FMZtDEPhVyabrgKNCz5ZGMX158I+lNul8vfKZo5QjoL958bT2KQJjLULVT5YR7CwOpbkAReoNE41QbDHI186FWhE6Dx/IMOYBseevG6roYFf2bLSxIdC8rutyKWT25++BJzDIEdpoWaKJOdDykqWY1DlMQs0JstkYZPmiWt1CkkfMESfiYLEfT5DpfcclNqa+cv4/UmVdcHzeR4KcttIObep0HuSMLWh2B6sMUHClE5FHvUrH6jcHlN1RqrRByiVieF4knk935m5jkGktfM3UOlfsWPJdCFGWMd/oumvfPhImJN7daFgHflUEDwf37pRVeKcMtW/J+5ptbRud+wIYDef0qZkGDw/V1eSN3Od2L8E37ouzXgN+kVwEs4bIxXXiGslSApJcZvFsPJAhmPRngqkA1o5tcBAmTQ+56FhHdrMDp9+myTDrXufZBVHolRVvxJ3v34BtJvbxQdO5R15D+i9FSpV4EvVhL8XY41TrcRYDk2TP/HnkapsoA3oxdZxDY5rDi5KwM1tN/I9O/gHmNpoZHswvtSlaQtlwsp+b54ciNK8uz58C1zfw4//vJiqANF4NTepDk5GjkI6/Lr3IoXvKzKSt/s3cNGY+j9fxBCUmV+kmvYbBHKlMknT3zhjqarmOf6X5A/PMAoFJ3s+r8jlHKrHOgoaO5wcdcdC9qT96N/nEpVe5Zx6bbexXt6L3J1spzGZxqAZG7VNOXnf26WcUji2TzmNXDUPLLzfmgiDMHnfWjhk3avHcSqmFGlKTkQYjsSAnMw0BSvn4LJOsQ9hrJWW21lup/8lMUoXI3WKNzTdWOrBmPIdriVaeQnN6ZvvVbyjr5F8eK3Wm2k2VN58839tH9BN4yNfXwMMlMz3ieWXu7vbzhqUKFMdiQY+lvhozRrnW3A0Ryd1e/Cuqj0HaId9Tv0p/GnIdzD/+/puBzeLu9U+i/4ph2fwVnFEvLefB8crSdOeJH8QyJ+uf72+ux4adUE4VFOhB/Mv1x8/HaXn57Rg/Zhi+H2yq4aTUHrStOeG8LU410gm179eX93B72nR4cqawIF2YFXUTHIv0Bgap3nb13huD/bGbl1/HU39NUwdhej+Dqqt4XNw1WrMXdRhSxkE20oZtKih+/qgPYSTaxVtUY6/CvUSrO2lDXPcsbssODViYo58ZY3nUkzoKIkLxqmVq35ysTontdZavRZNmgbYJXuM8+3LIx05Z53PPjw8jCejDw8PILRitSdzXf/SSjpqjeqdhM2NtJ0BqdQi+wdYBz8eJPbTmMR+engAT25B7ozENAYyYpXNlPMhZ3FkZb/6TuNYkbtoRRVUSXWxUO/7+gplrTni8qC7F63fMXjCMdj6JYOtHZZepEhXL1PqIuZhwimzbsudYTmTxsrXVzh7uCdvp6245tu0vVLnPY2kAum53ddVgF/960q/r/6Mr1xN/pi8tuCzWpIPeUne45xynFPmSQy4ilhVzj6oEgNB8+oVu6W2C8aaizqhl9BgaO9OvkaKe2qt5slUC+BqsNu6u+1o0lrZArR+n6WxnS7njA0bV5L1KYepW6jKkqTCQHrPgdVxMTbkC+XVVI9T3HR0OgbKwEyrebHnFOqQnQXVrvuCU7RAvd7sR+qBpTQu0lavL0LWxqdxoXVZ7nQFArX2bTj8szb/n2aLodj/FmAHmSPNyGsuZR2x8ZAPqazCKm8cOOQBs0a0456Ptzet+3ivSFXv8Nq7gC2BPffTZNbx9Jk2bQ/umXUlhkvo+9ExlxfqkU7wcT00bNtv8sekiZlp3v8FAAD//26rI9M=" + return "eJzsWktz2zgSvudXdOU0MxWrdpLMHnzYqozj2knVPDxjp+bItICWiDUIMHhYlis/fqsBknpRtiyT2svqkHIECv19/UJ3g2dwS8tzwIV/BRBU0HQOr3HhX78CkOSFU3VQ1pzDv14BAHzBhf8ClZVREwirNYng4cPf11BZo4J1ysyhouCU8DBztkprF9pGucAgyskrAEea0NM5zPEVwEyRlv487X4GBitq0fAnLGt+0NlYN9/0gNrcZH0j0Qnulvr23Ltv/nxZbfMFhDUBlfEQSuqYhhIDLMgReOGwJrnF/W/+LSxKJcrVBj0a82QCTJcg1WxGjv/DPHyNgvxkDVOnwikFXPt+Ww/ruuh22lht1XFLy4V1cmvtEaXw56ak1bbgaxJqpkjCoiQDXyO55ZoFAGs16UXWsJ/80IvMTv9DImwt5S+L/IS0caq3aa09UVRY18rMm8df//D6eTR/W7eyoxCdaS18sUEvc+4nKVVFxitrjufZb6MBia6xyTZZR70TXCTeviiqSLw9UTjxDy8v3vZG0PzQ+BF1nAQbUE/qHTNl8l6gJlnMtMXtBw4IpJqcIBNwTmBngFpbgYEkAwdhqzoGgmhUaNSDjkBEx0lCL0EZiJ7AmqRHZXxAI6jfE5mIcCRVKKLHeX8+eBkXE6spOeZxcfUZsjDPCSLbYx0jzKxLT8WgtHpA3vZJ3FPU/NtRkROmMF8nkBVvVthL9IBCuEgSvOJvVIAFetAYjShJgnXgA7pAcj8pH12toy9OSK4RucmsxDuCKZFZWQoNRKNVpdgTO9opv/PPLq4+X6Qdfs6Y4Q51JFAeHsjZQxn7QpTo5rSd2QamnDj1EudYMjZAjUqCtAvD1Hft/wbQyCbthDJ6UEZExzpCKRWjQA2ZSj91Q2Fh3e1EmUmN4paCH5VxIwMcCVJ37IyG80oLA5QJ5GZcW2wH5ePwbQwnxZ/SuI1hKPzKTKbLQOOCTxJGUf2psA+ldqn8rbITRyhHwf5zo2lsygTG2qUqH6wjuLM6VuQB71BpnGqCYA9HvnAq0IjQef9AhjENjj1p3dZDA7+wVa2JD4Wkd1uTSye3P94EXMPgWhNRk1NWsjsGVR1ioDFZJgnrNI+01TEkfcAQ/USUJG6LGSq956DU1syfx+8vqq0Lns/zUJLbRMq1TY3ek4SpDeXmYsYECVM6FXnVL32ganNN5YpUow9QKRPD4SSLvN+JuY5BpJXzP6DSb7FDyXQpRljH/0TT3/nwkTAn9+JGwTryqSF4Or91q6rCOU1Uf0wcPdb49DEFJcPg/bm7khzMubB/Dr5VXzphGww4fvlkpOIeceUJkkLyuPVmWHkgw7loTwfSAa2dusNAE2l8wUvDKrTZHT7+fp0Et+rdqSoORKnqfk/c/voZ0D5d3b3nUt6R94DeW6FSB75QTfp7NtY41UqMpdC0+Y4+D/TKBtqAWmwV1+C45OSiBHy66la+YwV/D1MbjWwPxueqNIXQRFjZr82jE1Had1uHb4D7e/jxn2dTFSAar+Ym9cFJyEFIh7d7L1L4riYjOdy/gYvG5L98GUNQZn6WetpvEMhVyiSf/sYVSxoWtn+S/P4JRqHkYs8XNbmCU/VYR0Ejh4uj7ljYHT/6d4VEpZcFl16bY6znzyK3N9saTKY1aNZGHVNev3vpnH8auWse2PF+bzIMwvW7VsJj0r16GKdjSpmm4kKE4UgMyMVM07ByDS5zif0YxuxphZ0VeXY+BsY0KV7z6UZSD8ZU73Av0bqX0Fy+9Qzc/bvC0ddIPrzU15tt1ry8+eb/vv2I3zQ68vkaYKBivs9Zfrm5ueqkQYUy9ZFo4EOFD9ascL4BR3N0UrcH77Lec4B22OfUX8Ifh3wL878vb7Zws3O3vs9Ov8vhCbx1HBHv1efB8UrStKfIHwTyx8tfL28uh0ZdEg41VOjB/Mvlh48H+fNTvmD9mM7wx/W2NxyF0pPevcgdBucKyfXlr5cXN/BHMjpcWBM40Q7sFZlJ4QUaQ+MMb/sGz+3B3sjN/dfB1F/CtL3cPznV7q2CE3DVaswo6rClCoJlpQpaZOg+H7SP4eReRVuU41shm2AlLwXMYcfuouTSiIk58rU1nlsxoaMkbhinVu55CyTWp6TWSsu2aMo0wK7YY5xvnp/pyDnr/OT9/f14bvT+/h6EVuztSVw3v7SSDrJRjiRsbqTtDEilEdk/wDr48VFiP41J7Kf7e/Dk7sidkJjGQEYsJzPlfCjYOSZVv/cdx7Emd9Y6VVAV5WYhx32+Qln5HHF70N2L5ncMdjgGm18y2Iiw9CJFunqZUpcxHyecKuu23RmWM2msfb7C2cM9aTuF4opvM/ZKk/e0khqkp6Kv6wC/+pe1fl/9CV+5uv7z+qUNn9WSfCgq8h7nVOCcJp7EgFbEunb2XlUYCJpXr1gtWS4Ya85yQS+hwdDenXyNFPf0Ws2TqRfA5WC3dTeb2aSVsgFo9T5LIztdzhkb1q4k8ymHaVqoqoqkwkB6z4HVcTE2FHfKq903LYdJjh2djoEyMNNqXu45hTpkJ0G1rb7gFN2hXgX7gf7ArjQu0tZfn4WszU/jQuuq3OkSBGrt23T4Vxb/WxNiKPa/BdhB5kwzss2lzBkbH9MhVXVYFo0ChzxgVoi21PPh6lOrPo4VqXKEZ+0CtgT23E+TWeXTJ8a0Pbhn1lUYzqHvR4dcXqgHOkLHeWnYsd/1n9dNzkz7/jcAAP//AykX4A==" } diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 2249f82e226..7ffab2a625f 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -36,8 +36,7 @@ func GetListMetricsOutput(namespace string, regionName string, svcCloudwatch clo // List metrics of a given namespace for each region listMetricsOutput, err := reqListMetrics.Send() if err != nil { - err = errors.Wrap(err, "ListMetricsRequest failed, skipping region "+regionName) - return nil, err + return nil, errors.Wrap(err, "ListMetricsRequest failed, skipping region "+regionName) } if listMetricsOutput.Metrics == nil || len(listMetricsOutput.Metrics) == 0 { @@ -58,8 +57,7 @@ func getMetricDataPerRegion(metricDataQueries []cloudwatch.MetricDataQuery, next reqGetMetricData := svc.GetMetricDataRequest(getMetricDataInput) getMetricDataOutput, err := reqGetMetricData.Send() if err != nil { - err = errors.Wrap(err, "Error GetMetricDataInput") - return nil, err + return nil, errors.Wrap(err, "Error GetMetricDataInput") } return getMetricDataOutput, nil } @@ -70,12 +68,26 @@ func GetMetricDataResults(metricDataQueries []cloudwatch.MetricDataQuery, svc cl getMetricDataOutput := &cloudwatch.GetMetricDataOutput{NextToken: nil} for init || getMetricDataOutput.NextToken != nil { init = false - output, err := getMetricDataPerRegion(metricDataQueries, getMetricDataOutput.NextToken, svc, startTime, endTime) - if err != nil { - err = errors.Wrap(err, "getMetricDataPerRegion failed") - return getMetricDataOutput.MetricDataResults, err + // Split metricDataQueries into smaller slices that length no longer than 100. + // To avoid ValidationError: The collection MetricDataQueries must not have a size greater than 100. + iter := len(metricDataQueries) / 100 + for i := 0; i <= iter; i++ { + metricDataQueriesPartial := metricDataQueries[iter*100:] + if i != iter { + metricDataQueriesPartial = metricDataQueries[i*100 : (i+1)*100-1] + } + + if len(metricDataQueriesPartial) == 0 { + return getMetricDataOutput.MetricDataResults, nil + } + + output, err := getMetricDataPerRegion(metricDataQueriesPartial, getMetricDataOutput.NextToken, svc, startTime, endTime) + if err != nil { + return getMetricDataOutput.MetricDataResults, errors.Wrap(err, "getMetricDataPerRegion failed") + } + + getMetricDataOutput.MetricDataResults = append(getMetricDataOutput.MetricDataResults, output.MetricDataResults...) } - getMetricDataOutput.MetricDataResults = append(getMetricDataOutput.MetricDataResults, output.MetricDataResults...) } return getMetricDataOutput.MetricDataResults, nil } diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index 00bab25d80c..3e539f4f122 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -146,7 +146,18 @@ func TestGetMetricDataResults(t *testing.T) { assert.NoError(t, err) mockSvc := &MockCloudWatchClient{} - metricDataQueries := []cloudwatch.MetricDataQuery{} + metricInfo := cloudwatch.Metric{ + MetricName: &metricName, + Namespace: &namespace, + } + metricStat := cloudwatch.MetricStat{Metric: &metricInfo} + metricDataQueries := []cloudwatch.MetricDataQuery{ + { + Id: &id1, + Label: &label1, + MetricStat: &metricStat, + }, + } getMetricDataResults, err := GetMetricDataResults(metricDataQueries, mockSvc, startTime, endTime) if err != nil { fmt.Println("failed getMetricDataPerRegion: ", err) diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 18a720f84fd..d1e541fb07d 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -1,8 +1,8 @@ - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -10,9 +10,24 @@ - module: aws period: 86400s metricsets: - - "s3_request" - - "s3_daily_storage" + - s3_request + - s3_daily_storage access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - cloudwatch + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS