From e80334c97b2ab99354467e33c2ff9723feb60d86 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 12 Apr 2019 17:08:06 -0600 Subject: [PATCH 01/17] Initial draft for adding cloudwatch metricset --- metricbeat/docs/fields.asciidoc | 6 + metricbeat/docs/modules/aws.asciidoc | 14 ++ .../docs/modules/aws/cloudwatch.asciidoc | 23 +++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/include/list.go | 1 + x-pack/metricbeat/metricbeat.reference.yml | 10 ++ x-pack/metricbeat/module/aws/_meta/config.yml | 10 ++ x-pack/metricbeat/module/aws/aws.go | 31 ++-- .../module/aws/cloudwatch/_meta/data.json | 25 +++ .../module/aws/cloudwatch/_meta/docs.asciidoc | 12 ++ .../module/aws/cloudwatch/_meta/fields.yml | 5 + .../module/aws/cloudwatch/cloudwatch.go | 166 ++++++++++++++++++ .../cloudwatch/cloudwatch_integration_test.go | 54 ++++++ x-pack/metricbeat/module/aws/fields.go | 2 +- x-pack/metricbeat/modules.d/aws.yml.disabled | 10 ++ 15 files changed, 356 insertions(+), 16 deletions(-) create mode 100644 metricbeat/docs/modules/aws/cloudwatch.asciidoc create mode 100644 x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json create mode 100644 x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go create mode 100644 x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c27b1c3000d..481faf4ddcf 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -830,6 +830,12 @@ Total. +[float] +== cloudwatch fields + +`cloudwatch` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by different namespaces. + + [float] == ec2 fields diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index ea919d6ee6a..e3b11ae28d8 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -130,6 +130,16 @@ metricbeat.modules: secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - "cloudwatch" + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/SNS ---- [float] @@ -137,6 +147,8 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> * <> @@ -145,6 +157,8 @@ The following metricsets are available: * <> +include::aws/cloudwatch.asciidoc[] + include::aws/ec2.asciidoc[] include::aws/s3_daily_storage.asciidoc[] diff --git a/metricbeat/docs/modules/aws/cloudwatch.asciidoc b/metricbeat/docs/modules/aws/cloudwatch.asciidoc new file mode 100644 index 00000000000..68df2f75b4c --- /dev/null +++ b/metricbeat/docs/modules/aws/cloudwatch.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-aws-cloudwatch]] +=== aws cloudwatch metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index ae042951010..ee2e22de58c 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,7 +10,8 @@ This file is generated! See scripts/docs_collector.py |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.4+| .4+| |<> +.5+| .5+| |<> beta[] +|<> |<> beta[] |<> beta[] |<> beta[] diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 761052d0df7..635c594e97e 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -9,6 +9,7 @@ package include import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/cloudwatch" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_daily_storage" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_request" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index b59e33b5cdf..8ba26083f30 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -175,6 +175,16 @@ metricbeat.modules: secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - "cloudwatch" + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/SNS #--------------------------------- Ceph Module --------------------------------- - module: ceph diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 18a720f84fd..412fb5784af 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -16,3 +16,13 @@ secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - "cloudwatch" + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/SNS diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index eb40e81c6a1..1d5acccf738 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -20,20 +20,22 @@ import ( // Config defines all required and optional parameters for aws metricsets type Config struct { - Period string `config:"period"` - AccessKeyID string `config:"access_key_id"` - SecretAccessKey string `config:"secret_access_key"` - SessionToken string `config:"session_token"` - DefaultRegion string `config:"default_region"` + Period string `config:"period"` + AccessKeyID string `config:"access_key_id"` + SecretAccessKey string `config:"secret_access_key"` + SessionToken string `config:"session_token"` + DefaultRegion string `config:"default_region"` + CloudwatchMetrics []map[string]interface{} `config:"cloudwatch_metrics"` } // MetricSet is the base metricset for all aws metricsets type MetricSet struct { mb.BaseMetricSet - RegionsList []string - DurationString string - PeriodInSec int - AwsConfig *awssdk.Config + RegionsList []string + DurationString string + PeriodInSec int + AwsConfig *awssdk.Config + CloudwatchMetrics []map[string]interface{} } // ModuleName is the name of this module. @@ -95,11 +97,12 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // Construct MetricSet metricSet := MetricSet{ - BaseMetricSet: base, - RegionsList: regionsList, - DurationString: durationString, - PeriodInSec: periodSec, - AwsConfig: &awsConfig, + BaseMetricSet: base, + RegionsList: regionsList, + DurationString: durationString, + PeriodInSec: periodSec, + AwsConfig: &awsConfig, + CloudwatchMetrics: config.CloudwatchMetrics, } return &metricSet, nil } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json new file mode 100644 index 00000000000..45013dbc67f --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json @@ -0,0 +1,25 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "aws": { + "cloudwatch": { + "InstanceId": "i-06fc36e56436ca041", + "StatusCheckFailed_Instance": 0, + "namespace": "AWS/EC2" + } + }, + "cloud": { + "region": "eu-north-1" + }, + "event": { + "dataset": "aws.cloudwatch", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "cloudwatch" + }, + "service": { + "name": "cloudwatch", + "type": "cloudwatch" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc new file mode 100644 index 00000000000..7e1fa891671 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc @@ -0,0 +1,12 @@ +The cloudwatch metricset of aws module allows you to monitor varies services on +AWS. `cloudwatch` metricset fetches metrics from given namespace periodically +and sto + +[float] +=== AWS Permissions +Some specific AWS permissions are required for IAM user to collect AWS SQS metrics. +---- +ec2:DescribeRegions +cloudwatch:GetMetricData +cloudwatch:ListMetrics +---- diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml new file mode 100644 index 00000000000..a4490ef4934 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml @@ -0,0 +1,5 @@ +- name: cloudwatch + type: group + description: > + `cloudwatch` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by different namespaces. + release: beta diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go new file mode 100644 index 00000000000..3780eaadd61 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -0,0 +1,166 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cloudwatch + +import ( + "fmt" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/aws" + "github.com/pkg/errors" + "strconv" + "strings" +) + +var metricsetName = "cloudwatch" + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(aws.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *aws.MetricSet + Namespace string +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + metricSet, err := aws.NewMetricSet(base) + if err != nil { + return nil, errors.Wrap(err, "error creating aws metricset") + } + + + return &MetricSet{ + MetricSet: metricSet, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + // Get CloudwatchMetrics Config + cloudwatchConfig := m.CloudwatchMetrics + if len(cloudwatchConfig) == 0 { + return errors.New("cloudwatch_metrics in config is missing") + } + // Get startTime and endTime + startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) + if err != nil { + return errors.Wrap(err, "Error ParseDuration") + } + + for _, cw := range cloudwatchConfig { + namespace := cw["namespace"].(string) + for _, regionName := range m.MetricSet.RegionsList { + svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) + if err != nil { + m.Logger().Errorf(err.Error()) + report.Error(err) + continue + } + + if listMetricsOutput == nil || len(listMetricsOutput) == 0 { + continue + } + + // Construct metricDataQueries + metricDataQueries := constructMetricQueries(listMetricsOutput, int64(m.PeriodInSec)) + if len(metricDataQueries) == 0 { + continue + } + + // Use metricDataQueries to make GetMetricData API calls + metricDataResults, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) + if err != nil { + err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName) + m.Logger().Error(err.Error()) + report.Error(err) + continue + } + + // Find a timestamp for all metrics in output + timestamp := aws.FindTimestamp(metricDataResults) + if !timestamp.IsZero() { + for _, output := range metricDataResults { + event := mb.Event{} + event.Service = metricsetName + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", metricsetName) + event.RootFields.Put("cloud.region", regionName) + + if len(output.Values) == 0 { + continue + } + + exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) + if exists { + event.MetricSetFields = common.MapStr{} + event.MetricSetFields.Put("namespace", namespace) + labels := strings.Split(*output.Label, " ") + for i := 0; i < len(labels)/2; i++ { + event.MetricSetFields.Put(labels[i+1], labels[i+2]) + } + if len(output.Values) > timestampIdx { + event.MetricSetFields.Put(labels[0], output.Values[timestampIdx]) + } + } + + if reported := report.Event(event); !reported { + return nil + } + } + } + } + } + + return nil +} + +func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery { + metricDataQueries := []cloudwatch.MetricDataQuery{} + for i, listMetric := range listMetricsOutput { + metricDataQuery := createMetricDataQuery(listMetric, i, period) + metricDataQueries = append(metricDataQueries, metricDataQuery) + } + return metricDataQueries +} + +func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (metricDataQuery cloudwatch.MetricDataQuery) { + statistic := "Average" + id := "cw" + strconv.Itoa(index) + metricDims := metric.Dimensions + metricName := *metric.MetricName + label := metricName + " " + for _, dim := range metricDims { + label += *dim.Name + label += " " + *dim.Value + } + + metricDataQuery = cloudwatch.MetricDataQuery{ + Id: &id, + MetricStat: &cloudwatch.MetricStat{ + Period: &period, + Stat: &statistic, + Metric: &metric, + }, + Label: &label, + } + return +} diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go new file mode 100644 index 00000000000..6537fef7f39 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go @@ -0,0 +1,54 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package cloudwatch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" +) + +func TestFetch(t *testing.T) { + config, info := mtest.GetConfigForTest("cloudwatch", "60s") + if info != "" { + t.Skip("Skipping TestFetch: " + info) + } + + config = addCloudwatchMetricsToConfig(config) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) +} + +func TestData(t *testing.T) { + config, info := mtest.GetConfigForTest("cloudwatch", "300s") + if info != "" { + t.Skip("Skipping TestData: " + info) + } + + config = addCloudwatchMetricsToConfig(config) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + if err := mbtest.WriteEventsReporterV2Error(metricSet, t, "/"); err != nil { + t.Fatal("write", err) + } +} + +func addCloudwatchMetricsToConfig(config map[string]interface{}) map[string]interface{} { + cloudwatchMetricsConfig := []map[string]interface{}{} + cloudwatchMetric := map[string]interface{}{} + cloudwatchMetric["namespace"] = "AWS/SQS" + cloudwatchMetricsConfig = append(cloudwatchMetricsConfig, cloudwatchMetric) + config["cloudwatch_metrics"] = cloudwatchMetricsConfig + return config +} diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 5d5ff0faf27..90a123e21e1 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsWktvIzcSvs+vKOSUAOPGZmayBx8WmHiMjYFs4kQe5NhTIktqrtlkDx+SZcyPXxT7oYdbsix3K5fVwTDEFuv7ih+LVcW+gHtaXQIu/RuAoIKmS/gOl/67NwCSvHCqCsqaS/jXGwCAL7j0X6C0MmoCYbUmETx8/GsCpTUqWKfMHEoKTgkPM2fLNHalbZRLDKLI3gA40oSeLmGObwBmirT0l2n2CzBYUouGP2FV8YPOxqr5pgfU9iSbE5F4133XN9neCevPFxLvvoCwJqAyHkJBHbdQYIAlOQIvHFYkd9j+xWxhWShRrCfo8ZEnE2C6Sj+8vnqXbdjf9lP72aW6SVdUMQs2oM4qEbaeaMl7gZpkPtMWdx844Af+3BUEFTlBJuCcwM4AtbYCA0kGDsKWVQwE0ajQuAcdgYjOkQl6BcpA9ATWJD8q4wMaQdleIsKRVCGPHuc0AhcTyyk55nF1+xlqYx581azHJkaYWZeeikFp9Yg87bO4p6j5t6MiJ3SG5BaB2vFmjb1ADyiEiyTBK/5GBViiB43RiIIkWAc+oAsk95Py0VU6+vyM5BqT28wKXBBMicx6pdBANFqVipXY0V4WZIB/dnX7+SrN8HONGRaoI4Hy8EjOHsvY56JANyc5LuXEqZc47yVjA1SoJEi7NEz96fq/BTSyCTuhiB6UEdGxj1BKxShQQ02ln7qhsLTuPlMmq1DcU/CjMm5sgCNBasFiNBxXWhigTCA3Q0F+d1Mehm9jOCv+FMZtDEPhVyabrgKNCz5ZGMX158I+lNul8vfKZo5QjoL958bT2KQJjLULVT5YR7CwOpbkAReoNE41QbDHI186FWhE6Dx/IMOYBseevG6roYFf2bLSxIdC8rutyKWT25++BJzDIEdpoWaKJOdDykqWY1DlMQs0JstkYZPmiWt1CkkfMESfiYLEfT5DpfcclNqa+cv4/UmVdcHzeR4KcttIObep0HuSMLWh2B6sMUHClE5FHvUrH6jcHlN1RqrRByiVieF4knk935m5jkGktfM3UOlfsWPJdCFGWMd/oumvfPhImJN7daFgHflUEDwf37pRVeKcMtW/J+5ptbRud+wIYDef0qZkGDw/V1eSN3Od2L8E37ouzXgN+kVwEs4bIxXXiGslSApJcZvFsPJAhmPRngqkA1o5tcBAmTQ+56FhHdrMDp9+myTDrXufZBVHolRVvxJ3v34BtJvbxQdO5R15D+i9FSpV4EvVhL8XY41TrcRYDk2TP/HnkapsoA3oxdZxDY5rDi5KwM1tN/I9O/gHmNpoZHswvtSlaQtlwsp+b54ciNK8uz58C1zfw4//vJiqANF4NTepDk5GjkI6/Lr3IoXvKzKSt/s3cNGY+j9fxBCUmV+kmvYbBHKlMknT3zhjqarmOf6X5A/PMAoFJ3s+r8jlHKrHOgoaO5wcdcdC9qT96N/nEpVe5Zx6bbexXt6L3J1spzGZxqAZG7VNOXnf26WcUji2TzmNXDUPLLzfmgiDMHnfWjhk3avHcSqmFGlKTkQYjsSAnMw0BSvn4LJOsQ9hrJWW21lup/8lMUoXI3WKNzTdWOrBmPIdriVaeQnN6ZvvVbyjr5F8eK3Wm2k2VN58839tH9BN4yNfXwMMlMz3ieWXu7vbzhqUKFMdiQY+lvhozRrnW3A0Ryd1e/Cuqj0HaId9Tv0p/GnIdzD/+/puBzeLu9U+i/4ph2fwVnFEvLefB8crSdOeJH8QyJ+uf72+ux4adUE4VFOhB/Mv1x8/HaXn57Rg/Zhi+H2yq4aTUHrStOeG8LU410gm179eX93B72nR4cqawIF2YFXUTHIv0Bgap3nb13huD/bGbl1/HU39NUwdhej+Dqqt4XNw1WrMXdRhSxkE20oZtKih+/qgPYSTaxVtUY6/CvUSrO2lDXPcsbssODViYo58ZY3nUkzoKIkLxqmVq35ysTontdZavRZNmgbYJXuM8+3LIx05Z53PPjw8jCejDw8PILRitSdzXf/SSjpqjeqdhM2NtJ0BqdQi+wdYBz8eJPbTmMR+engAT25B7ozENAYyYpXNlPMhZ3FkZb/6TuNYkbtoRRVUSXWxUO/7+gplrTni8qC7F63fMXjCMdj6JYOtHZZepEhXL1PqIuZhwimzbsudYTmTxsrXVzh7uCdvp6245tu0vVLnPY2kAum53ddVgF/960q/r/6Mr1xN/pi8tuCzWpIPeUne45xynFPmSQy4ilhVzj6oEgNB8+oVu6W2C8aaizqhl9BgaO9OvkaKe2qt5slUC+BqsNu6u+1o0lrZArR+n6WxnS7njA0bV5L1KYepW6jKkqTCQHrPgdVxMTbkC+XVVI9T3HR0OgbKwEyrebHnFOqQnQXVrvuCU7RAvd7sR+qBpTQu0lavL0LWxqdxoXVZ7nQFArX2bTj8szb/n2aLodj/FmAHmSPNyGsuZR2x8ZAPqazCKm8cOOQBs0a0456Ptzet+3ivSFXv8Nq7gC2BPffTZNbx9Jk2bQ/umXUlhkvo+9ExlxfqkU7wcT00bNtv8sekiZlp3v8FAAD//26rI9M=" + return "eJzsWktvG0cSvvtXFHJKAGuwsZ096LCAIwsbAdlECWXkOC52Fzm96uke94MUBf/4RfU8+NCQoqgZ5rI8GDJ72PV9VdX16rmAe1pdAi79G4CggqZL+A6X/rs3AJK8cKoKyppL+NcbAIAvuPRfoLQyagJhtSYRPHz8awKlNSpYp8wcSgpOCQ8zZ8u0dqVtlEsMosjeADjShJ4uYY5vAGaKtPSXafcLMFhSi4Y/YVXxg87GqvmmB9T2JpsbiU5wt9S3595968+X9TZfQFgTUBkPoaCOaSgwwJIcgRcOK5I73P/i38KyUKJYb9CjMU8mwHQFUs1m5Pg/zMNXKMhnG5g6FU4p4BPWJN69ii6Jd2fiyT+8vnrXS22OG9/uGnjLyFXMgg2os0qErSda8l6gJpnPtMXdBw7ogT93BUFFTpAJOCewM0CtrcBAkoGDsGUVA0E0KjTqQUcgomPr6RUoA9ETWJP0qIwPaARle4kIR1KFPHqc0whcTCyn5JjH1e1nqIV58FVjj02MMLMuPRWD0uoRedtncU9R829HRU7oDMktArXizRp7gR5QCBdJglf8jQqwRA8aoxEFSbAOfEAXSO4n5aOrdPT5Gck1IreZFbggmBKZtaXQQDRalYo9saO9LMgA/+zq9vNV2uHnGjMsUEcC5eGRnD2Wsc9FgW5OclzKiVMvcT5LxgaoUEmQdmmY+lP7vwU0sgk7oYgelBHRsY5QSsUoUENNpZ+6obC07j5TJqtQ3FPwozJuZIAjQWrBzmg4rrQwQJlAbsZBf/dQHoZvYzgr/hTGbQxD4Vcmm64CjQs+SRhF9efCPpTapfL3ymaOUI6C/edG09iUCYy1C1U+WEewsDqW5AEXqDRONUGwxyNfOhVoROi8fyDDmAbHnrRuq6GBX9my0sRJIendVuRS5vanm4BrGOQoLdRMkeR6SFnJ7hhUeYyBxmSZJGzSPNFWp5D0AUP0mShI3OczVHpPotTWzF/G70+qrAue83koyG0j5dqmQu9JwtSGYnuxxgQJU8qKvOpXPlC5vabqilSjD1AqE8PxJPN6vzNzHYNIK+dvoNJvsWPJdCFGWMf/RNPf+XBKmJN7daNgHfnUEDwf37pVVeKcMtV/Ju5ptbRud+0IYDef0qFkGLw/d1eSD3Nd2L8E37ovzdgG/U5wEs4bIxX3iGtPkBSSx202w8oDGY5FezqQDmjl1AIDZdL4nJeGVWizO3z6bZIEt+p9UlUciVJV/Z64+/ULoN3cLj5wKe/Ie0DvrVCpA1+qJvy9GGucaiXGUmja/Ik+j/TKBtqAWmwV1+C45uCiBNzcdivfs4J/gKmNRraJ8aUqTUcoE1b2a/PkQJT23dXhW+D+Hn7858VUBYjGq7lJfXASchTS4e3eixS+r8hIPu7fwEVj6r98EUNQZn6RetpvEMiVyiSf/sYVS1U1z/GfJH94hlEouNjzeUUu51A9Vipo5HBx1KWF7Mn40b/PJSq9yrn02h5jvXwWubvZzmAyrUGzNuqYcvL++QHs4TnlNHLXPLDj/dZEGITJ+1bCIelePY7TMaVIU3IhwnAkBuRipmlYuQaXdYl9CGPtabmd5Xb6XxKjTDHSpHjDpxtJPRhTvcO9ROteQnP55ns93tHXSD681tebbTa8vPnm/759wG8aHfn6GmCgYr7PWX65u7vtpEGJMvWRaOBjiY/WrHG+BUdzdFK3iXdV7UmgHfY59ZfwpyHfwfzv67sd3Ozcre+z0z/l8AzeKo6I9/bz4HgladpT5A8C+dP1r9d310OjLgiHGir0YP7l+uOno/z5OV+wfkxn+H2y6w0nofSkac8N4WtxrpFMrn+9vrqD35PR4cqawIF2YK+omeReoDE0zvC2b/DcJvZGbt1/HU39NUwdhej+Dqqt4HNw1WrMU9RhSxUEy0oVtKih+zrRHsLJvYq2KMe3Qm2Ctbx0YI5Lu8uCSyMm5shX1nhuxYSOkrhhnFq56icXq3NSa6XVtmjKNMCu2GOcb18e6cg563z24eFhPDf68PAAQiv29iSum19aSUfZqD5J2NxI2xmQSiOyf4B18ONBYj+NSeynhwfw5BbkzkhMYyAjVtlMOR9ydo6s7Pe+0zhW5C5apwqqpLpZqM99fYWy9jni9qC7F63fMXjCMdj6JYOtE5ZepEhXL1PqIuZhwqmybtudYTmTxsrXVzh7uCdtp6O45tuMvdLkPa2kBum509d1gF/961q/r/6Mr1xN/pi8tuGzWpIPeUne45xynFPmSQxoRawqZx9UiYGgefWK1VLLBWPNRV3QS2gwtHcnXyPFPb1W82TqBXA12G3d3XY0aaVsAVq/z9LITpdzxoaNK8k6y2GaFqqyJKkwkN6TsDouxoZ8obya6nGam45Ox0AZmGk1L/ZkoQ7ZWVDtqi84RQvU68N+pD+wK42LtPXXFyFr49O40Loqd7oCgVr7Nhz+WYv/T3PEUOx/C7CDzJFmZJtLWUdsPKRDKquwyhsFDplg1oh21PPx9qZVH58VqeoTXmsXsCWw536azDqePjOm7cE9s67EcAl9Pzrm8kI90gk6rpeGHftN/pg0MTPt+78AAAD//3QEcdg=" } diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 18a720f84fd..412fb5784af 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -16,3 +16,13 @@ secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 300s + metricsets: + - "cloudwatch" + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' + cloudwatch_metrics: + - namespace: AWS/SNS From 0351f1c2a7cd7ed5fd2dd9b4418a4eada6547748 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 15 Apr 2019 16:34:59 -0600 Subject: [PATCH 02/17] Combine metrics into same events by identifiers(instanceID, queueName,...) --- .../module/aws/cloudwatch/_meta/data.json | 17 ++- .../module/aws/cloudwatch/cloudwatch.go | 128 +++++++++++++++--- 2 files changed, 123 insertions(+), 22 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json index 45013dbc67f..648cd27eba7 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json @@ -2,13 +2,28 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "aws": { "cloudwatch": { + "CPUCreditBalance": 144, + "CPUCreditUsage": 0.002752, + "CPUSurplusCreditBalance": 0, + "CPUSurplusCreditsCharged": 0, + "CPUUtilization": 0.066666666666606, + "DiskReadBytes": 0, + "DiskReadOps": 0, + "DiskWriteBytes": 0, + "DiskWriteOps": 0, "InstanceId": "i-06fc36e56436ca041", + "NetworkIn": 1865, + "NetworkOut": 2527.4, + "NetworkPacketsIn": 14.2, + "NetworkPacketsOut": 13.8, + "StatusCheckFailed": 0, "StatusCheckFailed_Instance": 0, + "StatusCheckFailed_System": 0, "namespace": "AWS/EC2" } }, "cloud": { - "region": "eu-north-1" + "region": "us-west-1" }, "event": { "dataset": "aws.cloudwatch", diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 3780eaadd61..8f84ed7b611 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -5,14 +5,16 @@ package cloudwatch import ( - "fmt" + "strconv" + "strings" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/x-pack/metricbeat/module/aws" - "github.com/pkg/errors" - "strconv" - "strings" ) var metricsetName = "cloudwatch" @@ -39,12 +41,12 @@ type MetricSet struct { // New creates a new instance of the MetricSet. New is responsible for unpacking // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The aws cloudwatch metricset is beta.") metricSet, err := aws.NewMetricSet(base) if err != nil { return nil, errors.Wrap(err, "error creating aws metricset") } - return &MetricSet{ MetricSet: metricSet, }, nil @@ -68,6 +70,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, cw := range cloudwatchConfig { namespace := cw["namespace"].(string) for _, regionName := range m.MetricSet.RegionsList { + m.MetricSet.AwsConfig.Region = regionName svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { @@ -95,36 +98,46 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } + // Get IdentifierName + identifier := getIdentifierName(listMetricsOutput) + + // Get IdentifierValues + identifierValues := getIdentifierValues(listMetricsOutput, identifier) + + // Initialize events map per region, which stores one event per identifierValue(eg: InstanceId, BucketName,...) + events := map[string]mb.Event{} + for _, idValue := range identifierValues { + events[idValue] = initEvent(regionName) + } + // Find a timestamp for all metrics in output timestamp := aws.FindTimestamp(metricDataResults) if !timestamp.IsZero() { for _, output := range metricDataResults { - event := mb.Event{} - event.Service = metricsetName - event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", metricsetName) - event.RootFields.Put("cloud.region", regionName) - if len(output.Values) == 0 { continue } exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) if exists { - event.MetricSetFields = common.MapStr{} - event.MetricSetFields.Put("namespace", namespace) labels := strings.Split(*output.Label, " ") - for i := 0; i < len(labels)/2; i++ { - event.MetricSetFields.Put(labels[i+1], labels[i+2]) - } - if len(output.Values) > timestampIdx { - event.MetricSetFields.Put(labels[0], output.Values[timestampIdx]) + identifierValue := getIdentifierFromLabels(identifier, labels) + if identifierValue != "" { + events[identifierValue] = insertMetricSetFields(events[identifierValue], namespace, output.Values[timestampIdx], labels) + } else { + eventNew := initEvent(regionName) + eventNew = insertMetricSetFields(eventNew, namespace, output.Values[timestampIdx], labels) + if reported := report.Event(eventNew); !reported { + return nil + } } } + } + } - if reported := report.Event(event); !reported { - return nil - } + for _, event := range events { + if reported := report.Event(event); !reported { + return nil } } } @@ -164,3 +177,76 @@ func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (m } return } + +func getIdentifierName(listMetricsOutputs []cloudwatch.Metric) string { + if len(listMetricsOutputs) > 0 { + if len(listMetricsOutputs[0].Dimensions) == 0 { + return *listMetricsOutputs[0].Dimensions[0].Name + } else { + for _, dim := range listMetricsOutputs[0].Dimensions { + switch *dim.Name { + case "BucketName": + return "BucketName" + case "InstanceId": + return "InstanceId" + case "TopicName": + return "TopicName" + default: + return "" + } + } + } + } + return "" +} + +func getIdentifierValues(listMetricsOutputs []cloudwatch.Metric, identifierName string) (identifierValues []string) { + for _, output := range listMetricsOutputs { + for _, dim := range output.Dimensions { + if *dim.Name == identifierName { + if aws.StringInSlice(*dim.Value, identifierValues) { + continue + } + identifierValues = append(identifierValues, *dim.Value) + } + } + } + return +} + +func initEvent(regionName string) mb.Event { + event := mb.Event{} + event.Service = metricsetName + event.RootFields = common.MapStr{} + event.MetricSetFields = common.MapStr{} + event.RootFields.Put("service.name", metricsetName) + event.RootFields.Put("cloud.region", regionName) + return event +} + +func getIdentifierFromLabels(identifier string, labels []string) string { + identifierValue := "" + if len(labels) <= 2 { + return identifierValue + } + for i := 0; i < len(labels)/2; i++ { + if labels[i+1] == identifier { + identifierValue = labels[i+2] + break + } + } + return identifierValue +} + +func insertMetricSetFields(event mb.Event, namespace string, metricValue float64, labels []string) mb.Event { + event.MetricSetFields.Put("namespace", namespace) + event.MetricSetFields.Put(labels[0], metricValue) + if len(labels) <= 2 { + return event + } + + for i := 0; i < len(labels)/2; i++ { + event.MetricSetFields.Put(labels[i+1], labels[i+2]) + } + return event +} From 559822485537b1c98ad0d12f104239401c0c61de Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 16 Apr 2019 15:03:48 -0600 Subject: [PATCH 03/17] Split metricDataQueries into 100 length in GetMetricDataResults --- .../module/aws/cloudwatch/_meta/data.json | 14 +- .../module/aws/cloudwatch/cloudwatch.go | 70 ++++----- .../cloudwatch/cloudwatch_integration_test.go | 4 +- .../module/aws/cloudwatch/cloudwatch_test.go | 136 ++++++++++++++++++ x-pack/metricbeat/module/aws/utils.go | 21 ++- 5 files changed, 197 insertions(+), 48 deletions(-) create mode 100644 x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json index 648cd27eba7..b0f888a7a5d 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json @@ -3,19 +3,19 @@ "aws": { "cloudwatch": { "CPUCreditBalance": 144, - "CPUCreditUsage": 0.002752, + "CPUCreditUsage": 0.001614, "CPUSurplusCreditBalance": 0, "CPUSurplusCreditsCharged": 0, - "CPUUtilization": 0.066666666666606, + "CPUUtilization": 0.0333333333331514, "DiskReadBytes": 0, "DiskReadOps": 0, "DiskWriteBytes": 0, "DiskWriteOps": 0, - "InstanceId": "i-06fc36e56436ca041", - "NetworkIn": 1865, - "NetworkOut": 2527.4, - "NetworkPacketsIn": 14.2, - "NetworkPacketsOut": 13.8, + "InstanceId": "i-07ddaa40ed5c79161", + "NetworkIn": 1101.8, + "NetworkOut": 1423.8, + "NetworkPacketsIn": 9, + "NetworkPacketsOut": 8.2, "StatusCheckFailed": 0, "StatusCheckFailed_Instance": 0, "StatusCheckFailed_System": 0, diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 8f84ed7b611..47dfb753825 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -17,7 +17,11 @@ import ( "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) -var metricsetName = "cloudwatch" +var ( + metricsetName = "cloudwatch" + suppportedIdentifiers = []string{"InstanceId", "BucketName", "QueueName", + "TopicName"} +) // init registers the MetricSet with the central registry as soon as the program // starts. The New function will be called later to instantiate an instance of @@ -98,13 +102,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - // Get IdentifierName - identifier := getIdentifierName(listMetricsOutput) - - // Get IdentifierValues - identifierValues := getIdentifierValues(listMetricsOutput, identifier) + // Get identifier name and values + identifier, identifierValues := getIdentifiers(listMetricsOutput) - // Initialize events map per region, which stores one event per identifierValue(eg: InstanceId, BucketName,...) + // Initialize events map per region, which stores one event per identifierValue events := map[string]mb.Event{} for _, idValue := range identifierValues { events[idValue] = initEvent(regionName) @@ -155,16 +156,22 @@ func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) return metricDataQueries } -func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (metricDataQuery cloudwatch.MetricDataQuery) { - statistic := "Average" - id := "cw" + strconv.Itoa(index) +func constructLabel(metric cloudwatch.Metric) string { metricDims := metric.Dimensions metricName := *metric.MetricName - label := metricName + " " + label := metricName for _, dim := range metricDims { + label += " " label += *dim.Name label += " " + *dim.Value } + return label +} + +func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (metricDataQuery cloudwatch.MetricDataQuery) { + statistic := "Average" + id := "cw" + strconv.Itoa(index) + label := constructLabel(metric) metricDataQuery = cloudwatch.MetricDataQuery{ Id: &id, @@ -178,29 +185,24 @@ func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (m return } -func getIdentifierName(listMetricsOutputs []cloudwatch.Metric) string { - if len(listMetricsOutputs) > 0 { - if len(listMetricsOutputs[0].Dimensions) == 0 { - return *listMetricsOutputs[0].Dimensions[0].Name - } else { - for _, dim := range listMetricsOutputs[0].Dimensions { - switch *dim.Name { - case "BucketName": - return "BucketName" - case "InstanceId": - return "InstanceId" - case "TopicName": - return "TopicName" - default: - return "" - } - } +func getIdentifiers(listMetricsOutputs []cloudwatch.Metric) (string, []string) { + if len(listMetricsOutputs) == 0 { + return "", nil + } + + if len(listMetricsOutputs[0].Dimensions) == 0 { + return "", nil + } + + identifierName := "" + identifierValues := []string{} + for _, dim := range listMetricsOutputs[0].Dimensions { + if aws.StringInSlice(*dim.Name, suppportedIdentifiers) { + identifierName = *dim.Name + break } } - return "" -} -func getIdentifierValues(listMetricsOutputs []cloudwatch.Metric, identifierName string) (identifierValues []string) { for _, output := range listMetricsOutputs { for _, dim := range output.Dimensions { if *dim.Name == identifierName { @@ -211,7 +213,7 @@ func getIdentifierValues(listMetricsOutputs []cloudwatch.Metric, identifierName } } } - return + return identifierName, identifierValues } func initEvent(regionName string) mb.Event { @@ -226,7 +228,7 @@ func initEvent(regionName string) mb.Event { func getIdentifierFromLabels(identifier string, labels []string) string { identifierValue := "" - if len(labels) <= 2 { + if len(labels) <= 1 { return identifierValue } for i := 0; i < len(labels)/2; i++ { @@ -241,7 +243,7 @@ func getIdentifierFromLabels(identifier string, labels []string) string { func insertMetricSetFields(event mb.Event, namespace string, metricValue float64, labels []string) mb.Event { event.MetricSetFields.Put("namespace", namespace) event.MetricSetFields.Put(labels[0], metricValue) - if len(labels) <= 2 { + if len(labels) <= 1 { return event } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go index 6537fef7f39..2841b3005c6 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// +build integration package cloudwatch @@ -47,7 +47,7 @@ func TestData(t *testing.T) { func addCloudwatchMetricsToConfig(config map[string]interface{}) map[string]interface{} { cloudwatchMetricsConfig := []map[string]interface{}{} cloudwatchMetric := map[string]interface{}{} - cloudwatchMetric["namespace"] = "AWS/SQS" + cloudwatchMetric["namespace"] = "AWS/EC2" cloudwatchMetricsConfig = append(cloudwatchMetricsConfig, cloudwatchMetric) config["cloudwatch_metrics"] = cloudwatchMetricsConfig return config diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go new file mode 100644 index 00000000000..9ace9528ca4 --- /dev/null +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -0,0 +1,136 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package cloudwatch + +import ( + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/stretchr/testify/assert" +) + +var ( + instanceID1 = "i-1" + instanceID2 = "i-2" + namespace = "AWS/EC2" + dimName = "InstanceId" + metricName1 = "CPUUtilization" + metricName2 = "StatusCheckFailed" + metricName3 = "StatusCheckFailed_System" + metricName4 = "StatusCheckFailed_Instance" + listMetric1 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName, + Value: &instanceID1, + }}, + MetricName: &metricName1, + Namespace: &namespace, + } + + listMetric2 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName, + Value: &instanceID1, + }}, + MetricName: &metricName2, + Namespace: &namespace, + } + + listMetric3 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName, + Value: &instanceID2, + }}, + MetricName: &metricName3, + Namespace: &namespace, + } + + listMetric4 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName, + Value: &instanceID2, + }}, + MetricName: &metricName4, + Namespace: &namespace, + } + + listMetric5 = cloudwatch.Metric{ + MetricName: &metricName1, + Namespace: &namespace, + } +) + +func TestGetIdentifiers(t *testing.T) { + listMetricsOutput := []cloudwatch.Metric{listMetric1, listMetric2, listMetric3, listMetric4} + identifierName, identifierValues := getIdentifiers(listMetricsOutput) + assert.Equal(t, "InstanceId", identifierName) + assert.Equal(t, []string{instanceID1, instanceID2}, identifierValues) +} + +func TestConstructLabel(t *testing.T) { + cases := []struct { + listMetric cloudwatch.Metric + expectedLabel string + }{ + { + listMetric1, + "CPUUtilization InstanceId i-1", + }, + { + listMetric2, + "StatusCheckFailed InstanceId i-1", + }, + { + listMetric3, + "StatusCheckFailed_System InstanceId i-2", + }, + { + listMetric4, + "StatusCheckFailed_Instance InstanceId i-2", + }, + { + listMetric5, + "CPUUtilization", + }, + } + + for _, c := range cases { + label := constructLabel(c.listMetric) + assert.Equal(t, c.expectedLabel, label) + } +} + +func TestGetIdentifierFromLabels(t *testing.T) { + cases := []struct { + label string + expectedIdentifier string + }{ + {"CPUUtilization InstanceId i-1", + "i-1", + }, + {"StatusCheckFailed InstanceId i-1", + "i-1", + }, + {"StatusCheckFailed_System InstanceId i-2", + "i-2", + }, + {"StatusCheckFailed_Instance InstanceId i-2", + "i-2", + }, + { + "CPUUtilization", + "", + }, + } + + for _, c := range cases { + labels := strings.Split(c.label, " ") + identifierValue := getIdentifierFromLabels("InstanceId", labels) + assert.Equal(t, c.expectedIdentifier, identifierValue) + } +} diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 2249f82e226..744e5b20daf 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -70,12 +70,23 @@ func GetMetricDataResults(metricDataQueries []cloudwatch.MetricDataQuery, svc cl getMetricDataOutput := &cloudwatch.GetMetricDataOutput{NextToken: nil} for init || getMetricDataOutput.NextToken != nil { init = false - output, err := getMetricDataPerRegion(metricDataQueries, getMetricDataOutput.NextToken, svc, startTime, endTime) - if err != nil { - err = errors.Wrap(err, "getMetricDataPerRegion failed") - return getMetricDataOutput.MetricDataResults, err + // Split metricDataQueries into smaller slices that length no longer than 100. + // To avoid ValidationError: The collection MetricDataQueries must not have a size greater than 100. + iter := len(metricDataQueries) / 100 + for i := 0; i <= iter; i++ { + metricDataQueriesPartial := metricDataQueries[iter*100:] + if i != iter { + metricDataQueriesPartial = metricDataQueries[i*100 : (i+1)*100-1] + } + + output, err := getMetricDataPerRegion(metricDataQueriesPartial, getMetricDataOutput.NextToken, svc, startTime, endTime) + if err != nil { + err = errors.Wrap(err, "getMetricDataPerRegion failed") + return getMetricDataOutput.MetricDataResults, err + } + + getMetricDataOutput.MetricDataResults = append(getMetricDataOutput.MetricDataResults, output.MetricDataResults...) } - getMetricDataOutput.MetricDataResults = append(getMetricDataOutput.MetricDataResults, output.MetricDataResults...) } return getMetricDataOutput.MetricDataResults, nil } From 84a64b307b3026fa25577bc50001aa7039dc8ada Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 17 Apr 2019 12:08:41 -0600 Subject: [PATCH 04/17] Construct labels by all dimensions from ListMetricsOutput --- .../module/aws/cloudwatch/_meta/data.json | 54 ++++++--- .../module/aws/cloudwatch/cloudwatch.go | 110 +++++++++--------- .../cloudwatch/cloudwatch_integration_test.go | 4 +- .../module/aws/cloudwatch/cloudwatch_test.go | 36 +----- 4 files changed, 96 insertions(+), 108 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json index b0f888a7a5d..1aab15061ee 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json @@ -2,28 +2,44 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "aws": { "cloudwatch": { - "CPUCreditBalance": 144, - "CPUCreditUsage": 0.001614, - "CPUSurplusCreditBalance": 0, - "CPUSurplusCreditsCharged": 0, - "CPUUtilization": 0.0333333333331514, - "DiskReadBytes": 0, - "DiskReadOps": 0, - "DiskWriteBytes": 0, - "DiskWriteOps": 0, - "InstanceId": "i-07ddaa40ed5c79161", - "NetworkIn": 1101.8, - "NetworkOut": 1423.8, - "NetworkPacketsIn": 9, - "NetworkPacketsOut": 8.2, - "StatusCheckFailed": 0, - "StatusCheckFailed_Instance": 0, - "StatusCheckFailed_System": 0, - "namespace": "AWS/EC2" + "ActiveTransactions": 0, + "AuroraBinlogReplicaLag": 0, + "AuroraReplicaLag": 17.2328, + "AuroraReplicaLagMaximum": 19.85419979095459, + "AuroraReplicaLagMinimum": 19.85419979095459, + "BinLogDiskUsage": 0, + "BlockedTransactions": 0, + "BufferCacheHitRatio": 100, + "CPUUtilization": 3, + "CommitLatency": 3.800633333333333, + "CommitThroughput": 0.2499866861369206, + "DBClusterIdentifier": "test1-cluster", + "DDLLatency": 0, + "DDLThroughput": 0, + "DMLLatency": 0.07957666666666667, + "DMLThroughput": 0.2499866861369206, + "DatabaseConnections": 0, + "Deadlocks": 0, + "DeleteLatency": 0, + "DeleteThroughput": 0, + "EngineUptime": 1284587, + "FreeLocalStorage": 32815485747.2, + "FreeableMemory": 4826584268.8, + "InsertLatency": 0.07957666666666667, + "InsertThroughput": 0.2499866861369206, + "NetworkReceiveThroughput": 0.7699930051722573, + "NetworkThroughput": 1.5399860103445147, + "NetworkTransmitThroughput": 0.7699930051722573, + "Queries": 7.515309216451994, + "ResultSetCacheHitRatio": 0, + "SelectLatency": 0.201887653138847, + "SelectThroughput": 2.8182418034367194, + "UpdateLatency": 0, + "namespace": "AWS/RDS" } }, "cloud": { - "region": "us-west-1" + "region": "us-east-2" }, "event": { "dataset": "aws.cloudwatch", diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 47dfb753825..1598362ea24 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -18,9 +18,7 @@ import ( ) var ( - metricsetName = "cloudwatch" - suppportedIdentifiers = []string{"InstanceId", "BucketName", "QueueName", - "TopicName"} + metricsetName = "cloudwatch" ) // init registers the MetricSet with the central registry as soon as the program @@ -87,6 +85,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } + // get identifiers from listMetrics + identifiers := getIdentifiers(listMetricsOutput) + // Construct metricDataQueries metricDataQueries := constructMetricQueries(listMetricsOutput, int64(m.PeriodInSec)) if len(metricDataQueries) == 0 { @@ -102,13 +103,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - // Get identifier name and values - identifier, identifierValues := getIdentifiers(listMetricsOutput) - // Initialize events map per region, which stores one event per identifierValue events := map[string]mb.Event{} - for _, idValue := range identifierValues { - events[idValue] = initEvent(regionName) + for _, values := range identifiers { + for _, v := range values { + events[v] = initEvent(regionName) + } } // Find a timestamp for all metrics in output @@ -122,8 +122,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) if exists { labels := strings.Split(*output.Label, " ") - identifierValue := getIdentifierFromLabels(identifier, labels) - if identifierValue != "" { + if len(labels) == 3 { + identifierValue := labels[2] events[identifierValue] = insertMetricSetFields(events[identifierValue], namespace, output.Values[timestampIdx], labels) } else { eventNew := initEvent(regionName) @@ -137,8 +137,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } for _, event := range events { - if reported := report.Event(event); !reported { - return nil + if len(event.MetricSetFields) != 0 { + if reported := report.Event(event); !reported { + return nil + } } } } @@ -160,10 +162,20 @@ func constructLabel(metric cloudwatch.Metric) string { metricDims := metric.Dimensions metricName := *metric.MetricName label := metricName - for _, dim := range metricDims { - label += " " - label += *dim.Name - label += " " + *dim.Value + dimNames := "" + dimValues := "" + for i, dim := range metricDims { + dimNames += *dim.Name + dimValues += *dim.Value + if i != len(metricDims)-1 { + dimNames += "," + dimValues += "," + } + } + + if dimNames != "" && dimValues != "" { + label += " " + dimNames + label += " " + dimValues } return label } @@ -185,35 +197,39 @@ func createMetricDataQuery(metric cloudwatch.Metric, index int, period int64) (m return } -func getIdentifiers(listMetricsOutputs []cloudwatch.Metric) (string, []string) { +func getIdentifiers(listMetricsOutputs []cloudwatch.Metric) map[string][]string { if len(listMetricsOutputs) == 0 { - return "", nil + return nil } - if len(listMetricsOutputs[0].Dimensions) == 0 { - return "", nil - } + identifiers := map[string][]string{} + for _, listMetrics := range listMetricsOutputs { + identifierName := "" + identifierValue := "" + if len(listMetrics.Dimensions) == 0 { + continue + } + + for i, dim := range listMetrics.Dimensions { + identifierName += *dim.Name + identifierValue += *dim.Value + if i != len(listMetrics.Dimensions)-1 { + identifierName += "," + identifierValue += "," + } - identifierName := "" - identifierValues := []string{} - for _, dim := range listMetricsOutputs[0].Dimensions { - if aws.StringInSlice(*dim.Name, suppportedIdentifiers) { - identifierName = *dim.Name - break } - } - for _, output := range listMetricsOutputs { - for _, dim := range output.Dimensions { - if *dim.Name == identifierName { - if aws.StringInSlice(*dim.Value, identifierValues) { - continue - } - identifierValues = append(identifierValues, *dim.Value) + if identifiers[identifierName] != nil { + if !aws.StringInSlice(identifierValue, identifiers[identifierName]) { + identifiers[identifierName] = append(identifiers[identifierName], identifierValue) } + } else { + identifiers[identifierName] = []string{identifierValue} } } - return identifierName, identifierValues + + return identifiers } func initEvent(regionName string) mb.Event { @@ -226,29 +242,17 @@ func initEvent(regionName string) mb.Event { return event } -func getIdentifierFromLabels(identifier string, labels []string) string { - identifierValue := "" - if len(labels) <= 1 { - return identifierValue - } - for i := 0; i < len(labels)/2; i++ { - if labels[i+1] == identifier { - identifierValue = labels[i+2] - break - } - } - return identifierValue -} - func insertMetricSetFields(event mb.Event, namespace string, metricValue float64, labels []string) mb.Event { event.MetricSetFields.Put("namespace", namespace) event.MetricSetFields.Put(labels[0], metricValue) - if len(labels) <= 1 { + if len(labels) == 1 { return event } - for i := 0; i < len(labels)/2; i++ { - event.MetricSetFields.Put(labels[i+1], labels[i+2]) + dimNames := strings.Split(labels[1], ",") + dimValues := strings.Split(labels[2], ",") + for i := 0; i < len(dimNames); i++ { + event.MetricSetFields.Put(dimNames[i], dimValues[i]) } return event } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go index 2841b3005c6..c76808d6de7 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_integration_test.go @@ -16,7 +16,7 @@ import ( ) func TestFetch(t *testing.T) { - config, info := mtest.GetConfigForTest("cloudwatch", "60s") + config, info := mtest.GetConfigForTest("cloudwatch", "300s") if info != "" { t.Skip("Skipping TestFetch: " + info) } @@ -47,7 +47,7 @@ func TestData(t *testing.T) { func addCloudwatchMetricsToConfig(config map[string]interface{}) map[string]interface{} { cloudwatchMetricsConfig := []map[string]interface{}{} cloudwatchMetric := map[string]interface{}{} - cloudwatchMetric["namespace"] = "AWS/EC2" + cloudwatchMetric["namespace"] = "AWS/RDS" cloudwatchMetricsConfig = append(cloudwatchMetricsConfig, cloudwatchMetric) config["cloudwatch_metrics"] = cloudwatchMetricsConfig return config diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 9ace9528ca4..1f6ad7b2497 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -7,7 +7,6 @@ package cloudwatch import ( - "strings" "testing" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" @@ -67,9 +66,8 @@ var ( func TestGetIdentifiers(t *testing.T) { listMetricsOutput := []cloudwatch.Metric{listMetric1, listMetric2, listMetric3, listMetric4} - identifierName, identifierValues := getIdentifiers(listMetricsOutput) - assert.Equal(t, "InstanceId", identifierName) - assert.Equal(t, []string{instanceID1, instanceID2}, identifierValues) + identifiers := getIdentifiers(listMetricsOutput) + assert.Equal(t, []string{instanceID1, instanceID2}, identifiers["InstanceId"]) } func TestConstructLabel(t *testing.T) { @@ -104,33 +102,3 @@ func TestConstructLabel(t *testing.T) { assert.Equal(t, c.expectedLabel, label) } } - -func TestGetIdentifierFromLabels(t *testing.T) { - cases := []struct { - label string - expectedIdentifier string - }{ - {"CPUUtilization InstanceId i-1", - "i-1", - }, - {"StatusCheckFailed InstanceId i-1", - "i-1", - }, - {"StatusCheckFailed_System InstanceId i-2", - "i-2", - }, - {"StatusCheckFailed_Instance InstanceId i-2", - "i-2", - }, - { - "CPUUtilization", - "", - }, - } - - for _, c := range cases { - labels := strings.Split(c.label, " ") - identifierValue := getIdentifierFromLabels("InstanceId", labels) - assert.Equal(t, c.expectedIdentifier, identifierValue) - } -} From 208cad74f7c92f711f6a3d42cb76e56793686b43 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 18 Apr 2019 17:16:03 -0600 Subject: [PATCH 05/17] Add support to read metricname, dimensions from config for cloudwatch --- x-pack/metricbeat/metricbeat.reference.yml | 7 +- x-pack/metricbeat/module/aws/_meta/config.yml | 7 +- .../module/aws/cloudwatch/_meta/docs.asciidoc | 2 +- .../module/aws/cloudwatch/cloudwatch.go | 198 ++++++++++++------ .../module/aws/cloudwatch/cloudwatch_test.go | 10 +- x-pack/metricbeat/modules.d/aws.yml.disabled | 7 +- 6 files changed, 155 insertions(+), 76 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 8ba26083f30..2dde7234df6 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -184,7 +184,12 @@ metricbeat.modules: session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' cloudwatch_metrics: - - namespace: AWS/SNS + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS #--------------------------------- Ceph Module --------------------------------- - module: ceph diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 412fb5784af..b4f3a4ab130 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -25,4 +25,9 @@ session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' cloudwatch_metrics: - - namespace: AWS/SNS + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc index 7e1fa891671..4337db82968 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc @@ -1,6 +1,6 @@ The cloudwatch metricset of aws module allows you to monitor varies services on AWS. `cloudwatch` metricset fetches metrics from given namespace periodically -and sto +by calling `GetMetricData` api. [float] === AWS Permissions diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 1598362ea24..602008b79d8 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -7,8 +7,10 @@ package cloudwatch import ( "strconv" "strings" + "time" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface" "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" @@ -59,8 +61,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get CloudwatchMetrics Config - cloudwatchConfig := m.CloudwatchMetrics - if len(cloudwatchConfig) == 0 { + cloudwatchConfigs := m.CloudwatchMetrics + if len(cloudwatchConfigs) == 0 { return errors.New("cloudwatch_metrics in config is missing") } // Get startTime and endTime @@ -69,8 +71,28 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return errors.Wrap(err, "Error ParseDuration") } - for _, cw := range cloudwatchConfig { - namespace := cw["namespace"].(string) + // Get listMetricsTotal and namespaces from configuration + listMetricsTotal := []cloudwatch.Metric{} + namespaces := []string{} + for _, cloudwatchConfig := range cloudwatchConfigs { + namespace := cloudwatchConfig["namespace"].(string) + if cloudwatchConfig["metricname"] != nil { + listMetricsOutput := convertConfigToListMetrics(cloudwatchConfig, namespace) + listMetricsTotal = append(listMetricsTotal, listMetricsOutput) + } else { + namespaces = append(namespaces, namespace) + } + } + + // Use listMetricsTotal from config + svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + err = createEvents(svcCloudwatch, listMetricsTotal, m.PeriodInSec, startTime, endTime, report) + if err != nil { + return errors.New("createEvents failed") + } + + // Use namespaces from config + for _, namespace := range namespaces { for _, regionName := range m.MetricSet.RegionsList { m.MetricSet.AwsConfig.Region = regionName svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) @@ -85,63 +107,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - // get identifiers from listMetrics - identifiers := getIdentifiers(listMetricsOutput) - - // Construct metricDataQueries - metricDataQueries := constructMetricQueries(listMetricsOutput, int64(m.PeriodInSec)) - if len(metricDataQueries) == 0 { - continue - } - - // Use metricDataQueries to make GetMetricData API calls - metricDataResults, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) + err = createEvents(svcCloudwatch, listMetricsOutput, m.PeriodInSec, startTime, endTime, report) if err != nil { - err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName) - m.Logger().Error(err.Error()) - report.Error(err) - continue - } - - // Initialize events map per region, which stores one event per identifierValue - events := map[string]mb.Event{} - for _, values := range identifiers { - for _, v := range values { - events[v] = initEvent(regionName) - } - } - - // Find a timestamp for all metrics in output - timestamp := aws.FindTimestamp(metricDataResults) - if !timestamp.IsZero() { - for _, output := range metricDataResults { - if len(output.Values) == 0 { - continue - } - - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { - labels := strings.Split(*output.Label, " ") - if len(labels) == 3 { - identifierValue := labels[2] - events[identifierValue] = insertMetricSetFields(events[identifierValue], namespace, output.Values[timestampIdx], labels) - } else { - eventNew := initEvent(regionName) - eventNew = insertMetricSetFields(eventNew, namespace, output.Values[timestampIdx], labels) - if reported := report.Event(eventNew); !reported { - return nil - } - } - } - } - } - - for _, event := range events { - if len(event.MetricSetFields) != 0 { - if reported := report.Event(event); !reported { - return nil - } - } + return errors.New("createEvents failed for region " + regionName) } } } @@ -161,7 +129,7 @@ func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) func constructLabel(metric cloudwatch.Metric) string { metricDims := metric.Dimensions metricName := *metric.MetricName - label := metricName + label := metricName + " " + *metric.Namespace dimNames := "" dimValues := "" for i, dim := range metricDims { @@ -238,21 +206,117 @@ func initEvent(regionName string) mb.Event { event.RootFields = common.MapStr{} event.MetricSetFields = common.MapStr{} event.RootFields.Put("service.name", metricsetName) - event.RootFields.Put("cloud.region", regionName) + if regionName != "" { + event.RootFields.Put("cloud.region", regionName) + } return event } -func insertMetricSetFields(event mb.Event, namespace string, metricValue float64, labels []string) mb.Event { - event.MetricSetFields.Put("namespace", namespace) +func insertMetricSetFields(event mb.Event, metricValue float64, labels []string) mb.Event { + event.MetricSetFields.Put("namespace", labels[1]) event.MetricSetFields.Put(labels[0], metricValue) - if len(labels) == 1 { + if len(labels) == 2 { return event } - dimNames := strings.Split(labels[1], ",") - dimValues := strings.Split(labels[2], ",") + dimNames := strings.Split(labels[2], ",") + dimValues := strings.Split(labels[3], ",") for i := 0; i < len(dimNames); i++ { event.MetricSetFields.Put(dimNames[i], dimValues[i]) } return event } + +func convertConfigToListMetrics(cloudwatchConfig map[string]interface{}, namespace string) cloudwatch.Metric { + // convert config input to []cloudwatch.Metric + metricName := cloudwatchConfig["metricname"].(string) + dimensions := cloudwatchConfig["dimensions"].([]interface{}) + cloudwatchDimensions := []cloudwatch.Dimension{} + for _, dim := range dimensions { + d := dim.(map[string]interface{}) + cloudwatchDim := cloudwatch.Dimension{} + for n, v := range d { + if n == "name" { + name := v.(string) + cloudwatchDim.Name = &name + } else if n == "value" { + value := v.(string) + cloudwatchDim.Value = &value + } + } + cloudwatchDimensions = append(cloudwatchDimensions, cloudwatchDim) + } + + listMetricsOutput := cloudwatch.Metric{ + Namespace: &namespace, + MetricName: &metricName, + Dimensions: cloudwatchDimensions, + } + return listMetricsOutput +} + +func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwatch.Metric, period int, startTime time.Time, endTime time.Time, report mb.ReporterV2) error { + identifiers := getIdentifiers(listMetricsTotal) + // Initialize events map per region, which stores one event per identifierValue + events := map[string]mb.Event{} + for _, values := range identifiers { + for _, v := range values { + events[v] = initEvent("") + } + } + // Initialize events for the ones without identifiers. + eventsNoIdentifier := []mb.Event{} + + // Construct metricDataQueries + metricDataQueries := constructMetricQueries(listMetricsTotal, int64(period)) + if len(metricDataQueries) == 0 { + return nil + } + + // Use metricDataQueries to make GetMetricData API calls + metricDataResults, err := aws.GetMetricDataResults(metricDataQueries, svc, startTime, endTime) + if err != nil { + return errors.Wrap(err, "GetMetricDataResults failed") + } + + // Find a timestamp for all metrics in output + timestamp := aws.FindTimestamp(metricDataResults) + if !timestamp.IsZero() { + for _, output := range metricDataResults { + if len(output.Values) == 0 { + continue + } + + exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) + if exists { + labels := strings.Split(*output.Label, " ") + if len(labels) == 4 { + identifierValue := labels[3] + events[identifierValue] = insertMetricSetFields(events[identifierValue], output.Values[timestampIdx], labels) + } else { + eventNew := initEvent("") + eventNew = insertMetricSetFields(eventNew, output.Values[timestampIdx], labels) + eventsNoIdentifier = append(eventsNoIdentifier, eventNew) + } + } + } + } + + for _, event := range events { + if len(event.MetricSetFields) != 0 { + if reported := report.Event(event); !reported { + return nil + } + } + } + + for _, event := range eventsNoIdentifier { + if len(event.MetricSetFields) != 0 { + if reported := report.Event(event); !reported { + return nil + } + } + } + + return nil +} diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 1f6ad7b2497..138982971d1 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -77,23 +77,23 @@ func TestConstructLabel(t *testing.T) { }{ { listMetric1, - "CPUUtilization InstanceId i-1", + "CPUUtilization AWS/EC2 InstanceId i-1", }, { listMetric2, - "StatusCheckFailed InstanceId i-1", + "StatusCheckFailed AWS/EC2 InstanceId i-1", }, { listMetric3, - "StatusCheckFailed_System InstanceId i-2", + "StatusCheckFailed_System AWS/EC2 InstanceId i-2", }, { listMetric4, - "StatusCheckFailed_Instance InstanceId i-2", + "StatusCheckFailed_Instance AWS/EC2 InstanceId i-2", }, { listMetric5, - "CPUUtilization", + "CPUUtilization AWS/EC2", }, } diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 412fb5784af..b4f3a4ab130 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -25,4 +25,9 @@ session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' cloudwatch_metrics: - - namespace: AWS/SNS + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS From faf725700b7545df3cb6d9be78fd554e279776d0 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 18 Apr 2019 21:33:27 -0600 Subject: [PATCH 06/17] Update docs --- metricbeat/docs/modules/aws.asciidoc | 7 ++++++- x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 2 +- x-pack/metricbeat/module/aws/utils.go | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index e3b11ae28d8..028467c20cf 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -139,7 +139,12 @@ metricbeat.modules: session_token: '${AWS_SESSION_TOKEN:""}' default_region: '${AWS_REGION:us-west-1}' cloudwatch_metrics: - - namespace: AWS/SNS + - namespace: AWS/EC2 + metricname: CPUUtilization + dimensions: + - name: InstanceId + value: i-0686946e22cf9494a + - namespace: AWS/EBS ---- [float] diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 602008b79d8..01eceb7ee17 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -109,7 +109,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { err = createEvents(svcCloudwatch, listMetricsOutput, m.PeriodInSec, startTime, endTime, report) if err != nil { - return errors.New("createEvents failed for region " + regionName) + return errors.Wrap(err, "createEvents failed for region "+regionName) } } } diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 744e5b20daf..7f43cbb3b76 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -79,6 +79,10 @@ func GetMetricDataResults(metricDataQueries []cloudwatch.MetricDataQuery, svc cl metricDataQueriesPartial = metricDataQueries[i*100 : (i+1)*100-1] } + if len(metricDataQueriesPartial) == 0 { + return getMetricDataOutput.MetricDataResults, nil + } + output, err := getMetricDataPerRegion(metricDataQueriesPartial, getMetricDataOutput.NextToken, svc, startTime, endTime) if err != nil { err = errors.Wrap(err, "getMetricDataPerRegion failed") From 600a550e195afdb6b436cb4e9b807a500ce0e225 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 18 Apr 2019 22:39:51 -0600 Subject: [PATCH 07/17] Add unit test for readCloudwatchConfig --- .../module/aws/cloudwatch/cloudwatch.go | 30 +++--- .../module/aws/cloudwatch/cloudwatch_test.go | 102 ++++++++++++++++-- x-pack/metricbeat/module/aws/utils_test.go | 13 ++- 3 files changed, 122 insertions(+), 23 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 01eceb7ee17..36ce16b58ec 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -72,17 +72,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // Get listMetricsTotal and namespaces from configuration - listMetricsTotal := []cloudwatch.Metric{} - namespaces := []string{} - for _, cloudwatchConfig := range cloudwatchConfigs { - namespace := cloudwatchConfig["namespace"].(string) - if cloudwatchConfig["metricname"] != nil { - listMetricsOutput := convertConfigToListMetrics(cloudwatchConfig, namespace) - listMetricsTotal = append(listMetricsTotal, listMetricsOutput) - } else { - namespaces = append(namespaces, namespace) - } - } + listMetricsTotal, namespacesTotal := readCloudwatchConfig(cloudwatchConfigs) // Use listMetricsTotal from config svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) @@ -92,7 +82,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // Use namespaces from config - for _, namespace := range namespaces { + for _, namespace := range namespacesTotal { for _, regionName := range m.MetricSet.RegionsList { m.MetricSet.AwsConfig.Region = regionName svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) @@ -117,6 +107,21 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } +func readCloudwatchConfig(cloudwatchConfigs []map[string]interface{}) ([]cloudwatch.Metric, []string) { + listMetricsTotal := []cloudwatch.Metric{} + namespacesTotal := []string{} + for _, cloudwatchConfig := range cloudwatchConfigs { + namespace := cloudwatchConfig["namespace"].(string) + if cloudwatchConfig["metricname"] != nil { + listMetricsOutput := convertConfigToListMetrics(cloudwatchConfig, namespace) + listMetricsTotal = append(listMetricsTotal, listMetricsOutput) + } else { + namespacesTotal = append(namespacesTotal, namespace) + } + } + return listMetricsTotal, namespacesTotal +} + func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery { metricDataQueries := []cloudwatch.MetricDataQuery{} for i, listMetric := range listMetricsOutput { @@ -185,7 +190,6 @@ func getIdentifiers(listMetricsOutputs []cloudwatch.Metric) map[string][]string identifierName += "," identifierValue += "," } - } if identifiers[identifierName] != nil { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 138982971d1..79a9468541a 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -14,15 +14,17 @@ import ( ) var ( - instanceID1 = "i-1" - instanceID2 = "i-2" - namespace = "AWS/EC2" - dimName = "InstanceId" - metricName1 = "CPUUtilization" - metricName2 = "StatusCheckFailed" - metricName3 = "StatusCheckFailed_System" - metricName4 = "StatusCheckFailed_Instance" - listMetric1 = cloudwatch.Metric{ + instanceID1 = "i-1" + instanceID2 = "i-2" + namespace = "AWS/EC2" + dimName = "InstanceId" + metricName1 = "CPUUtilization" + metricName2 = "StatusCheckFailed" + metricName3 = "StatusCheckFailed_System" + metricName4 = "StatusCheckFailed_Instance" + metricName5 = "CommitThroughput" + namespaceRDS = "AWS/RDS" + listMetric1 = cloudwatch.Metric{ Dimensions: []cloudwatch.Dimension{{ Name: &dimName, Value: &instanceID1, @@ -62,6 +64,23 @@ var ( MetricName: &metricName1, Namespace: &namespace, } + + dimName1 = "DBClusterIdentifier" + dimValue1 = "test1-cluster" + dimName2 = "Role" + dimValue2 = "READER" + listMetric6 = cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: &dimName1, + Value: &dimValue1, + }, + { + Name: &dimName2, + Value: &dimValue2, + }}, + MetricName: &metricName5, + Namespace: &namespaceRDS, + } ) func TestGetIdentifiers(t *testing.T) { @@ -102,3 +121,68 @@ func TestConstructLabel(t *testing.T) { assert.Equal(t, c.expectedLabel, label) } } + +func TestReadCloudwatchConfig(t *testing.T) { + cloudwatchMetricsConfig1 := []map[string]interface{}{} + cloudwatchMetric1 := map[string]interface{}{} + cloudwatchMetric1["namespace"] = "AWS/EC2" + cloudwatchMetric1["metricname"] = "CPUUtilization" + dimensions1 := []interface{}{} + dim1 := map[string]interface{}{} + dim1["name"] = "InstanceId" + dim1["value"] = instanceID1 + dimensions1 = append(dimensions1, dim1) + cloudwatchMetric1["dimensions"] = dimensions1 + cloudwatchMetricsConfig1 = append(cloudwatchMetricsConfig1, cloudwatchMetric1) + + cloudwatchMetricsConfig2 := []map[string]interface{}{} + cloudwatchMetric2 := map[string]interface{}{} + cloudwatchMetric2["namespace"] = "AWS/EBS" + cloudwatchMetricsConfig2 = append(cloudwatchMetricsConfig2, cloudwatchMetric1) + cloudwatchMetricsConfig2 = append(cloudwatchMetricsConfig2, cloudwatchMetric2) + + cloudwatchMetricsConfig3 := []map[string]interface{}{} + cloudwatchMetricsConfig3 = append(cloudwatchMetricsConfig3, cloudwatchMetric1) + cloudwatchMetricsConfig3 = append(cloudwatchMetricsConfig3, cloudwatchMetric2) + cloudwatchMetric3 := map[string]interface{}{} + cloudwatchMetric3["namespace"] = "AWS/RDS" + cloudwatchMetric3["metricname"] = "CommitThroughput" + dimensions3 := []interface{}{} + dim31 := map[string]interface{}{} + dim31["name"] = "DBClusterIdentifier" + dim31["value"] = "test1-cluster" + dimensions3 = append(dimensions3, dim31) + dim32 := map[string]interface{}{} + dim32["name"] = "Role" + dim32["value"] = "READER" + dimensions3 = append(dimensions3, dim32) + cloudwatchMetric3["dimensions"] = dimensions3 + cloudwatchMetricsConfig3 = append(cloudwatchMetricsConfig3, cloudwatchMetric3) + + cases := []struct { + cloudwatchMetricsConfig []map[string]interface{} + expectedListMetric []cloudwatch.Metric + expectedNamespace []string + }{ + { + cloudwatchMetricsConfig1, + []cloudwatch.Metric{listMetric1}, + []string{}, + }, + { + cloudwatchMetricsConfig2, + []cloudwatch.Metric{listMetric1}, + []string{"AWS/EBS"}, + }, + { + cloudwatchMetricsConfig3, + []cloudwatch.Metric{listMetric1, listMetric6}, + []string{"AWS/EBS"}, + }, + } + for _, c := range cases { + listMetrics, namespaces := readCloudwatchConfig(c.cloudwatchMetricsConfig) + assert.Equal(t, c.expectedListMetric, listMetrics) + assert.Equal(t, c.expectedNamespace, namespaces) + } +} diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index 00bab25d80c..3e539f4f122 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -146,7 +146,18 @@ func TestGetMetricDataResults(t *testing.T) { assert.NoError(t, err) mockSvc := &MockCloudWatchClient{} - metricDataQueries := []cloudwatch.MetricDataQuery{} + metricInfo := cloudwatch.Metric{ + MetricName: &metricName, + Namespace: &namespace, + } + metricStat := cloudwatch.MetricStat{Metric: &metricInfo} + metricDataQueries := []cloudwatch.MetricDataQuery{ + { + Id: &id1, + Label: &label1, + MetricStat: &metricStat, + }, + } getMetricDataResults, err := GetMetricDataResults(metricDataQueries, mockSvc, startTime, endTime) if err != nil { fmt.Println("failed getMetricDataPerRegion: ", err) From c58657c73ae5f9421de9eae4c0d3a25c4f67dcfa Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 19 Apr 2019 09:00:47 -0600 Subject: [PATCH 08/17] refactor TestReadCloudwatchConfig --- .../module/aws/cloudwatch/cloudwatch_test.go | 95 +++++++++++-------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 79a9468541a..057526b22ab 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -123,59 +123,76 @@ func TestConstructLabel(t *testing.T) { } func TestReadCloudwatchConfig(t *testing.T) { - cloudwatchMetricsConfig1 := []map[string]interface{}{} - cloudwatchMetric1 := map[string]interface{}{} - cloudwatchMetric1["namespace"] = "AWS/EC2" - cloudwatchMetric1["metricname"] = "CPUUtilization" - dimensions1 := []interface{}{} - dim1 := map[string]interface{}{} - dim1["name"] = "InstanceId" - dim1["value"] = instanceID1 - dimensions1 = append(dimensions1, dim1) - cloudwatchMetric1["dimensions"] = dimensions1 - cloudwatchMetricsConfig1 = append(cloudwatchMetricsConfig1, cloudwatchMetric1) - - cloudwatchMetricsConfig2 := []map[string]interface{}{} - cloudwatchMetric2 := map[string]interface{}{} - cloudwatchMetric2["namespace"] = "AWS/EBS" - cloudwatchMetricsConfig2 = append(cloudwatchMetricsConfig2, cloudwatchMetric1) - cloudwatchMetricsConfig2 = append(cloudwatchMetricsConfig2, cloudwatchMetric2) - - cloudwatchMetricsConfig3 := []map[string]interface{}{} - cloudwatchMetricsConfig3 = append(cloudwatchMetricsConfig3, cloudwatchMetric1) - cloudwatchMetricsConfig3 = append(cloudwatchMetricsConfig3, cloudwatchMetric2) - cloudwatchMetric3 := map[string]interface{}{} - cloudwatchMetric3["namespace"] = "AWS/RDS" - cloudwatchMetric3["metricname"] = "CommitThroughput" - dimensions3 := []interface{}{} - dim31 := map[string]interface{}{} - dim31["name"] = "DBClusterIdentifier" - dim31["value"] = "test1-cluster" - dimensions3 = append(dimensions3, dim31) - dim32 := map[string]interface{}{} - dim32["name"] = "Role" - dim32["value"] = "READER" - dimensions3 = append(dimensions3, dim32) - cloudwatchMetric3["dimensions"] = dimensions3 - cloudwatchMetricsConfig3 = append(cloudwatchMetricsConfig3, cloudwatchMetric3) - cases := []struct { cloudwatchMetricsConfig []map[string]interface{} expectedListMetric []cloudwatch.Metric expectedNamespace []string }{ { - cloudwatchMetricsConfig1, + []map[string]interface{}{ + { + "namespace": "AWS/EC2", + "metricname": "CPUUtilization", + "dimensions": []interface{}{ + map[string]interface{}{ + "name": "InstanceId", + "value": instanceID1, + }, + }, + }, + }, []cloudwatch.Metric{listMetric1}, []string{}, }, { - cloudwatchMetricsConfig2, + []map[string]interface{}{ + { + "namespace": "AWS/EC2", + "metricname": "CPUUtilization", + "dimensions": []interface{}{ + map[string]interface{}{ + "name": "InstanceId", + "value": instanceID1, + }, + }, + }, + { + "namespace": "AWS/EBS", + }, + }, []cloudwatch.Metric{listMetric1}, []string{"AWS/EBS"}, }, { - cloudwatchMetricsConfig3, + []map[string]interface{}{ + { + "namespace": "AWS/EC2", + "metricname": "CPUUtilization", + "dimensions": []interface{}{ + map[string]interface{}{ + "name": "InstanceId", + "value": instanceID1, + }, + }, + }, + { + "namespace": "AWS/EBS", + }, + { + "namespace": "AWS/RDS", + "metricname": "CommitThroughput", + "dimensions": []interface{}{ + map[string]interface{}{ + "name": "DBClusterIdentifier", + "value": "test1-cluster", + }, + map[string]interface{}{ + "name": "Role", + "value": "READER", + }, + }, + }, + }, []cloudwatch.Metric{listMetric1, listMetric6}, []string{"AWS/EBS"}, }, From 202b78771f755c070e6fae4132b52b3dde9b8557 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 23 Apr 2019 16:26:59 -0600 Subject: [PATCH 09/17] Move CloudwatchMetricsConfigs into cloudwatch metricset from aws module --- metricbeat/docs/modules/aws.asciidoc | 10 +-- x-pack/metricbeat/metricbeat.reference.yml | 10 +-- x-pack/metricbeat/module/aws/_meta/config.yml | 10 +-- x-pack/metricbeat/module/aws/aws.go | 31 ++++--- .../module/aws/cloudwatch/_meta/docs.asciidoc | 2 +- .../module/aws/cloudwatch/cloudwatch.go | 82 +++++++++++-------- .../module/aws/cloudwatch/cloudwatch_test.go | 68 +++++++-------- x-pack/metricbeat/modules.d/aws.yml.disabled | 10 +-- 8 files changed, 119 insertions(+), 104 deletions(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 028467c20cf..0f30a777645 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -115,8 +115,8 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -124,8 +124,8 @@ metricbeat.modules: - module: aws period: 86400s metricsets: - - "s3_request" - - "s3_daily_storage" + - s3_request + - s3_daily_storage access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -133,7 +133,7 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "cloudwatch" + - cloudwatch access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 2dde7234df6..07edfe8968c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -160,8 +160,8 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -169,8 +169,8 @@ metricbeat.modules: - module: aws period: 86400s metricsets: - - "s3_request" - - "s3_daily_storage" + - s3_request + - s3_daily_storage access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -178,7 +178,7 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "cloudwatch" + - cloudwatch access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index b4f3a4ab130..d1e541fb07d 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -1,8 +1,8 @@ - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -10,8 +10,8 @@ - module: aws period: 86400s metricsets: - - "s3_request" - - "s3_daily_storage" + - s3_request + - s3_daily_storage access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -19,7 +19,7 @@ - module: aws period: 300s metricsets: - - "cloudwatch" + - cloudwatch access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 1d5acccf738..eb40e81c6a1 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -20,22 +20,20 @@ import ( // Config defines all required and optional parameters for aws metricsets type Config struct { - Period string `config:"period"` - AccessKeyID string `config:"access_key_id"` - SecretAccessKey string `config:"secret_access_key"` - SessionToken string `config:"session_token"` - DefaultRegion string `config:"default_region"` - CloudwatchMetrics []map[string]interface{} `config:"cloudwatch_metrics"` + Period string `config:"period"` + AccessKeyID string `config:"access_key_id"` + SecretAccessKey string `config:"secret_access_key"` + SessionToken string `config:"session_token"` + DefaultRegion string `config:"default_region"` } // MetricSet is the base metricset for all aws metricsets type MetricSet struct { mb.BaseMetricSet - RegionsList []string - DurationString string - PeriodInSec int - AwsConfig *awssdk.Config - CloudwatchMetrics []map[string]interface{} + RegionsList []string + DurationString string + PeriodInSec int + AwsConfig *awssdk.Config } // ModuleName is the name of this module. @@ -97,12 +95,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // Construct MetricSet metricSet := MetricSet{ - BaseMetricSet: base, - RegionsList: regionsList, - DurationString: durationString, - PeriodInSec: periodSec, - AwsConfig: &awsConfig, - CloudwatchMetrics: config.CloudwatchMetrics, + BaseMetricSet: base, + RegionsList: regionsList, + DurationString: durationString, + PeriodInSec: periodSec, + AwsConfig: &awsConfig, } return &metricSet, nil } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc index 4337db82968..5032cb91697 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc @@ -1,4 +1,4 @@ -The cloudwatch metricset of aws module allows you to monitor varies services on +The cloudwatch metricset of aws module allows you to monitor various services on AWS. `cloudwatch` metricset fetches metrics from given namespace periodically by calling `GetMetricData` api. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 36ce16b58ec..223c94492f8 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -39,7 +39,25 @@ func init() { // interface methods except for Fetch. type MetricSet struct { *aws.MetricSet - Namespace string + CloudwatchMetrics Configs +} + +// Dimension holds name and value for cloudwatch metricset dimension config. +type Dimension struct { + Name string `config:"name" validate:"nonzero"` + Value string `config:"value" validate:"nonzero"` +} + +// CloudwatchConfig holds a configuration specific for cloudwatch metricset. +type Config struct { + Namespace string `config:"namespace" validate:"nonzero"` + MetricName string `config:"metric_name"` + Dimensions []Dimension `config:"dimensions"` +} + +// Configs is a set of CloudwatchConfig. +type Configs struct { + CloudwatchConfigs []Config `config:"cloudwatch_metrics" validate:"nonzero,required"` } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -51,8 +69,19 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "error creating aws metricset") } + config := Configs{} + err = base.Module().UnpackConfig(&config) + if err != nil { + return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig") + } + + if len(config.CloudwatchConfigs) == 0 { + return nil, errors.New("cloudwatch_metrics in config is missing") + } + return &MetricSet{ - MetricSet: metricSet, + MetricSet: metricSet, + CloudwatchMetrics: config, }, nil } @@ -60,25 +89,20 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // format. It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { - // Get CloudwatchMetrics Config - cloudwatchConfigs := m.CloudwatchMetrics - if len(cloudwatchConfigs) == 0 { - return errors.New("cloudwatch_metrics in config is missing") - } // Get startTime and endTime startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) if err != nil { - return errors.Wrap(err, "Error ParseDuration") + return errors.Wrap(err, "error ParseDuration") } // Get listMetricsTotal and namespaces from configuration - listMetricsTotal, namespacesTotal := readCloudwatchConfig(cloudwatchConfigs) + listMetricsTotal, namespacesTotal := readCloudwatchConfig(m.CloudwatchMetrics.CloudwatchConfigs) // Use listMetricsTotal from config svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) err = createEvents(svcCloudwatch, listMetricsTotal, m.PeriodInSec, startTime, endTime, report) if err != nil { - return errors.New("createEvents failed") + return errors.Wrap(err, "createEvents failed") } // Use namespaces from config @@ -107,12 +131,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } -func readCloudwatchConfig(cloudwatchConfigs []map[string]interface{}) ([]cloudwatch.Metric, []string) { - listMetricsTotal := []cloudwatch.Metric{} - namespacesTotal := []string{} +func readCloudwatchConfig(cloudwatchConfigs []Config) ([]cloudwatch.Metric, []string) { + var listMetricsTotal []cloudwatch.Metric + var namespacesTotal []string for _, cloudwatchConfig := range cloudwatchConfigs { - namespace := cloudwatchConfig["namespace"].(string) - if cloudwatchConfig["metricname"] != nil { + namespace := cloudwatchConfig.Namespace + if cloudwatchConfig.MetricName != "" { listMetricsOutput := convertConfigToListMetrics(cloudwatchConfig, namespace) listMetricsTotal = append(listMetricsTotal, listMetricsOutput) } else { @@ -123,7 +147,7 @@ func readCloudwatchConfig(cloudwatchConfigs []map[string]interface{}) ([]cloudwa } func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery { - metricDataQueries := []cloudwatch.MetricDataQuery{} + var metricDataQueries []cloudwatch.MetricDataQuery for i, listMetric := range listMetricsOutput { metricDataQuery := createMetricDataQuery(listMetric, i, period) metricDataQueries = append(metricDataQueries, metricDataQuery) @@ -231,24 +255,18 @@ func insertMetricSetFields(event mb.Event, metricValue float64, labels []string) return event } -func convertConfigToListMetrics(cloudwatchConfig map[string]interface{}, namespace string) cloudwatch.Metric { +func convertConfigToListMetrics(cloudwatchConfig Config, namespace string) cloudwatch.Metric { // convert config input to []cloudwatch.Metric - metricName := cloudwatchConfig["metricname"].(string) - dimensions := cloudwatchConfig["dimensions"].([]interface{}) - cloudwatchDimensions := []cloudwatch.Dimension{} + metricName := cloudwatchConfig.MetricName + dimensions := cloudwatchConfig.Dimensions + var cloudwatchDimensions []cloudwatch.Dimension for _, dim := range dimensions { - d := dim.(map[string]interface{}) - cloudwatchDim := cloudwatch.Dimension{} - for n, v := range d { - if n == "name" { - name := v.(string) - cloudwatchDim.Name = &name - } else if n == "value" { - value := v.(string) - cloudwatchDim.Value = &value - } - } - cloudwatchDimensions = append(cloudwatchDimensions, cloudwatchDim) + name := dim.Name + value := dim.Value + cloudwatchDimensions = append(cloudwatchDimensions, cloudwatch.Dimension{ + Name: &name, + Value: &value, + }) } listMetricsOutput := cloudwatch.Metric{ diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 057526b22ab..87aa52d2832 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -124,71 +124,71 @@ func TestConstructLabel(t *testing.T) { func TestReadCloudwatchConfig(t *testing.T) { cases := []struct { - cloudwatchMetricsConfig []map[string]interface{} + cloudwatchMetricsConfig []CloudwatchConfig expectedListMetric []cloudwatch.Metric expectedNamespace []string }{ { - []map[string]interface{}{ + []CloudwatchConfig{ { - "namespace": "AWS/EC2", - "metricname": "CPUUtilization", - "dimensions": []interface{}{ - map[string]interface{}{ - "name": "InstanceId", - "value": instanceID1, + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: []Dimension{ + { + Name: "InstanceId", + Value: instanceID1, }, }, }, }, []cloudwatch.Metric{listMetric1}, - []string{}, + nil, }, { - []map[string]interface{}{ + []CloudwatchConfig{ { - "namespace": "AWS/EC2", - "metricname": "CPUUtilization", - "dimensions": []interface{}{ - map[string]interface{}{ - "name": "InstanceId", - "value": instanceID1, + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: []Dimension{ + { + Name: "InstanceId", + Value: instanceID1, }, }, }, { - "namespace": "AWS/EBS", + Namespace: "AWS/EBS", }, }, []cloudwatch.Metric{listMetric1}, []string{"AWS/EBS"}, }, { - []map[string]interface{}{ + []CloudwatchConfig{ { - "namespace": "AWS/EC2", - "metricname": "CPUUtilization", - "dimensions": []interface{}{ - map[string]interface{}{ - "name": "InstanceId", - "value": instanceID1, + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Dimensions: []Dimension{ + { + Name: "InstanceId", + Value: instanceID1, }, }, }, { - "namespace": "AWS/EBS", + Namespace: "AWS/EBS", }, { - "namespace": "AWS/RDS", - "metricname": "CommitThroughput", - "dimensions": []interface{}{ - map[string]interface{}{ - "name": "DBClusterIdentifier", - "value": "test1-cluster", + Namespace: "AWS/RDS", + MetricName: "CommitThroughput", + Dimensions: []Dimension{ + { + Name: "DBClusterIdentifier", + Value: "test1-cluster", }, - map[string]interface{}{ - "name": "Role", - "value": "READER", + { + Name: "Role", + Value: "READER", }, }, }, diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index b4f3a4ab130..d1e541fb07d 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -1,8 +1,8 @@ - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -10,8 +10,8 @@ - module: aws period: 86400s metricsets: - - "s3_request" - - "s3_daily_storage" + - s3_request + - s3_daily_storage access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -19,7 +19,7 @@ - module: aws period: 300s metricsets: - - "cloudwatch" + - cloudwatch access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' From ade4ba49acc05cda6b11110d711efc97aee395f1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 23 Apr 2019 23:05:15 -0600 Subject: [PATCH 10/17] Change Period from string to time.Duration type --- x-pack/metricbeat/module/aws/aws.go | 44 ++++---------- x-pack/metricbeat/module/aws/aws_test.go | 60 ++++++++++--------- .../module/aws/cloudwatch/cloudwatch.go | 9 +-- .../module/aws/cloudwatch/cloudwatch_test.go | 8 +-- 4 files changed, 52 insertions(+), 69 deletions(-) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index eb40e81c6a1..8afda91bfcd 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -5,7 +5,7 @@ package aws import ( - "strconv" + "time" "github.com/elastic/beats/libbeat/common" @@ -20,11 +20,11 @@ import ( // Config defines all required and optional parameters for aws metricsets type Config struct { - Period string `config:"period"` - AccessKeyID string `config:"access_key_id"` - SecretAccessKey string `config:"secret_access_key"` - SessionToken string `config:"session_token"` - DefaultRegion string `config:"default_region"` + Period time.Duration `config:"period" validate:"nonzero,required"` + AccessKeyID string `config:"access_key_id"` + SecretAccessKey string `config:"secret_access_key"` + SessionToken string `config:"session_token"` + DefaultRegion string `config:"default_region"` } // MetricSet is the base metricset for all aws metricsets @@ -82,13 +82,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return nil, err } - // Calculate duration based on period - if config.Period == "" { - err = errors.New("period is not set in AWS module config") - return nil, err - } - - durationString, periodSec, err := convertPeriodToDuration(config.Period) + durationString, periodSec := convertPeriodToDuration(config.Period) if err != nil { return nil, err } @@ -118,28 +112,12 @@ func getRegions(svc ec2iface.EC2API) (regionsList []string, err error) { return } -func convertPeriodToDuration(period string) (string, int, error) { +func convertPeriodToDuration(period time.Duration) (string, int) { // Set starttime double the default frequency earlier than the endtime in order to make sure // GetMetricDataRequest gets the latest data point for each metric. - numberPeriod, err := strconv.Atoi(period[0 : len(period)-1]) - if err != nil { - return "", 0, err - } - - unitPeriod := period[len(period)-1:] - switch unitPeriod { - case "s": - duration := "-" + strconv.Itoa(numberPeriod*2) + unitPeriod - return duration, numberPeriod, nil - case "m": - duration := "-" + strconv.Itoa(numberPeriod*2) + unitPeriod - periodInSec := numberPeriod * 60 - return duration, periodInSec, nil - default: - err = errors.New("invalid period in config. Please reset period in config") - duration := "-" + strconv.Itoa(numberPeriod*2) + "s" - return duration, numberPeriod, err - } + duration := "-" + (period * 2).String() + numberPeriod := int(period.Seconds()) + return duration, numberPeriod } // StringInSlice checks if a string is already exists in list diff --git a/x-pack/metricbeat/module/aws/aws_test.go b/x-pack/metricbeat/module/aws/aws_test.go index b503690464a..5f6c884f93f 100644 --- a/x-pack/metricbeat/module/aws/aws_test.go +++ b/x-pack/metricbeat/module/aws/aws_test.go @@ -9,6 +9,7 @@ package aws import ( "fmt" "testing" + "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -49,33 +50,36 @@ func TestGetRegions(t *testing.T) { } func TestConvertPeriodToDuration(t *testing.T) { - period1 := "300s" - duration1, periodSec1, err := convertPeriodToDuration(period1) - assert.NoError(t, nil, err) - assert.Equal(t, "-600s", duration1) - assert.Equal(t, 300, periodSec1) - - period2 := "30ss" - duration2, periodSec2, err := convertPeriodToDuration(period2) - assert.Error(t, err) - assert.Equal(t, "", duration2) - assert.Equal(t, 0, periodSec2) - - period3 := "10m" - duration3, periodSec3, err := convertPeriodToDuration(period3) - assert.NoError(t, nil, err) - assert.Equal(t, "-20m", duration3) - assert.Equal(t, 600, periodSec3) - - period4 := "30s" - duration4, periodSec4, err := convertPeriodToDuration(period4) - assert.NoError(t, nil, err) - assert.Equal(t, "-60s", duration4) - assert.Equal(t, 30, periodSec4) + cases := []struct { + period time.Duration + expectedDuration string + expectedPeriodNumber int + }{ + { + period: time.Duration(300) * time.Second, + expectedDuration: "-10m0s", + expectedPeriodNumber: 300, + }, + { + period: time.Duration(10) * time.Minute, + expectedDuration: "-20m0s", + expectedPeriodNumber: 600, + }, + { + period: time.Duration(30) * time.Second, + expectedDuration: "-1m0s", + expectedPeriodNumber: 30, + }, + { + period: time.Duration(60) * time.Second, + expectedDuration: "-2m0s", + expectedPeriodNumber: 60, + }, + } - period5 := "60s" - duration5, periodSec5, err := convertPeriodToDuration(period5) - assert.NoError(t, nil, err) - assert.Equal(t, "-120s", duration5) - assert.Equal(t, 60, periodSec5) + for _, c := range cases { + duration, periodSec := convertPeriodToDuration(c.period) + assert.Equal(t, c.expectedDuration, duration) + assert.Equal(t, c.expectedPeriodNumber, periodSec) + } } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 223c94492f8..a2cb323136e 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -48,10 +48,10 @@ type Dimension struct { Value string `config:"value" validate:"nonzero"` } -// CloudwatchConfig holds a configuration specific for cloudwatch metricset. +// Config holds a configuration specific for cloudwatch metricset. type Config struct { - Namespace string `config:"namespace" validate:"nonzero"` - MetricName string `config:"metric_name"` + Namespace string `config:"namespace" validate:"nonzero,required"` + MetricName string `config:"metricname"` Dimensions []Dimension `config:"dimensions"` } @@ -92,7 +92,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) if err != nil { - return errors.Wrap(err, "error ParseDuration") + return errors.Wrap(err, "error GetStartTimeEndTime") } // Get listMetricsTotal and namespaces from configuration @@ -134,6 +134,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { func readCloudwatchConfig(cloudwatchConfigs []Config) ([]cloudwatch.Metric, []string) { var listMetricsTotal []cloudwatch.Metric var namespacesTotal []string + for _, cloudwatchConfig := range cloudwatchConfigs { namespace := cloudwatchConfig.Namespace if cloudwatchConfig.MetricName != "" { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 87aa52d2832..edd95241540 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -124,12 +124,12 @@ func TestConstructLabel(t *testing.T) { func TestReadCloudwatchConfig(t *testing.T) { cases := []struct { - cloudwatchMetricsConfig []CloudwatchConfig + cloudwatchMetricsConfig []Config expectedListMetric []cloudwatch.Metric expectedNamespace []string }{ { - []CloudwatchConfig{ + []Config{ { Namespace: "AWS/EC2", MetricName: "CPUUtilization", @@ -145,7 +145,7 @@ func TestReadCloudwatchConfig(t *testing.T) { nil, }, { - []CloudwatchConfig{ + []Config{ { Namespace: "AWS/EC2", MetricName: "CPUUtilization", @@ -164,7 +164,7 @@ func TestReadCloudwatchConfig(t *testing.T) { []string{"AWS/EBS"}, }, { - []CloudwatchConfig{ + []Config{ { Namespace: "AWS/EC2", MetricName: "CPUUtilization", From e65ad4e3a50eff9086c9be24ee1c101b184eea37 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 24 Apr 2019 16:28:25 -0600 Subject: [PATCH 11/17] Add region as a part of cloudwatch_metrics config --- metricbeat/docs/fields.asciidoc | 31 +++++ x-pack/metricbeat/module/aws/aws.go | 2 +- .../module/aws/cloudwatch/_meta/data.json | 73 ++++++----- .../module/aws/cloudwatch/_meta/fields.yml | 17 +++ .../module/aws/cloudwatch/cloudwatch.go | 119 ++++++++++-------- .../module/aws/cloudwatch/cloudwatch_test.go | 48 +++++-- x-pack/metricbeat/module/aws/fields.go | 2 +- x-pack/metricbeat/module/aws/utils.go | 9 +- 8 files changed, 198 insertions(+), 103 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 481faf4ddcf..d3bc25959fd 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -836,6 +836,37 @@ Total. `cloudwatch` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by different namespaces. + +*`aws.cloudwatch.cloudwatch.namespace`*:: ++ +-- +type: keyword + +The namespace specified when query cloudwatch api. + + +-- + +*`aws.cloudwatch.cloudwatch.metrics.*`*:: ++ +-- +type: object + +Metrics that returned from Cloudwatch api query. + + +-- + +*`aws.cloudwatch.cloudwatch.dimensions.*`*:: ++ +-- +type: object + +Cloudwatch metric dimensions. + + +-- + [float] == ec2 fields diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 8afda91bfcd..ce0f636ea7f 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -134,8 +134,8 @@ func StringInSlice(str string, list []string) bool { func InitEvent(metricsetName string, regionName string) mb.Event { event := mb.Event{} event.Service = metricsetName - event.RootFields = common.MapStr{} event.MetricSetFields = common.MapStr{} + event.RootFields = common.MapStr{} event.RootFields.Put("service.name", metricsetName) event.RootFields.Put("cloud.provider", "aws") if regionName != "" { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json index 1aab15061ee..6110e7a127a 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json @@ -2,43 +2,50 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "aws": { "cloudwatch": { - "ActiveTransactions": 0, - "AuroraBinlogReplicaLag": 0, - "AuroraReplicaLag": 17.2328, - "AuroraReplicaLagMaximum": 19.85419979095459, - "AuroraReplicaLagMinimum": 19.85419979095459, - "BinLogDiskUsage": 0, - "BlockedTransactions": 0, - "BufferCacheHitRatio": 100, - "CPUUtilization": 3, - "CommitLatency": 3.800633333333333, - "CommitThroughput": 0.2499866861369206, - "DBClusterIdentifier": "test1-cluster", - "DDLLatency": 0, - "DDLThroughput": 0, - "DMLLatency": 0.07957666666666667, - "DMLThroughput": 0.2499866861369206, - "DatabaseConnections": 0, - "Deadlocks": 0, - "DeleteLatency": 0, - "DeleteThroughput": 0, - "EngineUptime": 1284587, - "FreeLocalStorage": 32815485747.2, - "FreeableMemory": 4826584268.8, - "InsertLatency": 0.07957666666666667, - "InsertThroughput": 0.2499866861369206, - "NetworkReceiveThroughput": 0.7699930051722573, - "NetworkThroughput": 1.5399860103445147, - "NetworkTransmitThroughput": 0.7699930051722573, - "Queries": 7.515309216451994, - "ResultSetCacheHitRatio": 0, - "SelectLatency": 0.201887653138847, - "SelectThroughput": 2.8182418034367194, - "UpdateLatency": 0, + "dimensions": { + "DBClusterIdentifier": "test1-cluster", + "Role": "WRITER" + }, + "metrics": { + "ActiveTransactions": 0, + "AuroraBinlogReplicaLag": 0, + "AuroraReplicaLagMaximum": 19.964599990844725, + "AuroraReplicaLagMinimum": 19.964599990844725, + "BinLogDiskUsage": 0, + "BlockedTransactions": 0, + "BufferCacheHitRatio": 100, + "CPUUtilization": 3, + "CommitLatency": 7.4758933333333335, + "CommitThroughput": 0.4999718865659406, + "DDLLatency": 0, + "DDLThroughput": 0, + "DMLLatency": 0.16068, + "DMLThroughput": 0.4999718865659406, + "DatabaseConnections": 0, + "Deadlocks": 0, + "DeleteLatency": 0, + "DeleteThroughput": 0, + "EngineUptime": 1906152, + "FreeLocalStorage": 32210518016, + "FreeableMemory": 4688029286.4, + "InsertLatency": 0.16068, + "InsertThroughput": 0.4999718865659406, + "LoginFailures": 0, + "NetworkReceiveThroughput": 0.7000140072335472, + "NetworkThroughput": 1.4000280144670945, + "NetworkTransmitThroughput": 0.7000140072335472, + "Queries": 8.796634525053722, + "ResultSetCacheHitRatio": 0, + "SelectLatency": 0.17715786436043143, + "SelectThroughput": 3.086495583173847, + "UpdateLatency": 0, + "UpdateThroughput": 0 + }, "namespace": "AWS/RDS" } }, "cloud": { + "provider": "aws", "region": "us-east-2" }, "event": { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml index a4490ef4934..f78d7a869f6 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml @@ -3,3 +3,20 @@ description: > `cloudwatch` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by different namespaces. release: beta + fields: + - name: cloudwatch.namespace + type: keyword + description: > + The namespace specified when query cloudwatch api. + - name: cloudwatch.metrics.* + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Metrics that returned from Cloudwatch api query. + - name: cloudwatch.dimensions.* + type: object + object_type: keyword + object_type_mapping_type: "*" + description: > + Cloudwatch metric dimensions. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index a2cb323136e..10f486c21ac 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -13,14 +13,17 @@ import ( "github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface" "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) var ( - metricsetName = "cloudwatch" + metricsetName = "cloudwatch" + metricNameIdx = 0 + namespaceIdx = 1 + identifierNameIdx = 2 + identifierValueIdx = 3 ) // init registers the MetricSet with the central registry as soon as the program @@ -39,7 +42,7 @@ func init() { // interface methods except for Fetch. type MetricSet struct { *aws.MetricSet - CloudwatchMetrics Configs + CloudwatchConfigs []Config `config:"cloudwatch_metrics" validate:"nonzero,required"` } // Dimension holds name and value for cloudwatch metricset dimension config. @@ -52,14 +55,10 @@ type Dimension struct { type Config struct { Namespace string `config:"namespace" validate:"nonzero,required"` MetricName string `config:"metricname"` + Region string `config:"region"` Dimensions []Dimension `config:"dimensions"` } -// Configs is a set of CloudwatchConfig. -type Configs struct { - CloudwatchConfigs []Config `config:"cloudwatch_metrics" validate:"nonzero,required"` -} - // New creates a new instance of the MetricSet. New is responsible for unpacking // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { @@ -69,19 +68,22 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "error creating aws metricset") } - config := Configs{} + config := struct { + CloudwatchMetrics []Config `config:"cloudwatch_metrics" validate:"nonzero,required"` + }{} + err = base.Module().UnpackConfig(&config) if err != nil { return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig") } - if len(config.CloudwatchConfigs) == 0 { + if len(config.CloudwatchMetrics) == 0 { return nil, errors.New("cloudwatch_metrics in config is missing") } return &MetricSet{ MetricSet: metricSet, - CloudwatchMetrics: config, + CloudwatchConfigs: config.CloudwatchMetrics, }, nil } @@ -96,13 +98,28 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } // Get listMetricsTotal and namespaces from configuration - listMetricsTotal, namespacesTotal := readCloudwatchConfig(m.CloudwatchMetrics.CloudwatchConfigs) + listMetricsWithRegion, listMetricsWithoutRegion, namespacesTotal := readCloudwatchConfig(m.CloudwatchConfigs) + if listMetricsWithoutRegion != nil { + m.Logger().Info("cloudwatch_metrics config is missing region name. " + + "To avoid extra costs, please make sure region is set in config.yml") + } - // Use listMetricsTotal from config - svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) - err = createEvents(svcCloudwatch, listMetricsTotal, m.PeriodInSec, startTime, endTime, report) - if err != nil { - return errors.Wrap(err, "createEvents failed") + for regionName, listMetricsPerRegion := range listMetricsWithRegion { + m.MetricSet.AwsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + err := createEvents(svcCloudwatch, listMetricsPerRegion, regionName, m.PeriodInSec, startTime, endTime, report) + if err != nil { + return errors.Wrap(err, "createEvents failed") + } + } + + for _, regionName := range m.MetricSet.RegionsList { + m.MetricSet.AwsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + err := createEvents(svcCloudwatch, listMetricsWithoutRegion, regionName, m.PeriodInSec, startTime, endTime, report) + if err != nil { + return errors.Wrap(err, "createEvents failed") + } } // Use namespaces from config @@ -112,8 +129,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { - m.Logger().Errorf(err.Error()) - report.Error(err) + m.Logger().Info(err.Error()) continue } @@ -121,7 +137,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - err = createEvents(svcCloudwatch, listMetricsOutput, m.PeriodInSec, startTime, endTime, report) + err = createEvents(svcCloudwatch, listMetricsOutput, regionName, m.PeriodInSec, startTime, endTime, report) if err != nil { return errors.Wrap(err, "createEvents failed for region "+regionName) } @@ -131,20 +147,29 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } -func readCloudwatchConfig(cloudwatchConfigs []Config) ([]cloudwatch.Metric, []string) { - var listMetricsTotal []cloudwatch.Metric +func readCloudwatchConfig(cloudwatchConfigs []Config) (map[string][]cloudwatch.Metric, []cloudwatch.Metric, []string) { + listMetricsWithRegion := map[string][]cloudwatch.Metric{} + var listMetricsWithoutRegion []cloudwatch.Metric var namespacesTotal []string for _, cloudwatchConfig := range cloudwatchConfigs { namespace := cloudwatchConfig.Namespace if cloudwatchConfig.MetricName != "" { listMetricsOutput := convertConfigToListMetrics(cloudwatchConfig, namespace) - listMetricsTotal = append(listMetricsTotal, listMetricsOutput) + if cloudwatchConfig.Region != "" { + if listMetricsWithRegion[cloudwatchConfig.Region] != nil { + listMetricsWithRegion[cloudwatchConfig.Region] = append(listMetricsWithRegion[cloudwatchConfig.Region], listMetricsOutput) + } else { + listMetricsWithRegion[cloudwatchConfig.Region] = []cloudwatch.Metric{listMetricsOutput} + } + } else { + listMetricsWithoutRegion = append(listMetricsWithoutRegion, listMetricsOutput) + } } else { namespacesTotal = append(namespacesTotal, namespace) } } - return listMetricsTotal, namespacesTotal + return listMetricsWithRegion, listMetricsWithoutRegion, namespacesTotal } func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery { @@ -157,15 +182,13 @@ func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) } func constructLabel(metric cloudwatch.Metric) string { - metricDims := metric.Dimensions - metricName := *metric.MetricName - label := metricName + " " + *metric.Namespace + label := *metric.MetricName + " " + *metric.Namespace dimNames := "" dimValues := "" - for i, dim := range metricDims { + for i, dim := range metric.Dimensions { dimNames += *dim.Name dimValues += *dim.Value - if i != len(metricDims)-1 { + if i != len(metric.Dimensions)-1 { dimNames += "," dimValues += "," } @@ -229,39 +252,25 @@ func getIdentifiers(listMetricsOutputs []cloudwatch.Metric) map[string][]string return identifiers } -func initEvent(regionName string) mb.Event { - event := mb.Event{} - event.Service = metricsetName - event.RootFields = common.MapStr{} - event.MetricSetFields = common.MapStr{} - event.RootFields.Put("service.name", metricsetName) - if regionName != "" { - event.RootFields.Put("cloud.region", regionName) - } - return event -} - func insertMetricSetFields(event mb.Event, metricValue float64, labels []string) mb.Event { - event.MetricSetFields.Put("namespace", labels[1]) - event.MetricSetFields.Put(labels[0], metricValue) + event.MetricSetFields.Put("metrics."+labels[metricNameIdx], metricValue) + event.MetricSetFields.Put("namespace", labels[namespaceIdx]) if len(labels) == 2 { return event } - dimNames := strings.Split(labels[2], ",") - dimValues := strings.Split(labels[3], ",") + dimNames := strings.Split(labels[identifierNameIdx], ",") + dimValues := strings.Split(labels[identifierValueIdx], ",") for i := 0; i < len(dimNames); i++ { - event.MetricSetFields.Put(dimNames[i], dimValues[i]) + event.MetricSetFields.Put("dimensions."+dimNames[i], dimValues[i]) } return event } func convertConfigToListMetrics(cloudwatchConfig Config, namespace string) cloudwatch.Metric { // convert config input to []cloudwatch.Metric - metricName := cloudwatchConfig.MetricName - dimensions := cloudwatchConfig.Dimensions var cloudwatchDimensions []cloudwatch.Dimension - for _, dim := range dimensions { + for _, dim := range cloudwatchConfig.Dimensions { name := dim.Name value := dim.Value cloudwatchDimensions = append(cloudwatchDimensions, cloudwatch.Dimension{ @@ -272,23 +281,23 @@ func convertConfigToListMetrics(cloudwatchConfig Config, namespace string) cloud listMetricsOutput := cloudwatch.Metric{ Namespace: &namespace, - MetricName: &metricName, + MetricName: &cloudwatchConfig.MetricName, Dimensions: cloudwatchDimensions, } return listMetricsOutput } -func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwatch.Metric, period int, startTime time.Time, endTime time.Time, report mb.ReporterV2) error { +func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwatch.Metric, regionName string, period int, startTime time.Time, endTime time.Time, report mb.ReporterV2) error { identifiers := getIdentifiers(listMetricsTotal) // Initialize events map per region, which stores one event per identifierValue events := map[string]mb.Event{} for _, values := range identifiers { for _, v := range values { - events[v] = initEvent("") + events[v] = aws.InitEvent(metricsetName, regionName) } } // Initialize events for the ones without identifiers. - eventsNoIdentifier := []mb.Event{} + var eventsNoIdentifier []mb.Event // Construct metricDataQueries metricDataQueries := constructMetricQueries(listMetricsTotal, int64(period)) @@ -314,10 +323,10 @@ func createEvents(svc cloudwatchiface.CloudWatchAPI, listMetricsTotal []cloudwat if exists { labels := strings.Split(*output.Label, " ") if len(labels) == 4 { - identifierValue := labels[3] + identifierValue := labels[identifierValueIdx] events[identifierValue] = insertMetricSetFields(events[identifierValue], output.Values[timestampIdx], labels) } else { - eventNew := initEvent("") + eventNew := aws.InitEvent(metricsetName, regionName) eventNew = insertMetricSetFields(eventNew, output.Values[timestampIdx], labels) eventsNoIdentifier = append(eventsNoIdentifier, eventNew) } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index edd95241540..245caeb24d1 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -81,6 +81,11 @@ var ( MetricName: &metricName5, Namespace: &namespaceRDS, } + + listMetric7 = cloudwatch.Metric{ + MetricName: &metricName1, + Namespace: &namespace, + } ) func TestGetIdentifiers(t *testing.T) { @@ -124,15 +129,17 @@ func TestConstructLabel(t *testing.T) { func TestReadCloudwatchConfig(t *testing.T) { cases := []struct { - cloudwatchMetricsConfig []Config - expectedListMetric []cloudwatch.Metric - expectedNamespace []string + cloudwatchMetricsConfig []Config + expectedListMetricWithRegion map[string][]cloudwatch.Metric + expectedListMetricWithoutRegion []cloudwatch.Metric + expectedNamespace []string }{ { []Config{ { Namespace: "AWS/EC2", MetricName: "CPUUtilization", + Region: "us-west-1", Dimensions: []Dimension{ { Name: "InstanceId", @@ -141,7 +148,10 @@ func TestReadCloudwatchConfig(t *testing.T) { }, }, }, - []cloudwatch.Metric{listMetric1}, + map[string][]cloudwatch.Metric{ + "us-west-1": {listMetric1}, + }, + nil, nil, }, { @@ -160,6 +170,7 @@ func TestReadCloudwatchConfig(t *testing.T) { Namespace: "AWS/EBS", }, }, + map[string][]cloudwatch.Metric{}, []cloudwatch.Metric{listMetric1}, []string{"AWS/EBS"}, }, @@ -168,6 +179,7 @@ func TestReadCloudwatchConfig(t *testing.T) { { Namespace: "AWS/EC2", MetricName: "CPUUtilization", + Region: "us-west-1", Dimensions: []Dimension{ { Name: "InstanceId", @@ -181,6 +193,7 @@ func TestReadCloudwatchConfig(t *testing.T) { { Namespace: "AWS/RDS", MetricName: "CommitThroughput", + Region: "us-west-1", Dimensions: []Dimension{ { Name: "DBClusterIdentifier", @@ -193,13 +206,34 @@ func TestReadCloudwatchConfig(t *testing.T) { }, }, }, - []cloudwatch.Metric{listMetric1, listMetric6}, + map[string][]cloudwatch.Metric{ + "us-west-1": {listMetric1, listMetric6}, + }, + nil, + []string{"AWS/EBS"}, + }, + { + []Config{ + { + Namespace: "AWS/EC2", + MetricName: "CPUUtilization", + Region: "us-east-1", + }, + { + Namespace: "AWS/EBS", + }, + }, + map[string][]cloudwatch.Metric{ + "us-east-1": {listMetric7}, + }, + nil, []string{"AWS/EBS"}, }, } for _, c := range cases { - listMetrics, namespaces := readCloudwatchConfig(c.cloudwatchMetricsConfig) - assert.Equal(t, c.expectedListMetric, listMetrics) + listMetricsWithRegion, listMetricsWithoutRegion, namespaces := readCloudwatchConfig(c.cloudwatchMetricsConfig) + assert.Equal(t, c.expectedListMetricWithRegion, listMetricsWithRegion) + assert.Equal(t, c.expectedListMetricWithoutRegion, listMetricsWithoutRegion) assert.Equal(t, c.expectedNamespace, namespaces) } } diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 90a123e21e1..afb0ff7cd38 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsWktvG0cSvvtXFHJKAGuwsZ096LCAIwsbAdlECWXkOC52Fzm96uke94MUBf/4RfU8+NCQoqgZ5rI8GDJ72PV9VdX16rmAe1pdAi79G4CggqZL+A6X/rs3AJK8cKoKyppL+NcbAIAvuPRfoLQyagJhtSYRPHz8awKlNSpYp8wcSgpOCQ8zZ8u0dqVtlEsMosjeADjShJ4uYY5vAGaKtPSXafcLMFhSi4Y/YVXxg87GqvmmB9T2JpsbiU5wt9S3595968+X9TZfQFgTUBkPoaCOaSgwwJIcgRcOK5I73P/i38KyUKJYb9CjMU8mwHQFUs1m5Pg/zMNXKMhnG5g6FU4p4BPWJN69ii6Jd2fiyT+8vnrXS22OG9/uGnjLyFXMgg2os0qErSda8l6gJpnPtMXdBw7ogT93BUFFTpAJOCewM0CtrcBAkoGDsGUVA0E0KjTqQUcgomPr6RUoA9ETWJP0qIwPaARle4kIR1KFPHqc0whcTCyn5JjH1e1nqIV58FVjj02MMLMuPRWD0uoRedtncU9R829HRU7oDMktArXizRp7gR5QCBdJglf8jQqwRA8aoxEFSbAOfEAXSO4n5aOrdPT5Gck1IreZFbggmBKZtaXQQDRalYo9saO9LMgA/+zq9vNV2uHnGjMsUEcC5eGRnD2Wsc9FgW5OclzKiVMvcT5LxgaoUEmQdmmY+lP7vwU0sgk7oYgelBHRsY5QSsUoUENNpZ+6obC07j5TJqtQ3FPwozJuZIAjQWrBzmg4rrQwQJlAbsZBf/dQHoZvYzgr/hTGbQxD4Vcmm64CjQs+SRhF9efCPpTapfL3ymaOUI6C/edG09iUCYy1C1U+WEewsDqW5AEXqDRONUGwxyNfOhVoROi8fyDDmAbHnrRuq6GBX9my0sRJIendVuRS5vanm4BrGOQoLdRMkeR6SFnJ7hhUeYyBxmSZJGzSPNFWp5D0AUP0mShI3OczVHpPotTWzF/G70+qrAue83koyG0j5dqmQu9JwtSGYnuxxgQJU8qKvOpXPlC5vabqilSjD1AqE8PxJPN6vzNzHYNIK+dvoNJvsWPJdCFGWMf/RNPf+XBKmJN7daNgHfnUEDwf37pVVeKcMtV/Ju5ptbRud+0IYDef0qFkGLw/d1eSD3Nd2L8E37ovzdgG/U5wEs4bIxX3iGtPkBSSx202w8oDGY5FezqQDmjl1AIDZdL4nJeGVWizO3z6bZIEt+p9UlUciVJV/Z64+/ULoN3cLj5wKe/Ie0DvrVCpA1+qJvy9GGucaiXGUmja/Ik+j/TKBtqAWmwV1+C45uCiBNzcdivfs4J/gKmNRraJ8aUqTUcoE1b2a/PkQJT23dXhW+D+Hn7858VUBYjGq7lJfXASchTS4e3eixS+r8hIPu7fwEVj6r98EUNQZn6RetpvEMiVyiSf/sYVS1U1z/GfJH94hlEouNjzeUUu51A9Vipo5HBx1KWF7Mn40b/PJSq9yrn02h5jvXwWubvZzmAyrUGzNuqYcvL++QHs4TnlNHLXPLDj/dZEGITJ+1bCIelePY7TMaVIU3IhwnAkBuRipmlYuQaXdYl9CGPtabmd5Xb6XxKjTDHSpHjDpxtJPRhTvcO9ROteQnP55ns93tHXSD681tebbTa8vPnm/759wG8aHfn6GmCgYr7PWX65u7vtpEGJMvWRaOBjiY/WrHG+BUdzdFK3iXdV7UmgHfY59ZfwpyHfwfzv67sd3Ozcre+z0z/l8AzeKo6I9/bz4HgladpT5A8C+dP1r9d310OjLgiHGir0YP7l+uOno/z5OV+wfkxn+H2y6w0nofSkac8N4WtxrpFMrn+9vrqD35PR4cqawIF2YK+omeReoDE0zvC2b/DcJvZGbt1/HU39NUwdhej+Dqqt4HNw1WrMU9RhSxUEy0oVtKih+zrRHsLJvYq2KMe3Qm2Ctbx0YI5Lu8uCSyMm5shX1nhuxYSOkrhhnFq56icXq3NSa6XVtmjKNMCu2GOcb18e6cg563z24eFhPDf68PAAQiv29iSum19aSUfZqD5J2NxI2xmQSiOyf4B18ONBYj+NSeynhwfw5BbkzkhMYyAjVtlMOR9ydo6s7Pe+0zhW5C5apwqqpLpZqM99fYWy9jni9qC7F63fMXjCMdj6JYOtE5ZepEhXL1PqIuZhwqmybtudYTmTxsrXVzh7uCdtp6O45tuMvdLkPa2kBum509d1gF/961q/r/6Mr1xN/pi8tuGzWpIPeUne45xynFPmSQxoRawqZx9UiYGgefWK1VLLBWPNRV3QS2gwtHcnXyPFPb1W82TqBXA12G3d3XY0aaVsAVq/z9LITpdzxoaNK8k6y2GaFqqyJKkwkN6TsDouxoZ8obya6nGam45Ox0AZmGk1L/ZkoQ7ZWVDtqi84RQvU68N+pD+wK42LtPXXFyFr49O40Loqd7oCgVr7Nhz+WYv/T3PEUOx/C7CDzJFmZJtLWUdsPKRDKquwyhsFDplg1oh21PPx9qZVH58VqeoTXmsXsCWw536azDqePjOm7cE9s67EcAl9Pzrm8kI90gk6rpeGHftN/pg0MTPt+78AAAD//3QEcdg=" + return "eJzsWktz2zgSvudXdOU0MxWzdpLMHnzYqozj2nHVPDwjp+bItICWiDUI0HhIlis/fgsASb2op0ntZXVIOQKF/r5+obvBK3ikxTXg3L4BcMJJuoa3OLdv3wBwssyIygmtruFfbwAAvuLcfoVScy8JmJaSmLPw6e8RlFoJp41QUyjJGcEsTIwu49qN1J7P0bEiewNgSBJauoYpvgGYCJLcXsfdr0BhSQ2a8HGLKjxotK/qbzpArW+yuhFrBbdLXXvu3Dd9vi63+QpMK4dCWXAFtUxdgQ7mZAgsM1gR3+D+d/gtzAvBiuUGHRqzpByMF8DFZEIm/CfwsBUystkKplaFY3K48v2mHrp1kbWbrj3YaOaRFnNt+MbaHv2Ez0NBS6xgK2JiIojDvCAFT57MYgUAYCWyQyBrnWQ/dILU4/8QcxtL6cs8PcG1H8tNhitP5CVWlVDT+vG3P7w9jfFvq7Y35LxRjd1v1pgm+gf5clGSskKr8yl3W65HzivEknlgBfVW9BF7/6qwI/b+QvEWfnh7874zxKZHB1jlM6cdyqzaMlMibxlK4vlEatx84IjwqsgwUg6nBHoCKKVm6IgH4MB0WXlH4JVwtXrQEDBvQhaRCxAKvCXQKupRKOtQMdrhlJXPmCEuXO4tTruzxOu4KF+OyQQeN/dfIAmzIW0ke6xihIk28SnvhBQvGLY9iHuMMvx2UOSEMeJXCSTFqyX2Ai0gY8YTByvCN8LBHC1I9IoVxEEbsA6NI76blPWmkt7mFyRXi1xnVuCMYEyklpZCBV5JUYrgiS3tmPXDz27uv9zEHX5OmGGG0hMICy9k9LGMbc4KNFPazGw9U46cOomHWFLaQYWCA9dzFahv2/8doOJ12nGFtyAU8yboCDkXAQVKSFS6qStyc20eM6GyCtkjOTso41oGGGIkZsEZVcgrDQwQypGZhOJjMyj3w9feXRR/TOPau77wC5WNF46GBR8lDKL6S2HvS+1c2EehM0PIB8H+c61prMuEgLVNVdZpQzDT0pdkAWcoJI4lgdPHI58b4WhA6GF/Rypg6h171Lqu+gZ+o8tKUjgUot51RSae3PZ8E4QaBldai4qM0Dy4oxPlMQYakmWUsErzTFudQ9I6dN5mrCD2mE9QyB0HpdRqehq/v6jSxtlwnruCzDrSUNtUaC1xGGtXrC8mTBAxxVMxrNqFdVSur4lUkUq0DkqhvDueZJ72uzDXIYg0cv4HVLotdiyZNsUwbcI/XnV3PuFImJJ5daOgDdnYEBzOb+2qKHFKmeiOibOHHXefY1AGGGH/0F3xEMypsD8F37IvzYINehzK3CkuQo+49AROLnrcajMsLJAKuWhHB9ICrYyYoaOMK5uHpX4VWu8On38fRcGNereqiiNRiqrbEze/PgHa3f3sYyjlDVkLaK1mInbgc1Gnv5Ox+rEUbCiFxs239HmkV9bQetRio7gax21ILoLB3X278l1Q8Pcw1l7x5mA8VaUxhDKmebc2z05Ecd9NHb6D0N/Dj/+8GgsHXlkxVbEPjkKOQtq/3TuRwncVKR7C/RsYr1T6yxbeOaGmV7Gn/QaOTClU9OlvoWKJw8LmT+LfH2DkilDs2bwik4dUPdRRUMsJxVF7LGyPH+2HnKOQizyUXutjrNNnkZubbQwm4xrUa4OOKUcfXnsRMPaha+7Z8X6vMwzC6EMjYZ90K16G6ZhipilDIRLgcHQYipm6YQ01OE8l9j6MydNyPcnT7HwIjHFSvOLTtaQOjLHeCb1E415MhvKtY+BuP+SGnjxZ91pfr7dZ8fL6m//79h6/qXVk0zVAT8V8l7P88vBw30qDEnnsI1HBpxJftFrifAeGpmi4bA7eRbXjAG2xT6m7hD8P+Qbmf98+bOAOzt34fnD6bQ4H8FZ+QLz3X3rHy0nSjiK/F8ifb3+9fbjtG3VB2NdQoQPzL7efPh/lz4d8QdshneGP0aY3nIXSkty+yO0H5xLJ6PbX25sH+CMaHW60ciHR9uwViUluGSpFwwxvuwbPzcFey03919HUX8O0uee/ONX2BYMLcJViyChqscUKIsiKFTRL0G06aPfhDL2K1MiHt0IywVJeDJjjjt15EUqjQMyQrbSyoRVj0nMKDeNY8x0vhPjqktQaackWdZkG2BZ7Aee70zMdGaONzT4+Pw/nRh+fn4FJEbw9imvnl5rTUTZKkYT1jbSeAIk4IvsHaAM/7iX205DEfnp+BktmRuaCxCQ6UmyRTYSxLg/OkZXd3ncex4rMVeNUTpSUmoUU9+kKZelzFNqD9l40vWOwxdHp9JLBWoTFFyni1cuY2oy5n3CsrJt2p1/OJLGy6QpnB/eo7RiKS7712CtO3uNKbJAORV/bAT7Z17V+T/aCr1yN/hy9tuHTkpN1eUnW4pRynFJmifVoRawqo59FiY6gfvUqqCXJBaXVVSroOdQYmruTJ09+R69VPxl7AVz0dlv3sJ5NGilrgJbvs9Sy4+Wc0m7lSjKdchinhaIsiQt0JHccWC0XpV0+E1Zsv3TZT3Js6bQMhIKJFNNixynUIrsIqk31OSNohnIZ7Ef6Q3ClYZE2/noSsiY/DQutrXLHC2AopW3S4V9J/G91iCHb/RZgCzlkmoFtznnK2LhPh1RWbpHXCuzzgFki2lDPp/u7Rn0hVrhIEZ60C9gQ2HE/TWqZTw+MaTtwT7Qp0V1D14+OubwQL3SGjtNSv2O/0Z+jOmfGff8bAAD//7YzJPQ=" } diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 7f43cbb3b76..7ffab2a625f 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -36,8 +36,7 @@ func GetListMetricsOutput(namespace string, regionName string, svcCloudwatch clo // List metrics of a given namespace for each region listMetricsOutput, err := reqListMetrics.Send() if err != nil { - err = errors.Wrap(err, "ListMetricsRequest failed, skipping region "+regionName) - return nil, err + return nil, errors.Wrap(err, "ListMetricsRequest failed, skipping region "+regionName) } if listMetricsOutput.Metrics == nil || len(listMetricsOutput.Metrics) == 0 { @@ -58,8 +57,7 @@ func getMetricDataPerRegion(metricDataQueries []cloudwatch.MetricDataQuery, next reqGetMetricData := svc.GetMetricDataRequest(getMetricDataInput) getMetricDataOutput, err := reqGetMetricData.Send() if err != nil { - err = errors.Wrap(err, "Error GetMetricDataInput") - return nil, err + return nil, errors.Wrap(err, "Error GetMetricDataInput") } return getMetricDataOutput, nil } @@ -85,8 +83,7 @@ func GetMetricDataResults(metricDataQueries []cloudwatch.MetricDataQuery, svc cl output, err := getMetricDataPerRegion(metricDataQueriesPartial, getMetricDataOutput.NextToken, svc, startTime, endTime) if err != nil { - err = errors.Wrap(err, "getMetricDataPerRegion failed") - return getMetricDataOutput.MetricDataResults, err + return getMetricDataOutput.MetricDataResults, errors.Wrap(err, "getMetricDataPerRegion failed") } getMetricDataOutput.MetricDataResults = append(getMetricDataOutput.MetricDataResults, output.MetricDataResults...) From f43120ddd6d5595e6009a665e5babc2fa82d2d67 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 24 Apr 2019 16:48:40 -0600 Subject: [PATCH 12/17] Fix unit test --- x-pack/metricbeat/module/aws/ec2/ec2_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index 8b3bdae475d..73badaa7e10 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -123,7 +123,7 @@ func TestGetInstanceIDs(t *testing.T) { func TestCreateCloudWatchEvents(t *testing.T) { mockModuleConfig := aws.Config{ - Period: "300s", + Period: time.Duration(300) * time.Second, DefaultRegion: regionName, } From 75f2ea45036e379333e7da455f7d372768006e24 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 25 Apr 2019 10:44:50 -0600 Subject: [PATCH 13/17] Remove extra cloudwatch from field names --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 6 ++-- metricbeat/docs/modules/aws.asciidoc | 1 + x-pack/metricbeat/metricbeat.reference.yml | 1 + x-pack/metricbeat/module/aws/_meta/config.yml | 1 + .../module/aws/cloudwatch/_meta/fields.yml | 6 ++-- .../module/aws/cloudwatch/cloudwatch.go | 31 ++++++++++--------- x-pack/metricbeat/module/aws/ec2/ec2_test.go | 2 +- x-pack/metricbeat/module/aws/fields.go | 2 +- x-pack/metricbeat/modules.d/aws.yml.disabled | 1 + 10 files changed, 30 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 01630976e2f..bf7c19ae98e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -171,6 +171,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added new disk states and raid level to the system/raid metricset. {pull}11613[11613] - Added `path_name` and `start_name` to service metricset on windows module {issue}8364[8364] {pull}11877[11877] - Add check on object name in the counter path if the instance name is missing {issue}6528[6528] {pull}11878[11878] +- Add AWS cloudwatch metricset. {pull}11798[11798] {issue}11734[11734] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d3bc25959fd..fad706b26b2 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -837,7 +837,7 @@ Total. -*`aws.cloudwatch.cloudwatch.namespace`*:: +*`aws.cloudwatch.namespace`*:: + -- type: keyword @@ -847,7 +847,7 @@ The namespace specified when query cloudwatch api. -- -*`aws.cloudwatch.cloudwatch.metrics.*`*:: +*`aws.cloudwatch.metrics.*`*:: + -- type: object @@ -857,7 +857,7 @@ Metrics that returned from Cloudwatch api query. -- -*`aws.cloudwatch.cloudwatch.dimensions.*`*:: +*`aws.cloudwatch.dimensions.*`*:: + -- type: object diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 0f30a777645..e62d7f1b872 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -141,6 +141,7 @@ metricbeat.modules: cloudwatch_metrics: - namespace: AWS/EC2 metricname: CPUUtilization + region: us-east-1 dimensions: - name: InstanceId value: i-0686946e22cf9494a diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 07edfe8968c..d5913d605bb 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -186,6 +186,7 @@ metricbeat.modules: cloudwatch_metrics: - namespace: AWS/EC2 metricname: CPUUtilization + region: us-east-1 dimensions: - name: InstanceId value: i-0686946e22cf9494a diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index d1e541fb07d..774a970b53f 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -27,6 +27,7 @@ cloudwatch_metrics: - namespace: AWS/EC2 metricname: CPUUtilization + region: us-east-1 dimensions: - name: InstanceId value: i-0686946e22cf9494a diff --git a/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml index f78d7a869f6..875b0aac111 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/cloudwatch/_meta/fields.yml @@ -4,17 +4,17 @@ `cloudwatch` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by different namespaces. release: beta fields: - - name: cloudwatch.namespace + - name: namespace type: keyword description: > The namespace specified when query cloudwatch api. - - name: cloudwatch.metrics.* + - name: metrics.* type: object object_type: double object_type_mapping_type: "*" description: > Metrics that returned from Cloudwatch api query. - - name: cloudwatch.dimensions.* + - name: dimensions.* type: object object_type: keyword object_type_mapping_type: "*" diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 10f486c21ac..c07d1a2a770 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -99,34 +99,37 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get listMetricsTotal and namespaces from configuration listMetricsWithRegion, listMetricsWithoutRegion, namespacesTotal := readCloudwatchConfig(m.CloudwatchConfigs) - if listMetricsWithoutRegion != nil { - m.Logger().Info("cloudwatch_metrics config is missing region name. " + - "To avoid extra costs, please make sure region is set in config.yml") - } for regionName, listMetricsPerRegion := range listMetricsWithRegion { - m.MetricSet.AwsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + awsConfig := *m.MetricSet.AwsConfig + awsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(awsConfig) err := createEvents(svcCloudwatch, listMetricsPerRegion, regionName, m.PeriodInSec, startTime, endTime, report) if err != nil { return errors.Wrap(err, "createEvents failed") } } - for _, regionName := range m.MetricSet.RegionsList { - m.MetricSet.AwsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) - err := createEvents(svcCloudwatch, listMetricsWithoutRegion, regionName, m.PeriodInSec, startTime, endTime, report) - if err != nil { - return errors.Wrap(err, "createEvents failed") + if listMetricsWithoutRegion != nil { + m.Logger().Info("cloudwatch_metrics config is missing region name. " + + "To avoid extra costs, please make sure region is set in config.yml") + for _, regionName := range m.MetricSet.RegionsList { + awsConfig := *m.MetricSet.AwsConfig + awsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(awsConfig) + err := createEvents(svcCloudwatch, listMetricsWithoutRegion, regionName, m.PeriodInSec, startTime, endTime, report) + if err != nil { + return errors.Wrap(err, "createEvents failed") + } } } // Use namespaces from config for _, namespace := range namespacesTotal { for _, regionName := range m.MetricSet.RegionsList { - m.MetricSet.AwsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + awsConfig := *m.MetricSet.AwsConfig + awsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(awsConfig) listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { m.Logger().Info(err.Error()) diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index 73badaa7e10..802b957afc0 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -123,7 +123,7 @@ func TestGetInstanceIDs(t *testing.T) { func TestCreateCloudWatchEvents(t *testing.T) { mockModuleConfig := aws.Config{ - Period: time.Duration(300) * time.Second, + Period: 300 * time.Second, DefaultRegion: regionName, } diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index afb0ff7cd38..2bb0e5771ba 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsWktz2zgSvudXdOU0MxWzdpLMHnzYqozj2nHVPDwjp+bItICWiDUI0HhIlis/fgsASb2op0ntZXVIOQKF/r5+obvBK3ikxTXg3L4BcMJJuoa3OLdv3wBwssyIygmtruFfbwAAvuLcfoVScy8JmJaSmLPw6e8RlFoJp41QUyjJGcEsTIwu49qN1J7P0bEiewNgSBJauoYpvgGYCJLcXsfdr0BhSQ2a8HGLKjxotK/qbzpArW+yuhFrBbdLXXvu3Dd9vi63+QpMK4dCWXAFtUxdgQ7mZAgsM1gR3+D+d/gtzAvBiuUGHRqzpByMF8DFZEIm/CfwsBUystkKplaFY3K48v2mHrp1kbWbrj3YaOaRFnNt+MbaHv2Ez0NBS6xgK2JiIojDvCAFT57MYgUAYCWyQyBrnWQ/dILU4/8QcxtL6cs8PcG1H8tNhitP5CVWlVDT+vG3P7w9jfFvq7Y35LxRjd1v1pgm+gf5clGSskKr8yl3W65HzivEknlgBfVW9BF7/6qwI/b+QvEWfnh7874zxKZHB1jlM6cdyqzaMlMibxlK4vlEatx84IjwqsgwUg6nBHoCKKVm6IgH4MB0WXlH4JVwtXrQEDBvQhaRCxAKvCXQKupRKOtQMdrhlJXPmCEuXO4tTruzxOu4KF+OyQQeN/dfIAmzIW0ke6xihIk28SnvhBQvGLY9iHuMMvx2UOSEMeJXCSTFqyX2Ai0gY8YTByvCN8LBHC1I9IoVxEEbsA6NI76blPWmkt7mFyRXi1xnVuCMYEyklpZCBV5JUYrgiS3tmPXDz27uv9zEHX5OmGGG0hMICy9k9LGMbc4KNFPazGw9U46cOomHWFLaQYWCA9dzFahv2/8doOJ12nGFtyAU8yboCDkXAQVKSFS6qStyc20eM6GyCtkjOTso41oGGGIkZsEZVcgrDQwQypGZhOJjMyj3w9feXRR/TOPau77wC5WNF46GBR8lDKL6S2HvS+1c2EehM0PIB8H+c61prMuEgLVNVdZpQzDT0pdkAWcoJI4lgdPHI58b4WhA6GF/Rypg6h171Lqu+gZ+o8tKUjgUot51RSae3PZ8E4QaBldai4qM0Dy4oxPlMQYakmWUsErzTFudQ9I6dN5mrCD2mE9QyB0HpdRqehq/v6jSxtlwnruCzDrSUNtUaC1xGGtXrC8mTBAxxVMxrNqFdVSur4lUkUq0DkqhvDueZJ72uzDXIYg0cv4HVLotdiyZNsUwbcI/XnV3PuFImJJ5daOgDdnYEBzOb+2qKHFKmeiOibOHHXefY1AGGGH/0F3xEMypsD8F37IvzYINehzK3CkuQo+49AROLnrcajMsLJAKuWhHB9ICrYyYoaOMK5uHpX4VWu8On38fRcGNereqiiNRiqrbEze/PgHa3f3sYyjlDVkLaK1mInbgc1Gnv5Ox+rEUbCiFxs239HmkV9bQetRio7gax21ILoLB3X278l1Q8Pcw1l7x5mA8VaUxhDKmebc2z05Ecd9NHb6D0N/Dj/+8GgsHXlkxVbEPjkKOQtq/3TuRwncVKR7C/RsYr1T6yxbeOaGmV7Gn/QaOTClU9OlvoWKJw8LmT+LfH2DkilDs2bwik4dUPdRRUMsJxVF7LGyPH+2HnKOQizyUXutjrNNnkZubbQwm4xrUa4OOKUcfXnsRMPaha+7Z8X6vMwzC6EMjYZ90K16G6ZhipilDIRLgcHQYipm6YQ01OE8l9j6MydNyPcnT7HwIjHFSvOLTtaQOjLHeCb1E415MhvKtY+BuP+SGnjxZ91pfr7dZ8fL6m//79h6/qXVk0zVAT8V8l7P88vBw30qDEnnsI1HBpxJftFrifAeGpmi4bA7eRbXjAG2xT6m7hD8P+Qbmf98+bOAOzt34fnD6bQ4H8FZ+QLz3X3rHy0nSjiK/F8ifb3+9fbjtG3VB2NdQoQPzL7efPh/lz4d8QdshneGP0aY3nIXSkty+yO0H5xLJ6PbX25sH+CMaHW60ciHR9uwViUluGSpFwwxvuwbPzcFey03919HUX8O0uee/ONX2BYMLcJViyChqscUKIsiKFTRL0G06aPfhDL2K1MiHt0IywVJeDJjjjt15EUqjQMyQrbSyoRVj0nMKDeNY8x0vhPjqktQaackWdZkG2BZ7Aee70zMdGaONzT4+Pw/nRh+fn4FJEbw9imvnl5rTUTZKkYT1jbSeAIk4IvsHaAM/7iX205DEfnp+BktmRuaCxCQ6UmyRTYSxLg/OkZXd3ncex4rMVeNUTpSUmoUU9+kKZelzFNqD9l40vWOwxdHp9JLBWoTFFyni1cuY2oy5n3CsrJt2p1/OJLGy6QpnB/eo7RiKS7712CtO3uNKbJAORV/bAT7Z17V+T/aCr1yN/hy9tuHTkpN1eUnW4pRynFJmifVoRawqo59FiY6gfvUqqCXJBaXVVSroOdQYmruTJ09+R69VPxl7AVz0dlv3sJ5NGilrgJbvs9Sy4+Wc0m7lSjKdchinhaIsiQt0JHccWC0XpV0+E1Zsv3TZT3Js6bQMhIKJFNNixynUIrsIqk31OSNohnIZ7Ef6Q3ClYZE2/noSsiY/DQutrXLHC2AopW3S4V9J/G91iCHb/RZgCzlkmoFtznnK2LhPh1RWbpHXCuzzgFki2lDPp/u7Rn0hVrhIEZ60C9gQ2HE/TWqZTw+MaTtwT7Qp0V1D14+OubwQL3SGjtNSv2O/0Z+jOmfGff8bAAD//7YzJPQ=" + return "eJzsWktz2zgSvudXdOU0MxWrdpLMHnzYqozj2knVPDxjp+bItICWiDUIMHhYlis/fqsBknpRtiyT2svqkHIECv19/UJ3g2dwS8tzwIV/BRBU0HQOr3HhX78CkOSFU3VQ1pzDv14BAHzBhf8ClZVREwirNYng4cPf11BZo4J1ysyhouCU8DBztkprF9pGucAgyskrAEea0NM5zPEVwEyRlv487X4GBitq0fAnLGt+0NlYN9/0gNrcZH0j0Qnulvr23Ltv/nxZbfMFhDUBlfEQSuqYhhIDLMgReOGwJrnF/W/+LSxKJcrVBj0a82QCTJcg1WxGjv/DPHyNgvxkDVOnwikFXPt+Ww/ruuh22lht1XFLy4V1cmvtEaXw56ak1bbgaxJqpkjCoiQDXyO55ZoFAGs16UXWsJ/80IvMTv9DImwt5S+L/IS0caq3aa09UVRY18rMm8df//D6eTR/W7eyoxCdaS18sUEvc+4nKVVFxitrjufZb6MBia6xyTZZR70TXCTeviiqSLw9UTjxDy8v3vZG0PzQ+BF1nAQbUE/qHTNl8l6gJlnMtMXtBw4IpJqcIBNwTmBngFpbgYEkAwdhqzoGgmhUaNSDjkBEx0lCL0EZiJ7AmqRHZXxAI6jfE5mIcCRVKKLHeX8+eBkXE6spOeZxcfUZsjDPCSLbYx0jzKxLT8WgtHpA3vZJ3FPU/NtRkROmMF8nkBVvVthL9IBCuEgSvOJvVIAFetAYjShJgnXgA7pAcj8pH12toy9OSK4RucmsxDuCKZFZWQoNRKNVpdgTO9opv/PPLq4+X6Qdfs6Y4Q51JFAeHsjZQxn7QpTo5rSd2QamnDj1EudYMjZAjUqCtAvD1Hft/wbQyCbthDJ6UEZExzpCKRWjQA2ZSj91Q2Fh3e1EmUmN4paCH5VxIwMcCVJ37IyG80oLA5QJ5GZcW2wH5ePwbQwnxZ/SuI1hKPzKTKbLQOOCTxJGUf2psA+ldqn8rbITRyhHwf5zo2lsygTG2qUqH6wjuLM6VuQB71BpnGqCYA9HvnAq0IjQef9AhjENjj1p3dZDA7+wVa2JD4Wkd1uTSye3P94EXMPgWhNRk1NWsjsGVR1ioDFZJgnrNI+01TEkfcAQ/USUJG6LGSq956DU1syfx+8vqq0Lns/zUJLbRMq1TY3ek4SpDeXmYsYECVM6FXnVL32ganNN5YpUow9QKRPD4SSLvN+JuY5BpJXzP6DSb7FDyXQpRljH/0TT3/nwkTAn9+JGwTryqSF4Or91q6rCOU1Uf0wcPdb49DEFJcPg/bm7khzMubB/Dr5VXzphGww4fvlkpOIeceUJkkLyuPVmWHkgw7loTwfSAa2dusNAE2l8wUvDKrTZHT7+fp0Et+rdqSoORKnqfk/c/voZ0D5d3b3nUt6R94DeW6FSB75QTfp7NtY41UqMpdC0+Y4+D/TKBtqAWmwV1+C45OSiBHy66la+YwV/D1MbjWwPxueqNIXQRFjZr82jE1Had1uHb4D7e/jxn2dTFSAar+Ym9cFJyEFIh7d7L1L4riYjOdy/gYvG5L98GUNQZn6WetpvEMhVyiSf/sYVSxoWtn+S/P4JRqHkYs8XNbmCU/VYR0Ejh4uj7ljYHT/6d4VEpZcFl16bY6znzyK3N9saTKY1aNZGHVNev3vpnH8auWse2PF+bzIMwvW7VsJj0r16GKdjSpmm4kKE4UgMyMVM07ByDS5zif0YxuxphZ0VeXY+BsY0KV7z6UZSD8ZU73Av0bqX0Fy+9Qzc/bvC0ddIPrzU15tt1ry8+eb/vv2I3zQ68vkaYKBivs9Zfrm5ueqkQYUy9ZFo4EOFD9ascL4BR3N0UrcH77Lec4B22OfUX8Ifh3wL878vb7Zws3O3vs9Ov8vhCbx1HBHv1efB8UrStKfIHwTyx8tfL28uh0ZdEg41VOjB/Mvlh48H+fNTvmD9mM7wx/W2NxyF0pPevcgdBucKyfXlr5cXN/BHMjpcWBM40Q7sFZlJ4QUaQ+MMb/sGz+3B3sjN/dfB1F/CtL3cPznV7q2CE3DVaswo6rClCoJlpQpaZOg+H7SP4eReRVuU41shm2AlLwXMYcfuouTSiIk58rU1nlsxoaMkbhinVu55CyTWp6TWSsu2aMo0wK7YY5xvnp/pyDnr/OT9/f14bvT+/h6EVuztSVw3v7SSDrJRjiRsbqTtDEilEdk/wDr48VFiP41J7Kf7e/Dk7sidkJjGQEYsJzPlfCjYOSZVv/cdx7Emd9Y6VVAV5WYhx32+Qln5HHF70N2L5ncMdjgGm18y2Iiw9CJFunqZUpcxHyecKuu23RmWM2msfb7C2cM9aTuF4opvM/ZKk/e0khqkp6Kv6wC/+pe1fl/9CV+5uv7z+qUNn9WSfCgq8h7nVOCcJp7EgFbEunb2XlUYCJpXr1gtWS4Ya85yQS+hwdDenXyNFPf0Ws2TqRfA5WC3dTeb2aSVsgFo9T5LIztdzhkb1q4k8ymHaVqoqoqkwkB6z4HVcTE2FHfKq903LYdJjh2djoEyMNNqXu45hTpkJ0G1rb7gFN2hXgX7gf7ArjQu0tZfn4WszU/jQuuq3OkSBGrt23T4Vxb/WxNiKPa/BdhB5kwzss2lzBkbH9MhVXVYFo0ChzxgVoi21PPh6lOrPo4VqXKEZ+0CtgT23E+TWeXTJ8a0Pbhn1lUYzqHvR4dcXqgHOkLHeWnYsd/1n9dNzkz7/jcAAP//AykX4A==" } diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index d1e541fb07d..774a970b53f 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -27,6 +27,7 @@ cloudwatch_metrics: - namespace: AWS/EC2 metricname: CPUUtilization + region: us-east-1 dimensions: - name: InstanceId value: i-0686946e22cf9494a From b41a0aeed2106bfbfaadfcf2d9c34fe862ad8ec8 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 26 Apr 2019 13:26:21 -0600 Subject: [PATCH 14/17] Remove region config in cloudwatch metricset --- metricbeat/docs/modules/aws.asciidoc | 8 +++- x-pack/metricbeat/metricbeat.reference.yml | 1 - x-pack/metricbeat/module/aws/_meta/config.yml | 1 - .../metricbeat/module/aws/_meta/docs.asciidoc | 7 ++- .../module/aws/cloudwatch/cloudwatch.go | 47 +++++-------------- x-pack/metricbeat/modules.d/aws.yml.disabled | 20 +------- 6 files changed, 25 insertions(+), 59 deletions(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index e62d7f1b872..a0d1c1252e3 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -9,7 +9,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for AWS services. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`. +The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch`. [float] === Module-specific configuration notes @@ -102,6 +102,11 @@ https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[Ho Configure Request Metrics for S3] for instructions on how to enable request metrics for each S3 bucket. +[float] +=== `cloudwatch` +This metricset gives users the freedom to query metrics from AWS Cloudwatch with +any given namespaces or specific instance with a given period. + [float] === Example configuration @@ -141,7 +146,6 @@ metricbeat.modules: cloudwatch_metrics: - namespace: AWS/EC2 metricname: CPUUtilization - region: us-east-1 dimensions: - name: InstanceId value: i-0686946e22cf9494a diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index d5913d605bb..07edfe8968c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -186,7 +186,6 @@ metricbeat.modules: cloudwatch_metrics: - namespace: AWS/EC2 metricname: CPUUtilization - region: us-east-1 dimensions: - name: InstanceId value: i-0686946e22cf9494a diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 774a970b53f..d1e541fb07d 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -27,7 +27,6 @@ cloudwatch_metrics: - namespace: AWS/EC2 metricname: CPUUtilization - region: us-east-1 dimensions: - name: InstanceId value: i-0686946e22cf9494a diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index d4a39dc7cf4..987959db0b4 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -2,7 +2,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for AWS services. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`. +The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch`. [float] === Module-specific configuration notes @@ -94,3 +94,8 @@ always adjust this to the granularity they want. Request metrics are not enabled https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[How to Configure Request Metrics for S3] for instructions on how to enable request metrics for each S3 bucket. + +[float] +=== `cloudwatch` +This metricset gives users the freedom to query metrics from AWS Cloudwatch with +any given namespaces or specific instance with a given period. diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index c07d1a2a770..486b01e8cc3 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -55,7 +55,6 @@ type Dimension struct { type Config struct { Namespace string `config:"namespace" validate:"nonzero,required"` MetricName string `config:"metricname"` - Region string `config:"region"` Dimensions []Dimension `config:"dimensions"` } @@ -97,37 +96,22 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return errors.Wrap(err, "error GetStartTimeEndTime") } - // Get listMetricsTotal and namespaces from configuration - listMetricsWithRegion, listMetricsWithoutRegion, namespacesTotal := readCloudwatchConfig(m.CloudwatchConfigs) - - for regionName, listMetricsPerRegion := range listMetricsWithRegion { - awsConfig := *m.MetricSet.AwsConfig + // Get listMetrics and namespacesTotal from configuration + listMetrics, namespacesTotal := readCloudwatchConfig(m.CloudwatchConfigs) + for _, regionName := range m.MetricSet.RegionsList { + awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName svcCloudwatch := cloudwatch.New(awsConfig) - err := createEvents(svcCloudwatch, listMetricsPerRegion, regionName, m.PeriodInSec, startTime, endTime, report) + err := createEvents(svcCloudwatch, listMetrics, regionName, m.PeriodInSec, startTime, endTime, report) if err != nil { return errors.Wrap(err, "createEvents failed") } } - if listMetricsWithoutRegion != nil { - m.Logger().Info("cloudwatch_metrics config is missing region name. " + - "To avoid extra costs, please make sure region is set in config.yml") - for _, regionName := range m.MetricSet.RegionsList { - awsConfig := *m.MetricSet.AwsConfig - awsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(awsConfig) - err := createEvents(svcCloudwatch, listMetricsWithoutRegion, regionName, m.PeriodInSec, startTime, endTime, report) - if err != nil { - return errors.Wrap(err, "createEvents failed") - } - } - } - // Use namespaces from config for _, namespace := range namespacesTotal { for _, regionName := range m.MetricSet.RegionsList { - awsConfig := *m.MetricSet.AwsConfig + awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName svcCloudwatch := cloudwatch.New(awsConfig) listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) @@ -150,29 +134,20 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return nil } -func readCloudwatchConfig(cloudwatchConfigs []Config) (map[string][]cloudwatch.Metric, []cloudwatch.Metric, []string) { - listMetricsWithRegion := map[string][]cloudwatch.Metric{} - var listMetricsWithoutRegion []cloudwatch.Metric +func readCloudwatchConfig(cloudwatchConfigs []Config) ([]cloudwatch.Metric, []string) { + var listMetrics []cloudwatch.Metric var namespacesTotal []string for _, cloudwatchConfig := range cloudwatchConfigs { namespace := cloudwatchConfig.Namespace if cloudwatchConfig.MetricName != "" { listMetricsOutput := convertConfigToListMetrics(cloudwatchConfig, namespace) - if cloudwatchConfig.Region != "" { - if listMetricsWithRegion[cloudwatchConfig.Region] != nil { - listMetricsWithRegion[cloudwatchConfig.Region] = append(listMetricsWithRegion[cloudwatchConfig.Region], listMetricsOutput) - } else { - listMetricsWithRegion[cloudwatchConfig.Region] = []cloudwatch.Metric{listMetricsOutput} - } - } else { - listMetricsWithoutRegion = append(listMetricsWithoutRegion, listMetricsOutput) - } + listMetrics = append(listMetrics, listMetricsOutput) } else { namespacesTotal = append(namespacesTotal, namespace) } } - return listMetricsWithRegion, listMetricsWithoutRegion, namespacesTotal + return listMetrics, namespacesTotal } func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) []cloudwatch.MetricDataQuery { @@ -185,6 +160,8 @@ func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, period int64) } func constructLabel(metric cloudwatch.Metric) string { + // label = metricName + namespace + dimensionKey1 + dimensionValue1 + + // dimensionKey2 + dimensionValue2 + ... label := *metric.MetricName + " " + *metric.Namespace dimNames := "" dimValues := "" diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 774a970b53f..84176ed99e5 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -1,21 +1,3 @@ -- module: aws - period: 300s - metricsets: - - ec2 - - sqs - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' -- module: aws - period: 86400s - metricsets: - - s3_request - - s3_daily_storage - access_key_id: '${AWS_ACCESS_KEY_ID:""}' - secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' - session_token: '${AWS_SESSION_TOKEN:""}' - default_region: '${AWS_REGION:us-west-1}' - module: aws period: 300s metricsets: @@ -27,8 +9,8 @@ cloudwatch_metrics: - namespace: AWS/EC2 metricname: CPUUtilization - region: us-east-1 dimensions: - name: InstanceId value: i-0686946e22cf9494a - namespace: AWS/EBS + - namespace: AWS/RDS From 3a934dfe7e73c80a85d035f574d1f10a4c23968e Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 26 Apr 2019 13:38:00 -0600 Subject: [PATCH 15/17] update doc --- metricbeat/docs/modules/aws.asciidoc | 4 ++-- .../metricbeat/module/aws/_meta/docs.asciidoc | 4 ++-- x-pack/metricbeat/modules.d/aws.yml.disabled | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index a0d1c1252e3..20a20a7077d 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -50,8 +50,8 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID}' secret_access_key: '${AWS_SECRET_ACCESS_KEY}' session_token: '${AWS_SESSION_TOKEN}' diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 987959db0b4..243aeb0ecc0 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -43,8 +43,8 @@ metricbeat.modules: - module: aws period: 300s metricsets: - - "ec2" - - "sqs" + - ec2 + - sqs access_key_id: '${AWS_ACCESS_KEY_ID}' secret_access_key: '${AWS_SECRET_ACCESS_KEY}' session_token: '${AWS_SESSION_TOKEN}' diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 84176ed99e5..d1e541fb07d 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -1,3 +1,21 @@ +- module: aws + period: 300s + metricsets: + - ec2 + - sqs + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' +- module: aws + period: 86400s + metricsets: + - s3_request + - s3_daily_storage + access_key_id: '${AWS_ACCESS_KEY_ID:""}' + secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + session_token: '${AWS_SESSION_TOKEN:""}' + default_region: '${AWS_REGION:us-west-1}' - module: aws period: 300s metricsets: @@ -13,4 +31,3 @@ - name: InstanceId value: i-0686946e22cf9494a - namespace: AWS/EBS - - namespace: AWS/RDS From d7aa6b9e2373f5357558b3b89e79f505413cce33 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 26 Apr 2019 13:59:18 -0600 Subject: [PATCH 16/17] Fix unit tests by removing Region --- .../metricbeat/module/aws/cloudwatch/cloudwatch_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 245caeb24d1..59b65666d75 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -139,7 +139,6 @@ func TestReadCloudwatchConfig(t *testing.T) { { Namespace: "AWS/EC2", MetricName: "CPUUtilization", - Region: "us-west-1", Dimensions: []Dimension{ { Name: "InstanceId", @@ -179,7 +178,6 @@ func TestReadCloudwatchConfig(t *testing.T) { { Namespace: "AWS/EC2", MetricName: "CPUUtilization", - Region: "us-west-1", Dimensions: []Dimension{ { Name: "InstanceId", @@ -193,7 +191,6 @@ func TestReadCloudwatchConfig(t *testing.T) { { Namespace: "AWS/RDS", MetricName: "CommitThroughput", - Region: "us-west-1", Dimensions: []Dimension{ { Name: "DBClusterIdentifier", @@ -217,7 +214,6 @@ func TestReadCloudwatchConfig(t *testing.T) { { Namespace: "AWS/EC2", MetricName: "CPUUtilization", - Region: "us-east-1", }, { Namespace: "AWS/EBS", @@ -231,9 +227,8 @@ func TestReadCloudwatchConfig(t *testing.T) { }, } for _, c := range cases { - listMetricsWithRegion, listMetricsWithoutRegion, namespaces := readCloudwatchConfig(c.cloudwatchMetricsConfig) - assert.Equal(t, c.expectedListMetricWithRegion, listMetricsWithRegion) - assert.Equal(t, c.expectedListMetricWithoutRegion, listMetricsWithoutRegion) + listMetrics, namespaces := readCloudwatchConfig(c.cloudwatchMetricsConfig) + assert.Equal(t, c.expectedListMetricWithoutRegion, listMetrics) assert.Equal(t, c.expectedNamespace, namespaces) } } From a518e6826a3fe8d52d02b91ec6fe63ccdba62ce5 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 26 Apr 2019 14:59:07 -0600 Subject: [PATCH 17/17] Fix unit test for cloudwatch_test.go --- .../module/aws/cloudwatch/cloudwatch_test.go | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 59b65666d75..deda6bcc9fd 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -129,10 +129,9 @@ func TestConstructLabel(t *testing.T) { func TestReadCloudwatchConfig(t *testing.T) { cases := []struct { - cloudwatchMetricsConfig []Config - expectedListMetricWithRegion map[string][]cloudwatch.Metric - expectedListMetricWithoutRegion []cloudwatch.Metric - expectedNamespace []string + cloudwatchMetricsConfig []Config + expectedListMetric []cloudwatch.Metric + expectedNamespace []string }{ { []Config{ @@ -147,10 +146,7 @@ func TestReadCloudwatchConfig(t *testing.T) { }, }, }, - map[string][]cloudwatch.Metric{ - "us-west-1": {listMetric1}, - }, - nil, + []cloudwatch.Metric{listMetric1}, nil, }, { @@ -169,7 +165,6 @@ func TestReadCloudwatchConfig(t *testing.T) { Namespace: "AWS/EBS", }, }, - map[string][]cloudwatch.Metric{}, []cloudwatch.Metric{listMetric1}, []string{"AWS/EBS"}, }, @@ -203,10 +198,7 @@ func TestReadCloudwatchConfig(t *testing.T) { }, }, }, - map[string][]cloudwatch.Metric{ - "us-west-1": {listMetric1, listMetric6}, - }, - nil, + []cloudwatch.Metric{listMetric1, listMetric6}, []string{"AWS/EBS"}, }, { @@ -219,16 +211,13 @@ func TestReadCloudwatchConfig(t *testing.T) { Namespace: "AWS/EBS", }, }, - map[string][]cloudwatch.Metric{ - "us-east-1": {listMetric7}, - }, - nil, + []cloudwatch.Metric{listMetric7}, []string{"AWS/EBS"}, }, } for _, c := range cases { listMetrics, namespaces := readCloudwatchConfig(c.cloudwatchMetricsConfig) - assert.Equal(t, c.expectedListMetricWithoutRegion, listMetrics) + assert.Equal(t, c.expectedListMetric, listMetrics) assert.Equal(t, c.expectedNamespace, namespaces) } }