From e9429e48770662ed868db8fb431e451951bba4c1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 21 Feb 2019 18:49:43 -0700 Subject: [PATCH 01/14] First patch on s3 daily storage metricset --- .../aws/s3_daily_storage/_meta/data.json | 0 .../aws/s3_daily_storage/_meta/docs.asciidoc | 10 ++ .../aws/s3_daily_storage/_meta/fields.yml | 22 +++ .../module/aws/s3_daily_storage/data.go | 37 +++++ .../aws/s3_daily_storage/s3_daily_storage.go | 142 ++++++++++++++++++ .../s3_daily_storage_integration_test.go | 73 +++++++++ 6 files changed, 284 insertions(+) create mode 100644 x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json create mode 100644 x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/aws/s3_daily_storage/data.go create mode 100644 x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go create mode 100644 x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc new file mode 100644 index 00000000000..fde92cc0964 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc @@ -0,0 +1,10 @@ +The s3_daily_storage metricset of aws module allows you to monitor your AWS S3 buckets. `s3_daily_storage` metricset +fetches Cloudwatch daily storage metrics for each S3 bucket from +https://docs.aws.amazon.com/AmazonS3/latest/dev/cloudwatch-monitoring.html[S3 CloudWatch Daily Storage Metrics for Buckets]. + + === AWS Permissions +Some specific AWS permissions are required for IAM user to collect AWS s3_daily_storage metrics. +---- +ec2:DescribeRegions +cloudwatch:GetMetricData +---- diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml new file mode 100644 index 00000000000..4bf207ac26d --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml @@ -0,0 +1,22 @@ +name: s3_daily_storage + type: group + description: > + `s3_daily_storage` contains the daily storage metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. + release: beta + fields: + - name: bucket.name + type: keyword + description: > + Name of a S3 bucket. + - name: bucket.size.bytes + type: scaled_float + description: > + The amount of data in bytes stored in a bucket. + - name: bucket.storage.type + type: keyword + description: > + Storage type specified for each S3 bucket. + - name: object.count + type: long + description: > + The total number of objects stored in a bucket for all storage classes. diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/data.go b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go new file mode 100644 index 00000000000..cd0474cfe1c --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go @@ -0,0 +1,37 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package s3_daily_storage + +import ( + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstrstr" +) + +var ( + schemaMetricSetFields = s.Schema{ + "bucket": s.Object{ + "name": c.Str("bucket.name", s.Optional), + "storage_type": c.Str("bucket.storage_type", s.Optional), + "size": s.Object{ + "bytes": c.Float("BucketSizeBytes", s.Optional), + }, + }, + "object": s.Object{ + "count": c.Int("NumberOfObjects", s.Optional), + }, + } +) + +var ( + schemaRootFields = s.Schema{ + "service": s.Object{ + "name": c.Str("service.name", s.Optional), + }, + "cloud": s.Object{ + "provider": c.Str("cloud.provider", s.Optional), + "region": c.Str("cloud.region", s.Optional), + }, + } +) diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go new file mode 100644 index 00000000000..339c5e0e110 --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -0,0 +1,142 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package s3_daily_storage + +import ( + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/aws" +) + +var metricsetName = "s3_daily_storage" + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(aws.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *aws.MetricSet + logger *logp.Logger +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The aws s3_daily_storage metricset is beta.") + s3Logger := logp.NewLogger(aws.ModuleName) + + moduleConfig := aws.Config{} + if err := base.Module().UnpackConfig(&moduleConfig); err != nil { + return nil, err + } + + if moduleConfig.Period == "" { + err := errors.New("period is not set in AWS module config") + s3Logger.Error(err) + } + + metricSet, err := aws.NewMetricSet(base) + if err != nil { + return nil, errors.Wrap(err, "error creating aws metricset") + } + + // Check if period is set to be multiple of 86400s + remainder := metricSet.PeriodInSec % 86400 + if remainder != 0 { + err := errors.New("period needs to be set to 86400s (or a multiple of 86400s). " + + "To avoid data missing or extra costs, please make sure period is set correctly " + + "in config.yml") + s3Logger.Info(err) + } + + return &MetricSet{ + MetricSet: metricSet, + logger: s3Logger, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) { + namespace := "AWS/S3" + // Get startTime and endTime + startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) + if err != nil { + logp.Error(errors.Wrap(err, "Error ParseDuration")) + m.logger.Error(err.Error()) + report.Error(err) + return + } + + // GetMetricData for AWS S3 from Cloudwatch + for _, regionName := range m.MetricSet.RegionsList { + m.MetricSet.AwsConfig.Region = regionName + svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) + listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) + if err != nil { + m.logger.Error(err.Error()) + report.Error(err) + continue + } + + if listMetricsOutput == nil || len(listMetricsOutput) == 0 { + continue + } + + dailyStorageMetricNames := []string{"NumberOfObjects", "BucketSizeBytes"} + metricDataQueries := constructMetricQueries(listMetricsOutput, int64(m.PeriodInSec), dailyStorageMetricNames, nil) + + // Use metricDataQueries to make GetMetricData API calls + metricDataOutput, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) + if err != nil { + err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName+" for instance "+instanceID) + m.logger.Error(err.Error()) + report.Error(err) + continue + } + + // Create Cloudwatch Events for S3 + event, info, err := createS3Events(metricDataOutput, metricsetName, regionName, schemaRootFields, schemaMetricSetFields) + if info != "" { + m.logger.Info(info) + } + + if err != nil { + m.logger.Error(err.Error()) + event.Error = err + report.Event(event) + continue + } + report.Event(event) + } +} + +func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, instanceID string, periodInSec int) []cloudwatch.MetricDataQuery { + metricDataQueries := []cloudwatch.MetricDataQuery{} + metricDataQueryEmpty := cloudwatch.MetricDataQuery{} + for i, listMetric := range listMetricsOutput { + metricDataQuery := createMetricDataQuery(listMetric, instanceID, i, periodInSec) + if metricDataQuery == metricDataQueryEmpty { + continue + } + metricDataQueries = append(metricDataQueries, metricDataQuery) + } + return metricDataQueries +} diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go new file mode 100644 index 00000000000..4c69456492a --- /dev/null +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go @@ -0,0 +1,73 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package s3_daily_storage + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/aws" +) + +func TestFetch(t *testing.T) { + accessKeyID, okAccessKeyID := os.LookupEnv("AWS_ACCESS_KEY_ID") + secretAccessKey, okSecretAccessKey := os.LookupEnv("AWS_SECRET_ACCESS_KEY") + sessionToken, okSessionToken := os.LookupEnv("AWS_SESSION_TOKEN") + defaultRegion, _ := os.LookupEnv("AWS_REGION") + + if !okAccessKeyID || accessKeyID == "" { + t.Skip("Skipping TestFetch; $AWS_ACCESS_KEY_ID not set or set to empty") + } else if !okSecretAccessKey || secretAccessKey == "" { + t.Skip("Skipping TestFetch; $AWS_SECRET_ACCESS_KEY not set or set to empty") + } else { + tempCreds := map[string]interface{}{ + "module": "aws", + "period": "86400s", + "metricsets": []string{"s3_daily_storage"}, + "access_key_id": accessKeyID, + "secret_access_key": secretAccessKey, + "default_region": defaultRegion, + } + + if okSessionToken && sessionToken != "" { + tempCreds["session_token"] = sessionToken + } + + s3MetricSet := mbtest.NewReportingMetricSetV2(t, tempCreds) + events, err := mbtest.ReportingFetchV2(s3MetricSet) + if err != nil { + t.Skip("Skipping TestFetch: failed to make api calls. Please check $AWS_ACCESS_KEY_ID, " + + "$AWS_SECRET_ACCESS_KEY and $AWS_SESSION_TOKEN in config.yml") + } + + assert.Empty(t, err) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + + t.Logf("Module: %s Metricset: %s", s3MetricSet.Module().Name(), s3MetricSet.Name()) + for _, event := range events { + // RootField + aws.CheckEventField("service.name", "string", event, t) + aws.CheckEventField("cloud.provider", "string", event, t) + aws.CheckEventField("cloud.region", "string", event, t) + // MetricSetField + aws.CheckEventField("bucket.name", "string", event, t) + aws.CheckEventField("bucket.storage.type", "string", event, t) + aws.CheckEventField("bucket.size.bytes", "float", event, t) + aws.CheckEventField("object.count", "int", event, t) + } + + errs := mbtest.WriteEventsReporterV2(s3MetricSet, t, "/") + if errs != nil { + t.Fatal("write", err) + } + } +} From 155c732298082bc45c3504205d96de16adb62412 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 22 Feb 2019 00:08:17 -0600 Subject: [PATCH 02/14] Add s3_daily_storage metricset --- x-pack/metricbeat/include/list.go | 2 + .../aws/s3_daily_storage/s3_daily_storage.go | 92 ++++++++++++++++--- 2 files changed, 82 insertions(+), 12 deletions(-) diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index c20490fb53b..08724e4bb87 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -11,7 +11,9 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_request" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/sqs" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_daily_storage" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/transaction_log" diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 339c5e0e110..daf7a909a05 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -5,9 +5,15 @@ package s3_daily_storage import ( + "fmt" + "strconv" + "strings" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" @@ -100,24 +106,18 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) { continue } - dailyStorageMetricNames := []string{"NumberOfObjects", "BucketSizeBytes"} - metricDataQueries := constructMetricQueries(listMetricsOutput, int64(m.PeriodInSec), dailyStorageMetricNames, nil) - + metricDataQueries := constructMetricQueries(listMetricsOutput, m.PeriodInSec) // Use metricDataQueries to make GetMetricData API calls metricDataOutput, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) if err != nil { - err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName+" for instance "+instanceID) + err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName) m.logger.Error(err.Error()) report.Error(err) continue } - // Create Cloudwatch Events for S3 - event, info, err := createS3Events(metricDataOutput, metricsetName, regionName, schemaRootFields, schemaMetricSetFields) - if info != "" { - m.logger.Info(info) - } - + // Create Cloudwatch Events for s3_daily_storage + event, err := createCloudWatchEvents(metricDataOutput, regionName) if err != nil { m.logger.Error(err.Error()) event.Error = err @@ -128,11 +128,16 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) { } } -func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, instanceID string, periodInSec int) []cloudwatch.MetricDataQuery { +func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, periodInSec int) []cloudwatch.MetricDataQuery { metricDataQueries := []cloudwatch.MetricDataQuery{} metricDataQueryEmpty := cloudwatch.MetricDataQuery{} + metricNames := []string{"NumberOfObjects", "BucketSizeBytes"} for i, listMetric := range listMetricsOutput { - metricDataQuery := createMetricDataQuery(listMetric, instanceID, i, periodInSec) + if !aws.StringInSlice(*listMetric.MetricName, metricNames) { + continue + } + + metricDataQuery := createMetricDataQuery(listMetric, periodInSec, i) if metricDataQuery == metricDataQueryEmpty { continue } @@ -140,3 +145,66 @@ func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, instanceID st } return metricDataQueries } + +func createMetricDataQuery(metric cloudwatch.Metric, periodInSec int, index int) (metricDataQuery cloudwatch.MetricDataQuery) { + statistic := "Average" + period := int64(periodInSec) + id := "s3d" + strconv.Itoa(index) + metricDims := metric.Dimensions + bucketName := "" + storageType := "" + for _, dim := range metricDims { + if *dim.Name == "BucketName" { + bucketName = *dim.Value + } else if *dim.Name == "StorageType" { + storageType = *dim.Value + } + } + metricName := *metric.MetricName + label := bucketName + " " + storageType + " " + metricName + + metricDataQuery = cloudwatch.MetricDataQuery{ + Id: &id, + MetricStat: &cloudwatch.MetricStat{ + Period: &period, + Stat: &statistic, + Metric: &metric, + }, + Label: &label, + } + return +} + +func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, regionName string) (event mb.Event, err error) { + event.Service = metricsetName + event.RootFields = common.MapStr{} + mapOfRootFieldsResults := make(map[string]interface{}) + mapOfRootFieldsResults["service.name"] = metricsetName + mapOfRootFieldsResults["cloud.region"] = regionName + + resultRootFields, err := aws.EventMapping(mapOfRootFieldsResults, schemaRootFields) + if err != nil { + err = errors.Wrap(err, "Error trying to apply schema schemaRootFields in AWS s3_daily_storage metricbeat module.") + return + } + event.RootFields = resultRootFields + + mapOfMetricSetFieldResults := make(map[string]interface{}) + for _, output := range getMetricDataResults { + if len(output.Values) == 0 { + continue + } + labels := strings.Split(*output.Label, " ") + mapOfMetricSetFieldResults["bucket.name"] = labels[0] + mapOfMetricSetFieldResults["bucket.storage.type"] = labels[1] + mapOfMetricSetFieldResults[labels[2]] = fmt.Sprint(output.Values[0]) + } + + resultMetricSetFields, err := aws.EventMapping(mapOfMetricSetFieldResults, schemaMetricSetFields) + if err != nil { + err = errors.Wrap(err, "Error trying to apply schema schemaMetricSetFields in AWS s3_daily_storage metricbeat module.") + return + } + event.MetricSetFields = resultMetricSetFields + return +} From 8f498ef89eb88a4f9986149a884feaccd7f1fbb7 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 25 Feb 2019 15:59:39 -0700 Subject: [PATCH 03/14] Add s3 daily storage metricset and TestData --- x-pack/metricbeat/module/aws/ec2/ec2_test.go | 6 -- .../aws/s3_daily_storage/_meta/data.json | 33 ++++++++ .../aws/s3_daily_storage/_meta/fields.yml | 6 +- .../module/aws/s3_daily_storage/data.go | 20 +---- .../aws/s3_daily_storage/s3_daily_storage.go | 70 +++++++++------- .../s3_daily_storage_integration_test.go | 83 ++++++++----------- 6 files changed, 111 insertions(+), 107 deletions(-) diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_test.go index caafbc01201..9e670c76d8d 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2_test.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2_test.go @@ -11,7 +11,6 @@ import ( awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" "github.com/stretchr/testify/assert" @@ -27,11 +26,6 @@ type MockEC2Client struct { ec2iface.EC2API } -// MockCloudWatchClient struct is used for unit tests. -type MockCloudWatchClient struct { - cloudwatchiface.CloudWatchAPI -} - var ( regionName = "us-west-1" instanceID = "i-123" diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json index e69de29bb2d..15c99b8e546 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json @@ -0,0 +1,33 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "aws": { + "s3_daily_storage": { + "bucket": { + "name": "test-s3-ks", + "size": { + "bytes": 969165 + } + }, + "number_of_object": 2 + } + }, + "cloud": { + "region": "ap-southeast-1" + }, + "event": { + "dataset": "aws.s3_daily_storage", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "s3_daily_storage" + }, + "service": { + "name": "s3_daily_storage", + "type": "s3_daily_storage" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml index 4bf207ac26d..6b73b645b2d 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml @@ -12,11 +12,7 @@ name: s3_daily_storage type: scaled_float description: > The amount of data in bytes stored in a bucket. - - name: bucket.storage.type - type: keyword - description: > - Storage type specified for each S3 bucket. - - name: object.count + - name: number_of_object type: long description: > The total number of objects stored in a bucket for all storage classes. diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/data.go b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go index cd0474cfe1c..6ef578c33dc 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/data.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go @@ -12,26 +12,10 @@ import ( var ( schemaMetricSetFields = s.Schema{ "bucket": s.Object{ - "name": c.Str("bucket.name", s.Optional), - "storage_type": c.Str("bucket.storage_type", s.Optional), "size": s.Object{ - "bytes": c.Float("BucketSizeBytes", s.Optional), + "bytes": c.Float("BucketSizeBytes"), }, }, - "object": s.Object{ - "count": c.Int("NumberOfObjects", s.Optional), - }, - } -) - -var ( - schemaRootFields = s.Schema{ - "service": s.Object{ - "name": c.Str("service.name", s.Optional), - }, - "cloud": s.Object{ - "provider": c.Str("cloud.provider", s.Optional), - "region": c.Str("cloud.region", s.Optional), - }, + "number_of_object": c.Int("NumberOfObjects"), } ) diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index daf7a909a05..20cf510aa28 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -95,20 +95,20 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) { for _, regionName := range m.MetricSet.RegionsList { m.MetricSet.AwsConfig.Region = regionName svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) - listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) + listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { m.logger.Error(err.Error()) report.Error(err) continue } - if listMetricsOutput == nil || len(listMetricsOutput) == 0 { + if listMetricsOutputs == nil || len(listMetricsOutputs) == 0 { continue } - metricDataQueries := constructMetricQueries(listMetricsOutput, m.PeriodInSec) + metricDataQueries := constructMetricQueries(listMetricsOutputs, m.PeriodInSec) // Use metricDataQueries to make GetMetricData API calls - metricDataOutput, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) + metricDataOutputs, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) if err != nil { err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName) m.logger.Error(err.Error()) @@ -117,22 +117,39 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) { } // Create Cloudwatch Events for s3_daily_storage - event, err := createCloudWatchEvents(metricDataOutput, regionName) - if err != nil { - m.logger.Error(err.Error()) - event.Error = err + bucketNames := getBucketNames(listMetricsOutputs) + for _, bucketName := range bucketNames { + event, err := createCloudWatchEvents(metricDataOutputs, regionName, bucketName) + if err != nil { + m.logger.Error(err.Error()) + event.Error = err + report.Event(event) + continue + } report.Event(event) - continue } - report.Event(event) } } -func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, periodInSec int) []cloudwatch.MetricDataQuery { +func getBucketNames(listMetricsOutputs []cloudwatch.Metric) (bucketNames []string) { + for _, output := range listMetricsOutputs { + for _, dim := range output.Dimensions { + if *dim.Name == "BucketName" { + if aws.StringInSlice(*dim.Value, bucketNames) { + continue + } + bucketNames = append(bucketNames, *dim.Value) + } + } + } + return +} + +func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, periodInSec int) []cloudwatch.MetricDataQuery { metricDataQueries := []cloudwatch.MetricDataQuery{} metricDataQueryEmpty := cloudwatch.MetricDataQuery{} metricNames := []string{"NumberOfObjects", "BucketSizeBytes"} - for i, listMetric := range listMetricsOutput { + for i, listMetric := range listMetricsOutputs { if !aws.StringInSlice(*listMetric.MetricName, metricNames) { continue } @@ -175,29 +192,20 @@ func createMetricDataQuery(metric cloudwatch.Metric, periodInSec int, index int) return } -func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, regionName string) (event mb.Event, err error) { +func createCloudWatchEvents(outputs []cloudwatch.MetricDataResult, regionName string, bucketName string) (event mb.Event, err error) { event.Service = metricsetName event.RootFields = common.MapStr{} - mapOfRootFieldsResults := make(map[string]interface{}) - mapOfRootFieldsResults["service.name"] = metricsetName - mapOfRootFieldsResults["cloud.region"] = regionName - - resultRootFields, err := aws.EventMapping(mapOfRootFieldsResults, schemaRootFields) - if err != nil { - err = errors.Wrap(err, "Error trying to apply schema schemaRootFields in AWS s3_daily_storage metricbeat module.") - return - } - event.RootFields = resultRootFields + // Cloud fields in ECS + event.RootFields.Put("service.name", metricsetName) + event.RootFields.Put("cloud.region", regionName) + // AWS s3_daily_storage metrics mapOfMetricSetFieldResults := make(map[string]interface{}) - for _, output := range getMetricDataResults { - if len(output.Values) == 0 { - continue - } + for _, output := range outputs { labels := strings.Split(*output.Label, " ") - mapOfMetricSetFieldResults["bucket.name"] = labels[0] - mapOfMetricSetFieldResults["bucket.storage.type"] = labels[1] - mapOfMetricSetFieldResults[labels[2]] = fmt.Sprint(output.Values[0]) + if labels[0] == bucketName { + mapOfMetricSetFieldResults[labels[2]] = fmt.Sprint(output.Values[0]) + } } resultMetricSetFields, err := aws.EventMapping(mapOfMetricSetFieldResults, schemaMetricSetFields) @@ -205,6 +213,8 @@ func createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, err = errors.Wrap(err, "Error trying to apply schema schemaMetricSetFields in AWS s3_daily_storage metricbeat module.") return } + + resultMetricSetFields.Put("bucket.name", bucketName) event.MetricSetFields = resultMetricSetFields return } diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go index 4c69456492a..56408a14418 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage_integration_test.go @@ -7,67 +7,54 @@ package s3_daily_storage import ( - "os" "testing" "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/x-pack/metricbeat/module/aws" + "github.com/elastic/beats/x-pack/metricbeat/module/aws/mtest" ) func TestFetch(t *testing.T) { - accessKeyID, okAccessKeyID := os.LookupEnv("AWS_ACCESS_KEY_ID") - secretAccessKey, okSecretAccessKey := os.LookupEnv("AWS_SECRET_ACCESS_KEY") - sessionToken, okSessionToken := os.LookupEnv("AWS_SESSION_TOKEN") - defaultRegion, _ := os.LookupEnv("AWS_REGION") + config, info := mtest.GetConfigForTest("s3_daily_storage", "86400s") + if info != "" { + t.Skip("Skipping TestFetch: " + info) + } - if !okAccessKeyID || accessKeyID == "" { - t.Skip("Skipping TestFetch; $AWS_ACCESS_KEY_ID not set or set to empty") - } else if !okSecretAccessKey || secretAccessKey == "" { - t.Skip("Skipping TestFetch; $AWS_SECRET_ACCESS_KEY not set or set to empty") - } else { - tempCreds := map[string]interface{}{ - "module": "aws", - "period": "86400s", - "metricsets": []string{"s3_daily_storage"}, - "access_key_id": accessKeyID, - "secret_access_key": secretAccessKey, - "default_region": defaultRegion, - } + s3DailyMetricSet := mbtest.NewReportingMetricSetV2(t, config) + events, err := mbtest.ReportingFetchV2(s3DailyMetricSet) + if err != nil { + t.Skip("Skipping TestFetch: failed to make api calls. Please check $AWS_ACCESS_KEY_ID, " + + "$AWS_SECRET_ACCESS_KEY and $AWS_SESSION_TOKEN in config.yml") + } - if okSessionToken && sessionToken != "" { - tempCreds["session_token"] = sessionToken - } + assert.Empty(t, err) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + t.Logf("Module: %s Metricset: %s", s3DailyMetricSet.Module().Name(), s3DailyMetricSet.Name()) - s3MetricSet := mbtest.NewReportingMetricSetV2(t, tempCreds) - events, err := mbtest.ReportingFetchV2(s3MetricSet) - if err != nil { - t.Skip("Skipping TestFetch: failed to make api calls. Please check $AWS_ACCESS_KEY_ID, " + - "$AWS_SECRET_ACCESS_KEY and $AWS_SESSION_TOKEN in config.yml") - } + for _, event := range events { + // RootField + mtest.CheckEventField("service.name", "string", event, t) + mtest.CheckEventField("cloud.region", "string", event, t) - assert.Empty(t, err) - if !assert.NotEmpty(t, events) { - t.FailNow() - } + // MetricSetField + mtest.CheckEventField("bucket.name", "string", event, t) + mtest.CheckEventField("bucket.size.bytes", "float", event, t) + mtest.CheckEventField("number_of_object", "int", event, t) + } +} - t.Logf("Module: %s Metricset: %s", s3MetricSet.Module().Name(), s3MetricSet.Name()) - for _, event := range events { - // RootField - aws.CheckEventField("service.name", "string", event, t) - aws.CheckEventField("cloud.provider", "string", event, t) - aws.CheckEventField("cloud.region", "string", event, t) - // MetricSetField - aws.CheckEventField("bucket.name", "string", event, t) - aws.CheckEventField("bucket.storage.type", "string", event, t) - aws.CheckEventField("bucket.size.bytes", "float", event, t) - aws.CheckEventField("object.count", "int", event, t) - } +func TestData(t *testing.T) { + config, info := mtest.GetConfigForTest("s3_daily_storage", "86400s") + if info != "" { + t.Skip("Skipping TestData: " + info) + } - errs := mbtest.WriteEventsReporterV2(s3MetricSet, t, "/") - if errs != nil { - t.Fatal("write", err) - } + ec2MetricSet := mbtest.NewReportingMetricSetV2(t, config) + errs := mbtest.WriteEventsReporterV2(ec2MetricSet, t, "/") + if errs != nil { + t.Fatal("write", errs) } } From df0711d89c8ac396467164c89f04f802b14974c3 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 25 Feb 2019 16:13:34 -0700 Subject: [PATCH 04/14] Add comment for StringInSlice --- x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go index 65497b7fc18..c3d758c82d5 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build !integration +// +build integration package sqs From 34f976a6ae9f89a94ec880ba4e1feb2515e62a7e Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 25 Feb 2019 17:42:24 -0700 Subject: [PATCH 05/14] Run mage fmt update --- x-pack/metricbeat/include/list.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 08724e4bb87..17caf332007 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -11,9 +11,8 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_request" - _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" - _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/sqs" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_daily_storage" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/sqs" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/transaction_log" From 2f40c0865701ab2a400c63c55c39d90273cf8730 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 25 Feb 2019 18:50:00 -0700 Subject: [PATCH 06/14] fix fields.yml formatting --- metricbeat/docs/fields.asciidoc | 20 ++++++++++------ metricbeat/docs/modules/aws.asciidoc | 14 ++++++++++- .../modules/aws/s3_daily_storage.asciidoc | 23 +++++++++++++++++++ metricbeat/docs/modules_list.asciidoc | 5 ++-- 4 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 metricbeat/docs/modules/aws/s3_daily_storage.asciidoc diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index a5715bba1a6..11303b50354 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1086,6 +1086,7 @@ The state of the instance (pending | running | shutting-down | terminated | stop -- [float] +<<<<<<< HEAD == s3_request fields `s3_request` contains request metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. @@ -1093,6 +1094,15 @@ The state of the instance (pending | running | shutting-down | terminated | stop *`aws.s3_request.bucket.name`*:: +======= +== s3_daily_storage fields + +`s3_daily_storage` contains the daily storage metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. + + + +*`aws.s3_daily_storage.bucket.name`*:: +>>>>>>> fix fields.yml formatting + -- type: keyword @@ -1102,6 +1112,7 @@ Name of a S3 bucket. -- +<<<<<<< HEAD *`aws.s3_request.requests.total`*:: + -- @@ -1173,10 +1184,6 @@ The number of Amazon S3 SELECT Object Content requests made for objects in an Am -- *`aws.s3_request.requests.select.scanned.bytes`*:: -+ --- -type: scaled_float - The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. @@ -1252,12 +1259,11 @@ The per-request time from the complete request being received by an Amazon S3 bu -- -*`aws.s3_request.latency.total_request.ms`*:: +*`aws.s3_daily_storage.number_of_object`*:: + -- type: long - -The elapsed per-request time from the first byte received to the last byte sent to an Amazon S3 bucket. +The total number of objects stored in a bucket for all storage classes. -- diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 542c4f03792..952ee65ffc6 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -9,7 +9,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for running EC2 instances. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2`, `sqs` and `s3_request`. +The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`. [float] === Module-specific configuration notes @@ -103,7 +103,11 @@ metricbeat.modules: - module: aws period: 86400s metricsets: +<<<<<<< HEAD - "s3_request" +======= + - "s3_daily_storage" +>>>>>>> fix fields.yml formatting access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -117,13 +121,21 @@ The following metricsets are available: * <> +<<<<<<< HEAD * <> +======= +* <> +>>>>>>> fix fields.yml formatting * <> include::aws/ec2.asciidoc[] +<<<<<<< HEAD include::aws/s3_request.asciidoc[] +======= +include::aws/s3_daily_storage.asciidoc[] +>>>>>>> fix fields.yml formatting include::aws/sqs.asciidoc[] diff --git a/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc b/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc new file mode 100644 index 00000000000..d2a730cd656 --- /dev/null +++ b/metricbeat/docs/modules/aws/s3_daily_storage.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-aws-s3_daily_storage]] +=== aws s3_daily_storage metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/s3_daily_storage/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 485cab8fb1f..7feaf8fd7fe 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -11,8 +11,9 @@ This file is generated! See scripts/docs_collector.py .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .3+| .3+| |<> -|<> beta[] -|<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .7+| .7+| |<> |<> From a07621f65ec7a24fff9448e2b17c0eb78e2239d4 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 25 Feb 2019 18:50:37 -0700 Subject: [PATCH 07/14] Add module config --- x-pack/metricbeat/metricbeat.reference.yml | 1 + x-pack/metricbeat/module/aws/_meta/config.yml | 1 + x-pack/metricbeat/module/aws/_meta/docs.asciidoc | 2 +- x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml | 2 +- x-pack/metricbeat/modules.d/aws.yml.disabled | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 828b34b47d9..37dff29caae 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -161,6 +161,7 @@ metricbeat.modules: period: 86400s metricsets: - "s3_request" + - "s3_daily_storage" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 3561b58b11f..18a720f84fd 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -11,6 +11,7 @@ period: 86400s metricsets: - "s3_request" + - "s3_daily_storage" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index 5b5ae11018f..c656b7949f4 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -2,7 +2,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for running EC2 instances. Note: extra AWS charges on GetMetricData API requests will be generated by this module. -The default metricsets are `ec2`, `sqs` and `s3_request`. +The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`. [float] === Module-specific configuration notes diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml index 6b73b645b2d..4f04e725935 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml @@ -1,4 +1,4 @@ -name: s3_daily_storage +- name: s3_daily_storage type: group description: > `s3_daily_storage` contains the daily storage metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index 3561b58b11f..18a720f84fd 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -11,6 +11,7 @@ period: 86400s metricsets: - "s3_request" + - "s3_daily_storage" access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' From a09b0dda40fbee4227b1d600c847cd6cd25442a1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 25 Feb 2019 22:14:21 -0700 Subject: [PATCH 08/14] Change number_of_object to number_of_objects --- x-pack/metricbeat/module/aws/fields.go | 2 +- x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json | 2 +- x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml | 2 +- x-pack/metricbeat/module/aws/s3_daily_storage/data.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index f65009a75f8..0b39fe5aa1d 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzEWk1vIzcSvftXFHJKgHFjk5nswYcFJh5jYyCbeGMPctSUyJKaazbZwyIly5gfvyj2hyy5Zcu2WtHBMJot1nuPVcUqUqdwS6szwCWfAEQTLZ3Bd7jk704ANLEKpo7GuzP41wkAwBdc8heovE6WQHlrSUWGj39dQ+WdiT4YN4eKYjCKYRZ8lcfOrU96iVGVxQlAIEvIdAZzPAGYGbKaz/Lsp+Cwog6NfOKqlheDT3X7ZADU5iQPJyL1U/9saLKdEzafL6R++gLKu4jGMcSSem6xxAhLCgSsAtakt9j+JWxhWRpVricY0IjJRZiu8hcvzn8qHtjf1Kn7bFN9SFfVqYg+oi1qFTfe6MizQkt6MrMet194Qgf53JQENQVFLuKcwM8ArfUKI2kBDspXdYoEyZnYyoOBQKUQyEW7AuMgMYF3WUfjOKJTVOwkogJpEyeJcU4jcHGpmlIQHudXn6ExxsB1ux4PMcLMh/xWisaae5Rpn8U9RSvfHRU5YXCkNwg0wrs19hIZUKmQSAMbeWIiLJHBYnKqJA0+AEcMkfRuUpxCbRNPjkiuNbnJrMQFwZTIrVcKHSRnTWXEE3vay5IcyNfOrz6f5xl+aTDDAm0iMAz3FPy+jHmiSgxz0uNSzpwGiUssOR+hRqNB+6UT6o/X/x2g023aiWViME6lIBqh1kZQoIWGyjB1R3Hpw21hXFGjuqXIozJubUAgRWYhzugkr3QwwLhIYYaKeDson4bvUzwq/pzGfYqHwm9cMV1FGhd8tjCK9MfCfijZteFb44tAqEfB/kurNLZlgmDtUxVHHwgW3qaKGHCBxuLUEkS/P/JlMJFGhC7zR3KC6eDYs+q+PjTwc1/VlmRTyLr7mkLeufn1SyA1DEqWVmZmSEs9ZLwWd4ym2meBxmSZLTyk+cq1eg1JjhgTF6okdTuZobE7Nkrr3fxl/P6k2ofIsp/HksImUqltamQmDVMfy83BBhNkTHlXlFFecaRqc8w0FalFjlAZl+L+JCfNfEfmOgaRzs7fQGV4xfYl06cY5YP8SW6485EtYU7hzY2CD8S5IXg+v/WjpsI5FWY4Jm5ptfRhe2wPYJefclAKDJlfuistwdwU9i/Bt+5LC1mDYSd4Fc5Lp430iGtP0BSzxz1shg0DOclFOzqQHmgdzAIjFdrxRIYOK2g7O3z6/Tob7uR9VFXsidLUw564/fgF0C6vFh+klA/EDMjslckd+NK06e/FWNPUGjWWoHnyR3ru6ZUttAOq2AnX4riQ5GIUXF71I9+LwD/A1Cenu43xpZLmECqU18NqvjoR5Xm3NXwH0t/Dj/88nZoIybGZu9wHZyPPII2llEY8qSlMJLEdAS98X5PTEvTfICTnmv+4TDEaNz/Nne03iBQq47Jnf5O6pa7b9+Rf0j8Uj076+P0k0NdEHN904Lee5sG5X/tk1LO/6/eDR39Tivse/k2TtKLFYaP49zZsEa7fdxYGrbcacXP+eKAqQrwoz/dgE/715uaqtwYV6lzAooOPFd57t8b5DgLNMWjbRfyq3hG5PfY5DdcOr0O+hfnfFzdbuKWK8NP/5SN0Kbofc3gGb51GxHv1+eB4NUmrMh7kTxe/XdxcHBp1SXiobmYA868XHz/t5c/P+YLnMZ3hj+ttb3gVSiZLO64m3opzjeT64reL8xv4Iy86nHsXJdEe2CsaJgUrdI7GOTUaOvHyM9AYEVq7TeG3N/W3MA0UU/g7qHaGj8HVmjGjqMeWKwixlWsj1UDnZqN9CqeUR9ajHn8VmiVY28sBs9+2uyylNBJigbj2jqX6UzZpkkp16vVqmFyqj0mts9asRVumAfbFnuB89/JMRyH4wMWHu7vx3OjD3R0oa8Tbs7n+4MRr2muNmkjC9irMz4BM7s3/AT7Aj08S+3lMYj/f3QFTWFA4IjGLkZxaFTMTOE7EOYpq2Ptex7GmcNo5VTQVNc1CE/fN2e3a50jag/5CprncfMQx+uZ2cyPC8g1uPvOdUp8xnyacK+uu3TksZ7JYc3N2vIN7VjuH4ppv22/nI788khuk56Kv7wC/8ttav698xN96XP/3+q0Nn7eaOE4qYsY5TXBOBZM64CpiXQd/Zypp49vffIgsjV1w3p02Bb2GFkN3aPs1UdrRa7VvSiuAq4PdEtxsJpPWCG/gWd+jt7bzpYDz8cFVSLPJYT6fMFVF2mAku2O/6qg4HycLw2Zqx2ltejY9AeNgZs283LEHdcCOAmpbvBgMLdCuI30/ZxA3Ghdo56svAdalpnGR9QXudAUKreUuE/7ZmP9PG12odv/yqEMsOWbkBde6ydX4lIJU1XE1afU75NayRrSlzsery049iRNtmuBuxAXsFRqEK7L1mZTN/Th36flU1NzTM3r+PwAA//+efQFO" + return "eJzEWU1v2zgQvedXDHpqgUbAtos95LBAm/aQwxbZpkWPCk2OLW4oUuUM7XXRH78YSvJX5MROrawORSDKM+89zgxn2HO4w+UFqAWdAbBlhxfwQi3oxRmAQdLRNmyDv4A/zwAAbtWCbqEOJjkEHZxDzQTvvt1AHbzlEK2fQY0crSaYxlDntUsXklko1lVxBhDRoSK8gJk6A5hadIYusvVz8KrGHo08vGzkwxhS070ZALVtZNMQ6jerd0PG9hpsn1vUb25BB8/KegKucMWNK8WwwIhAOqoGzQ7bb8IWFpXV1drAgEaEnmGyzD/8ePmm2PC/rVP/7FLdpKubVHBg5YpG89YXPXnSyqEppy6o3Q8e0EGeLxVCg1GjZzVDCFNQzgWtGI0ABx3qJjFC8pY7eVRE0ClG9OyWYD0kQgg+62g9sfIai71EdERjuUykZjgCF5/qCUbhcXn9FVpnBNR0+7GJEaYh5q8SW2d/KDH7KO6JcvLbUZGjih7NFoFWeL/GXikCpXVMaICsvLEMC0XgVPK6QgMhArGKjGY/KUqxcYnKZyTXudxmVqk5wgTRr3dKeUje2dpKJK5oLyr0ID+7vP56mS28bzHDXLmEYAl+YAyHMqZSVyrO0IxLOXMaJC655ANDo6wBExZeqN/f/9egvOnKDleJwHqdomikjLGCQjloqQxT98iLEO8K64tG6TtkGpVx5wMiarRzCUYvdaWHAdYzxqnSSLtJ+TD8kPhZ8ecyHhKfCr/1xWTJOC747GEU6Z8L+6lkN5bubCgiKjMK9ved0qprEwTrqlQRh4gwDy7VSKDmyjo1cQgcDke+iJZxROhin9ELppNjz6qH5tTAL0PdOJRDIeseGoz55Kanb4H0MEqqtLZTi0b6IRuMhCPb+pANGpNl9rBJ84l79RSSxIoTFbpCfVdOlXV7DkoX/Ow4fp+xCZFJznOuMG4jld6mUURoYBK42l5sMUHGlE9FWaUlMdbba7btSJ0ihtr6xIeTLFt7z8x1DCK9n/+ByvCOHUpmVWJ0iPJP8sOTjxwJM4y/PCiEiJQHgsfr22rV1mqGhR3OiTtcLkLcXTsA2NWHnJQCQ+zLdGUkmdvG/hh867m0kD0YDoIn4bzyxsqMuI4Eg5wjbnMYtgTopRbtmUBWQJto54qxMJ5KWTqtoJ11+PDpJjvu5b3XVRyI0jbDkbj7+ghoV9fz36WVj0gEiihomyfwhe3K39FY08RZPZag2fg9PQ+Myg7aCVXshetwfJTiYjVcXa9WXorAr2ASkjf9wXispDmFCh3MsJpPLkTZ7q6Gr0Hme/jtj/OJZUie7MznOTg7eQQpV9IaUdlgLKWwPQNeeNmgN5L0PyEm79u/qErM1s/O82T7ExhjbX2O7J/StzRN9538ieZVce+mj96WRlm3LKXL2b4xOv7ab9fYzh1gXoNubdQbwZu3gxeCE+RDrwQnSQbU4rS5/alLZgU3b3sPD3kn+2Oc4SQndS1nvsAxipX0Dd1sKO2uabvZhzC253sZpmWY/IN6z4XB8S2QYMuXsRsdROdhAFtuKaRd78NKO+mQaCDSv9OvBfd3esY77Zu/b341hIMzSFzWSKRmWEorRahPuEuqaWL419ZSrrq7bZGl9Qs++HOD7YzVYeib0+8J057ToPuyMOjU8mTT0JfthrRzQlt41veFne88/PjAGyOfBJuU/tx81TUa6SLc8mEqPnA5t2Qn7lRTwh42KwLWw9TZWbUnc3tgzwJqVzyOFufKtSlyeDBIGI0LtI/VY4D1d4DjIovIKV9YT5aglXPUd1ifW/d/ddml9P7/YekRS40ZecONaecp9ZCCWDe8LDv9Tnl0rBHtqPPu+qpXT/LE2Da5W3FBrRQaHv3RryvpqMeyGH9Ez/8CAAD//9FH0eM=" } diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json index 15c99b8e546..a1c9ec4263f 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json @@ -12,7 +12,7 @@ "bytes": 969165 } }, - "number_of_object": 2 + "number_of_objects": 2 } }, "cloud": { diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml index 4f04e725935..2678c1831bf 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/fields.yml @@ -12,7 +12,7 @@ type: scaled_float description: > The amount of data in bytes stored in a bucket. - - name: number_of_object + - name: number_of_objects type: long description: > The total number of objects stored in a bucket for all storage classes. diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/data.go b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go index 6ef578c33dc..8fc988e11a3 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/data.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/data.go @@ -16,6 +16,6 @@ var ( "bytes": c.Float("BucketSizeBytes"), }, }, - "number_of_object": c.Int("NumberOfObjects"), + "number_of_objects": c.Int("NumberOfObjects"), } ) From f7d6d7375d70b58fbc7e26a13a2b8d7bfedf0f2d Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 25 Feb 2019 22:31:07 -0700 Subject: [PATCH 09/14] Rerun make update --- metricbeat/docs/fields.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 11303b50354..31556095a10 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1259,7 +1259,7 @@ The per-request time from the complete request being received by an Amazon S3 bu -- -*`aws.s3_daily_storage.number_of_object`*:: +*`aws.s3_daily_storage.number_of_objects`*:: + -- type: long From 956fae2572713e7dafceb2855735f3fb5e79b79c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 26 Feb 2019 15:15:33 -0700 Subject: [PATCH 10/14] Add changelog and address comments --- CHANGELOG.next.asciidoc | 1 + .../module/aws/s3_daily_storage/s3_daily_storage.go | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 500d7c8c54d..612d486d325 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -317,6 +317,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add filters and pie chart for AWS EC2 dashboard. {pull}10596[10596] - Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053] - Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055] +- Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 20cf510aa28..5176401c291 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -85,7 +85,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) { // Get startTime and endTime startTime, endTime, err := aws.GetStartTimeEndTime(m.DurationString) if err != nil { - logp.Error(errors.Wrap(err, "Error ParseDuration")) + err = errors.Wrap(err, "Error ParseDuration") m.logger.Error(err.Error()) report.Error(err) return @@ -97,6 +97,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) { svcCloudwatch := cloudwatch.New(*m.MetricSet.AwsConfig) listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { + err = errors.Wrap(err, "GetListMetricsOutput failed, skipping region "+regionName) m.logger.Error(err.Error()) report.Error(err) continue @@ -111,7 +112,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) { metricDataOutputs, err := aws.GetMetricDataResults(metricDataQueries, svcCloudwatch, startTime, endTime) if err != nil { err = errors.Wrap(err, "GetMetricDataResults failed, skipping region "+regionName) - m.logger.Error(err.Error()) + m.logger.Error(err) report.Error(err) continue } @@ -121,7 +122,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) { for _, bucketName := range bucketNames { event, err := createCloudWatchEvents(metricDataOutputs, regionName, bucketName) if err != nil { - m.logger.Error(err.Error()) + err = errors.Wrap(err, "createCloudWatchEvents failed") + m.logger.Error(err) event.Error = err report.Event(event) continue @@ -203,7 +205,7 @@ func createCloudWatchEvents(outputs []cloudwatch.MetricDataResult, regionName st mapOfMetricSetFieldResults := make(map[string]interface{}) for _, output := range outputs { labels := strings.Split(*output.Label, " ") - if labels[0] == bucketName { + if len(labels) == 3 && labels[0] == bucketName && len(output.Values) >= 1 { mapOfMetricSetFieldResults[labels[2]] = fmt.Sprint(output.Values[0]) } } From f3ff3621e3bc438bfd8c63ea3fb1214ad15a1e67 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 28 Feb 2019 08:10:48 -0700 Subject: [PATCH 11/14] Add doc for s3_daily_storage --- metricbeat/docs/fields.asciidoc | 55 +++++++++++++++---- metricbeat/docs/modules/aws.asciidoc | 22 +++----- metricbeat/docs/modules_list.asciidoc | 8 +-- x-pack/metricbeat/include/list.go | 2 +- .../metricbeat/module/aws/_meta/docs.asciidoc | 7 ++- x-pack/metricbeat/module/aws/fields.go | 2 +- 6 files changed, 63 insertions(+), 33 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 31556095a10..5dfc609fadd 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1086,23 +1086,50 @@ The state of the instance (pending | running | shutting-down | terminated | stop -- [float] -<<<<<<< HEAD -== s3_request fields +== s3_daily_storage fields -`s3_request` contains request metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. +`s3_daily_storage` contains the daily storage metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. -*`aws.s3_request.bucket.name`*:: -======= -== s3_daily_storage fields +*`aws.s3_daily_storage.bucket.name`*:: ++ +-- +type: keyword -`s3_daily_storage` contains the daily storage metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. +Name of a S3 bucket. +-- -*`aws.s3_daily_storage.bucket.name`*:: ->>>>>>> fix fields.yml formatting +*`aws.s3_daily_storage.bucket.size.bytes`*:: ++ +-- +type: scaled_float + +The amount of data in bytes stored in a bucket. + + +-- + +*`aws.s3_daily_storage.number_of_objects`*:: ++ +-- +type: long + +The total number of objects stored in a bucket for all storage classes. + + +-- + +[float] +== s3_request fields + +`s3_request` contains request metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by AWS S3. + + + +*`aws.s3_request.bucket.name`*:: + -- type: keyword @@ -1112,7 +1139,6 @@ Name of a S3 bucket. -- -<<<<<<< HEAD *`aws.s3_request.requests.total`*:: + -- @@ -1184,6 +1210,10 @@ The number of Amazon S3 SELECT Object Content requests made for objects in an Am -- *`aws.s3_request.requests.select.scanned.bytes`*:: ++ +-- +type: scaled_float + The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. @@ -1259,11 +1289,12 @@ The per-request time from the complete request being received by an Amazon S3 bu -- -*`aws.s3_daily_storage.number_of_objects`*:: +*`aws.s3_request.latency.total_request.ms`*:: + -- type: long -The total number of objects stored in a bucket for all storage classes. + +The elapsed per-request time from the first byte received to the last byte sent to an Amazon S3 bucket. -- diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 952ee65ffc6..242b5cc8c62 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -72,8 +72,11 @@ image::./images/metricbeat-aws-ec2-overview.png[] Cloudwatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, the `period` for `sqs` metricset is recommended to be `300s` or multiples of `300s`. -=== s3_request metricset -Request metrics are available at 1-minute intervals with additional charges. The s3_request metricset will give more +=== s3_request and s3_daily_storage metricset +Daily storage metrics for S3 buckets are reported once per day with no additional cost. Since they are daily metrics, +`period` for `s3_daily_storage` metricset is recommended to be `86400s` or multiples of `86400s`. +Request metrics are available +at 1-minute intervals with additional charges. The s3_request metricset will give more granular data to track S3 bucket usage. The `period` for `s3_request` metricset can be set to `60s` or multiples of `60s`. But because of the extra charges for querying these metrics, the `period` is recommended to set to `86400s`. The user can always adjust this to the granularity they want. Request metrics are not enabled by default for S3 buckets. Please see @@ -103,11 +106,8 @@ metricbeat.modules: - module: aws period: 86400s metricsets: -<<<<<<< HEAD - "s3_request" -======= - "s3_daily_storage" ->>>>>>> fix fields.yml formatting access_key_id: '${AWS_ACCESS_KEY_ID:""}' secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' session_token: '${AWS_SESSION_TOKEN:""}' @@ -121,21 +121,17 @@ The following metricsets are available: * <> -<<<<<<< HEAD -* <> -======= * <> ->>>>>>> fix fields.yml formatting + +* <> * <> include::aws/ec2.asciidoc[] -<<<<<<< HEAD -include::aws/s3_request.asciidoc[] -======= include::aws/s3_daily_storage.asciidoc[] ->>>>>>> fix fields.yml formatting + +include::aws/s3_request.asciidoc[] include::aws/sqs.asciidoc[] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 7feaf8fd7fe..11108798182 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -10,10 +10,10 @@ This file is generated! See scripts/docs_collector.py |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.3+| .3+| |<> -|<> beta[] -|<> beta[] -|<> beta[] +.4+| .4+| |<> +|<> beta[] +|<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .7+| .7+| |<> |<> diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 17caf332007..761052d0df7 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -10,8 +10,8 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/metricbeat/module/aws" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/ec2" - _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_request" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_daily_storage" + _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/s3_request" _ "github.com/elastic/beats/x-pack/metricbeat/module/aws/sqs" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql" _ "github.com/elastic/beats/x-pack/metricbeat/module/mssql/performance" diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index c656b7949f4..8a864df0669 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -65,8 +65,11 @@ image::./images/metricbeat-aws-ec2-overview.png[] Cloudwatch metrics for Amazon SQS queues are automatically collected and pushed to CloudWatch every 5 minutes, the `period` for `sqs` metricset is recommended to be `300s` or multiples of `300s`. -=== s3_request metricset -Request metrics are available at 1-minute intervals with additional charges. The s3_request metricset will give more +=== s3_request and s3_daily_storage metricset +Daily storage metrics for S3 buckets are reported once per day with no additional cost. Since they are daily metrics, +`period` for `s3_daily_storage` metricset is recommended to be `86400s` or multiples of `86400s`. +Request metrics are available +at 1-minute intervals with additional charges. The s3_request metricset will give more granular data to track S3 bucket usage. The `period` for `s3_request` metricset can be set to `60s` or multiples of `60s`. But because of the extra charges for querying these metrics, the `period` is recommended to set to `86400s`. The user can always adjust this to the granularity they want. Request metrics are not enabled by default for S3 buckets. Please see diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 0b39fe5aa1d..2a571a19289 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzEWU1v2zgQvedXDHpqgUbAtos95LBAm/aQwxbZpkWPCk2OLW4oUuUM7XXRH78YSvJX5MROrawORSDKM+89zgxn2HO4w+UFqAWdAbBlhxfwQi3oxRmAQdLRNmyDv4A/zwAAbtWCbqEOJjkEHZxDzQTvvt1AHbzlEK2fQY0crSaYxlDntUsXklko1lVxBhDRoSK8gJk6A5hadIYusvVz8KrGHo08vGzkwxhS070ZALVtZNMQ6jerd0PG9hpsn1vUb25BB8/KegKucMWNK8WwwIhAOqoGzQ7bb8IWFpXV1drAgEaEnmGyzD/8ePmm2PC/rVP/7FLdpKubVHBg5YpG89YXPXnSyqEppy6o3Q8e0EGeLxVCg1GjZzVDCFNQzgWtGI0ABx3qJjFC8pY7eVRE0ClG9OyWYD0kQgg+62g9sfIai71EdERjuUykZjgCF5/qCUbhcXn9FVpnBNR0+7GJEaYh5q8SW2d/KDH7KO6JcvLbUZGjih7NFoFWeL/GXikCpXVMaICsvLEMC0XgVPK6QgMhArGKjGY/KUqxcYnKZyTXudxmVqk5wgTRr3dKeUje2dpKJK5oLyr0ID+7vP56mS28bzHDXLmEYAl+YAyHMqZSVyrO0IxLOXMaJC655ANDo6wBExZeqN/f/9egvOnKDleJwHqdomikjLGCQjloqQxT98iLEO8K64tG6TtkGpVx5wMiarRzCUYvdaWHAdYzxqnSSLtJ+TD8kPhZ8ecyHhKfCr/1xWTJOC747GEU6Z8L+6lkN5bubCgiKjMK9ved0qprEwTrqlQRh4gwDy7VSKDmyjo1cQgcDke+iJZxROhin9ELppNjz6qH5tTAL0PdOJRDIeseGoz55Kanb4H0MEqqtLZTi0b6IRuMhCPb+pANGpNl9rBJ84l79RSSxIoTFbpCfVdOlXV7DkoX/Ow4fp+xCZFJznOuMG4jld6mUURoYBK42l5sMUHGlE9FWaUlMdbba7btSJ0ihtr6xIeTLFt7z8x1DCK9n/+ByvCOHUpmVWJ0iPJP8sOTjxwJM4y/PCiEiJQHgsfr22rV1mqGhR3OiTtcLkLcXTsA2NWHnJQCQ+zLdGUkmdvG/hh867m0kD0YDoIn4bzyxsqMuI4Eg5wjbnMYtgTopRbtmUBWQJto54qxMJ5KWTqtoJ11+PDpJjvu5b3XVRyI0jbDkbj7+ghoV9fz36WVj0gEiihomyfwhe3K39FY08RZPZag2fg9PQ+Myg7aCVXshetwfJTiYjVcXa9WXorAr2ASkjf9wXispDmFCh3MsJpPLkTZ7q6Gr0Hme/jtj/OJZUie7MznOTg7eQQpV9IaUdlgLKWwPQNeeNmgN5L0PyEm79u/qErM1s/O82T7ExhjbX2O7J/StzRN9538ieZVce+mj96WRlm3LKXL2b4xOv7ab9fYzh1gXoNubdQbwZu3gxeCE+RDrwQnSQbU4rS5/alLZgU3b3sPD3kn+2Oc4SQndS1nvsAxipX0Dd1sKO2uabvZhzC253sZpmWY/IN6z4XB8S2QYMuXsRsdROdhAFtuKaRd78NKO+mQaCDSv9OvBfd3esY77Zu/b341hIMzSFzWSKRmWEorRahPuEuqaWL419ZSrrq7bZGl9Qs++HOD7YzVYeib0+8J057ToPuyMOjU8mTT0JfthrRzQlt41veFne88/PjAGyOfBJuU/tx81TUa6SLc8mEqPnA5t2Qn7lRTwh42KwLWw9TZWbUnc3tgzwJqVzyOFufKtSlyeDBIGI0LtI/VY4D1d4DjIovIKV9YT5aglXPUd1ifW/d/ddml9P7/YekRS40ZecONaecp9ZCCWDe8LDv9Tnl0rBHtqPPu+qpXT/LE2Da5W3FBrRQaHv3RryvpqMeyGH9Ez/8CAAD//9FH0eM=" + return "eJzsWs1uIzcSvvspCjklwLixmZnswYcFJh5jYyCbeGMPctSUyJKaazbZwyIlazAPvyj2jyy5Zcu2WrlEB8MQW6zvK9Y/+xRuaXUGuOQTgGiipTP4Dpf83QmAJlbB1NF4dwb/OgEA+IxL/gyV18kSKG8tqcjw4c9rqLwz0Qfj5lBRDEYxzIKv8tq59UkvMaqyOAEIZAmZzmCOJwAzQ1bzWd79FBxW1KGRT1zV8mDwqW6/GQC1ucn9jUi97b8b2mznhs3nM6m3n0F5F9E4hlhSzy2WGGFJgYBVwJr0Fts/hS0sS6PK9QYDOmJyEaar/MOL87fFPfmbeuo+21Tv01V1KqKPaItaxY0nOvKs0JKezKzH7Qce0YN8bkqCmoIiF3FO4GeA1nqFkbQAB+WrOkWC5Exs1YOBQKUQyEW7AuMgMYF3WY/GcUSnqNhJRAXSJk4S45xG4OJSNaUgPM6vPkEjjIHr9jzuY4SZD/mpFI01X1G2fRL3FK38dlTkhMGR3iDQKN6tsZfIgEqFRBrYyDcmwhIZLCanStLgA3DEEEnvJsUp1Dbx5IjkWpGbzEpcEEyJ3Pqk0EFy1lRGLLGnvSzJgfzs/OrTed7h5wYzLNAmAsPwlYLflzFPVIlhTnpcypnTIHHxJecj1Gg0aL90Qv3h+b8BdLoNO7FMDMapFERHqLURFGihoTJM3VFc+nBbGFfUqG4p8qiMWxkQSJFZiDE6iSsdDDAuUpihIt52ysfh+xSPij+HcZ/iofAbV0xXkcYFnyWMovpjYT+U2rXhW+OLQKhHwf5zq2lsywTB2ocqjj4QLLxNFTHgAo3FqSWIfn/ky2AijQhd9o/kBNPBsWet+/rQwM99VVuSpJD17msKOXPzy49AahiUKK3MzJCWesh4LeYYTbXPAY3JMku4T/OFZ/USkhwxJi5USep2MkNjdyRK6938efz+oNqHyJLPY0lhE6nUNjUyk4apj+XmYoMJMqacFWWVVxyp2lwzTUVqkSNUxqW4P8lJs9+RuY5BpJPzF1AZPrF9yfQhRvkgf5Ib7nwkJcwpvLpR8IE4NwRPx7d+1VQ4p8IM+8QtrZY+bK/tAezyY3ZKgSH7S3elxZmbwv45+NZ9aSFnMGwEL8J56bSRHnFtCZpitrj7zbBhICexaEcH0gOtg1lgpEI7nsjSYRXa7g4ff7vOgjv1Pqgq9kRp6mFL3P76GdAurxbvpZQPxAzI7JXJHfjStOHv2VjT1Bo1lkLz5g/0uadVttAOqMVOcS2OCwkuRsHlVb/yvSj4B5j65HSXGJ+r0uxChfJ6WJsvDkR5320dvgHp7+HHf55OTYTk2Mxd7oOzkCeQxlJKI57UFCYS2I6AF76vyWlx+m8QknPNf1ymGI2bn+bO9htECpVx2bK/Sd1S1+1z8i/pH4oHkz5+N9Fo7GoiVc7mxOj5Y7/tzbZmgHkN2rVRJ4LX7wYHglOK+44Ep0ka1OKwvv1b68wI1+86CY9JZ/N1nOYkO3UlOV/gaIwodUPbG0q5q5tq9jGMTX6f+NnET/9HasfA4PklkGDLw9h7FUQrYQBbLimkXO/MSlmpkHjQ0gN9ScTxtTbebnPPuttv/rbpR+yl1RE3k/YRjeWXm5urXhpUqHOrhg4+VPjVuzXONxBojkHbLret6h05qsc+p+Eq+WXItzD/++JmC7cYd2f7YvQPOTyBt04j4r36dHC8mqQpHw/yx4tfL24uDo26JDxU3z6A+ZeLDx/3suenbMHzmMbw+/W2NbwIJZOlHZdwr8W5RnJ98evF+Q38ng8dzr2LEmgPbBUNk4IVOkfjzEeHZrtdQm/lNi3O3tRfwzRQTOGvoNoJPgZXa8b0oh5briBEVq6cVQOdm0T7GE5pBKxHPf4pNEewlpcdZr+0uyylNBJigbj2jqXPUTZpkp5s6vVqmFyqj0mtk9acRVumAfbFnuB88/xIRyH4wMX7u7vxzOj93R0oa8Tas7h+ROg17XVGjSdhe+nrZ0AmT6H+AT7Aj48S+2lMYj/d3QFTWFA4IjGLkZxaFTMTOE7EOIrqkN1OTeG0M6poKmqahcbvm1uKtc2RtAf91WNzjf+AY/TNPf6Gh+V3FfLtxpT6iPk44VxZd+3OYTmTxZqbW5Id3LO2syuu+baTpTzcziu5QXrK+/oO8Au/rvX7wkd8q+n6v9evbfi81cRxUhEzzmmCcyqY1AFPEes6+DtTYSRo324StTRywXl32hT0GloM3fXEl0RpR6/VPimtAK4Odh92sxlMWiG8gWf9xkgrO19/OR/vXfo1SQ7zJM5UFWmDkeyOfNVRcT5OFobN1I7T2vRsegLGwcyaebkjB3XAjgJqW3kxGFqgXXv6fsYgZjQu0M5WnwOsC03jIusL3OkKFFrLXST8oxH/n9a7UO1+x65DLDFm5APXuonV+JgGqarjatLq75CpZY1oSzsfri477YmfaNM4d6NcwF5Dg3BFbX0kHXUwK5s/oc//BwAA///hXdWu" } From 862314c6da867e5d2440a1fc21c598ccd606246f Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 28 Feb 2019 09:31:09 -0700 Subject: [PATCH 12/14] Add timestamp check for s3_daily_storage --- .../module/aws/s3_daily_storage/_meta/data.json | 5 +++-- .../aws/s3_daily_storage/s3_daily_storage.go | 14 ++++++++++---- .../module/aws/s3_request/_meta/data.json | 3 ++- .../metricbeat/module/aws/s3_request/s3_request.go | 1 + 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json index a1c9ec4263f..0cdfffa3aab 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/_meta/data.json @@ -9,13 +9,14 @@ "bucket": { "name": "test-s3-ks", "size": { - "bytes": 969165 + "bytes": 1679631 } }, - "number_of_objects": 2 + "number_of_objects": 3 } }, "cloud": { + "provider": "aws", "region": "ap-southeast-1" }, "event": { diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 5176401c291..7110018bbf0 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -200,13 +200,19 @@ func createCloudWatchEvents(outputs []cloudwatch.MetricDataResult, regionName st // Cloud fields in ECS event.RootFields.Put("service.name", metricsetName) event.RootFields.Put("cloud.region", regionName) + event.RootFields.Put("cloud.provider", "aws") // AWS s3_daily_storage metrics mapOfMetricSetFieldResults := make(map[string]interface{}) - for _, output := range outputs { - labels := strings.Split(*output.Label, " ") - if len(labels) == 3 && labels[0] == bucketName && len(output.Values) >= 1 { - mapOfMetricSetFieldResults[labels[2]] = fmt.Sprint(output.Values[0]) + // Find a timestamp for all metrics in output + if len(outputs) > 0 && len(outputs[0].Timestamps) > 0 { + timestamp := outputs[0].Timestamps[0] + for _, output := range outputs { + labels := strings.Split(*output.Label, " ") + // check timestamp to make sure metrics come from the same timestamp + if len(labels) == 3 && labels[0] == bucketName && len(output.Values) > 0 && output.Timestamps[0] == timestamp { + mapOfMetricSetFieldResults[labels[2]] = fmt.Sprint(output.Values[0]) + } } } diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json index 4814b96c7db..22306231c82 100644 --- a/x-pack/metricbeat/module/aws/s3_request/_meta/data.json +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/data.json @@ -29,6 +29,7 @@ } }, "cloud": { + "provider": "aws", "region": "ap-southeast-1" }, "event": { @@ -43,4 +44,4 @@ "name": "s3_request", "type": "s3_request" } -} \ No newline at end of file +} diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index eb81ba06b34..67f2cd32425 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -201,6 +201,7 @@ func createS3RequestEvents(outputs []cloudwatch.MetricDataResult, regionName str // Cloud fields in ECS event.RootFields.Put("service.name", metricsetName) event.RootFields.Put("cloud.region", regionName) + event.RootFields.Put("cloud.provider", "aws") // AWS s3_request metrics mapOfMetricSetFieldResults := make(map[string]interface{}) From 1bcc2977f888669c94c91db73cf61ddd16410a0e Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 28 Feb 2019 17:39:26 -0700 Subject: [PATCH 13/14] Change select.scanned to select_scanned --- x-pack/metricbeat/module/aws/fields.go | 2 +- x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml | 4 ++-- x-pack/metricbeat/module/aws/s3_request/data.go | 4 ++-- .../module/aws/s3_request/s3_request_integration_test.go | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 2a571a19289..c5b0b471c5f 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzsWs1uIzcSvvspCjklwLixmZnswYcFJh5jYyCbeGMPctSUyJKaazbZwyIlazAPvyj2jyy5Zcu2WrlEB8MQW6zvK9Y/+xRuaXUGuOQTgGiipTP4Dpf83QmAJlbB1NF4dwb/OgEA+IxL/gyV18kSKG8tqcjw4c9rqLwz0Qfj5lBRDEYxzIKv8tq59UkvMaqyOAEIZAmZzmCOJwAzQ1bzWd79FBxW1KGRT1zV8mDwqW6/GQC1ucn9jUi97b8b2mznhs3nM6m3n0F5F9E4hlhSzy2WGGFJgYBVwJr0Fts/hS0sS6PK9QYDOmJyEaar/MOL87fFPfmbeuo+21Tv01V1KqKPaItaxY0nOvKs0JKezKzH7Qce0YN8bkqCmoIiF3FO4GeA1nqFkbQAB+WrOkWC5Exs1YOBQKUQyEW7AuMgMYF3WY/GcUSnqNhJRAXSJk4S45xG4OJSNaUgPM6vPkEjjIHr9jzuY4SZD/mpFI01X1G2fRL3FK38dlTkhMGR3iDQKN6tsZfIgEqFRBrYyDcmwhIZLCanStLgA3DEEEnvJsUp1Dbx5IjkWpGbzEpcEEyJ3Pqk0EFy1lRGLLGnvSzJgfzs/OrTed7h5wYzLNAmAsPwlYLflzFPVIlhTnpcypnTIHHxJecj1Gg0aL90Qv3h+b8BdLoNO7FMDMapFERHqLURFGihoTJM3VFc+nBbGFfUqG4p8qiMWxkQSJFZiDE6iSsdDDAuUpihIt52ysfh+xSPij+HcZ/iofAbV0xXkcYFnyWMovpjYT+U2rXhW+OLQKhHwf5zq2lsywTB2ocqjj4QLLxNFTHgAo3FqSWIfn/ky2AijQhd9o/kBNPBsWet+/rQwM99VVuSpJD17msKOXPzy49AahiUKK3MzJCWesh4LeYYTbXPAY3JMku4T/OFZ/USkhwxJi5USep2MkNjdyRK6938efz+oNqHyJLPY0lhE6nUNjUyk4apj+XmYoMJMqacFWWVVxyp2lwzTUVqkSNUxqW4P8lJs9+RuY5BpJPzF1AZPrF9yfQhRvkgf5Ib7nwkJcwpvLpR8IE4NwRPx7d+1VQ4p8IM+8QtrZY+bK/tAezyY3ZKgSH7S3elxZmbwv45+NZ9aSFnMGwEL8J56bSRHnFtCZpitrj7zbBhICexaEcH0gOtg1lgpEI7nsjSYRXa7g4ff7vOgjv1Pqgq9kRp6mFL3P76GdAurxbvpZQPxAzI7JXJHfjStOHv2VjT1Bo1lkLz5g/0uadVttAOqMVOcS2OCwkuRsHlVb/yvSj4B5j65HSXGJ+r0uxChfJ6WJsvDkR5320dvgHp7+HHf55OTYTk2Mxd7oOzkCeQxlJKI57UFCYS2I6AF76vyWlx+m8QknPNf1ymGI2bn+bO9htECpVx2bK/Sd1S1+1z8i/pH4oHkz5+N9Fo7GoiVc7mxOj5Y7/tzbZmgHkN2rVRJ4LX7wYHglOK+44Ep0ka1OKwvv1b68wI1+86CY9JZ/N1nOYkO3UlOV/gaIwodUPbG0q5q5tq9jGMTX6f+NnET/9HasfA4PklkGDLw9h7FUQrYQBbLimkXO/MSlmpkHjQ0gN9ScTxtTbebnPPuttv/rbpR+yl1RE3k/YRjeWXm5urXhpUqHOrhg4+VPjVuzXONxBojkHbLret6h05qsc+p+Eq+WXItzD/++JmC7cYd2f7YvQPOTyBt04j4r36dHC8mqQpHw/yx4tfL24uDo26JDxU3z6A+ZeLDx/3suenbMHzmMbw+/W2NbwIJZOlHZdwr8W5RnJ98evF+Q38ng8dzr2LEmgPbBUNk4IVOkfjzEeHZrtdQm/lNi3O3tRfwzRQTOGvoNoJPgZXa8b0oh5briBEVq6cVQOdm0T7GE5pBKxHPf4pNEewlpcdZr+0uyylNBJigbj2jqXPUTZpkp5s6vVqmFyqj0mtk9acRVumAfbFnuB88/xIRyH4wMX7u7vxzOj93R0oa8Tas7h+ROg17XVGjSdhe+nrZ0AmT6H+AT7Aj48S+2lMYj/d3QFTWFA4IjGLkZxaFTMTOE7EOIrqkN1OTeG0M6poKmqahcbvm1uKtc2RtAf91WNzjf+AY/TNPf6Gh+V3FfLtxpT6iPk44VxZd+3OYTmTxZqbW5Id3LO2syuu+baTpTzcziu5QXrK+/oO8Au/rvX7wkd8q+n6v9evbfi81cRxUhEzzmmCcyqY1AFPEes6+DtTYSRo324StTRywXl32hT0GloM3fXEl0RpR6/VPimtAK4Odh92sxlMWiG8gWf9xkgrO19/OR/vXfo1SQ7zJM5UFWmDkeyOfNVRcT5OFobN1I7T2vRsegLGwcyaebkjB3XAjgJqW3kxGFqgXXv6fsYgZjQu0M5WnwOsC03jIusL3OkKFFrLXST8oxH/n9a7UO1+x65DLDFm5APXuonV+JgGqarjatLq75CpZY1oSzsfri477YmfaNM4d6NcwF5Dg3BFbX0kHXUwK5s/oc//BwAA///hXdWu" + return "eJzsWs1uIzcSvvspCjklwLixmZnswYcFJh5jYyCbeGMPctSUyJKaazbZwyIlazAPvyj2jyy5Zcu2WrlEB8MQW6zvK9Y/+xRuaXUGuOQTgGiipTP4Dpf83QmAJlbB1NF4dwb/OgEA+IxL/gyV18kSKG8tqcjw4c9rqLwz0Qfj5lBRDEYxzIKv8tq59UkvMaqyOAEIZAmZzmCOJwAzQ1bzWd79FBxW1KGRT1zV8mDwqW6/GQC1ucn9jUi97b8b2mznhs3nM6m3n0F5F9E4hlhSzy2WGGFJgYBVwJr0Fts/hS0sS6PK9QYDOmJyEaar/MOL87fFPfmbeuo+21Tv01V1KqKPaItaxY0nOvKs0JKezKzH7Qce0YN8bkqCmoIiF3FO4GeA1nqFkbQAB+WrOkWC5Exs1YOBQKUQyEW7AuMgMYF3WY/GcUSnqNhJRAXSJk4S45xG4OJSNaUgPM6vPkEjjIHr9jzuY4SZD/mpFI01X1G2fRL3FK38dlTkhMGR3iDQKN6tsZfIgEqFRBrYyDcmwhIZLCanStLgA3DEEEnvJsUp1Dbx5IjkWpGbzEpcEEyJ3Pqk0EFy1lRGLLGnvSzJgfzs/OrTed7h5wYzLNAmAsPwlYLflzFPVIlhTnpcypnTIHHxJecj1Gg0aL90Qv3h+b8BdLoNO7FMDMapFERHqLURFGihoTJM3VFc+nBbGFfUqG4p8qiMWxkQSJFZiDE6iSsdDDAuUpihIt52ysfh+xSPij+HcZ/iofAbV0xXkcYFnyWMovpjYT+U2rXhW+OLQKhHwf5zq2lsywTB2ocqjj4QLLxNFTHgAo3FqSWIfn/ky2AijQhd9o/kBNPBsWet+/rQwM99VVuSpJD17msKOXPzy49AahiUKK3MzJCWesh4LeYYTbXPAY3JMku4T/OFZ/USkhwxJi5USep2MkNjdyRK6938efz+oNqHyJLPY0lhE6nUNjUyk4apj+XmYoMJMqacFWWVVxyp2lwzTUVqkSNUxqW4P8lJs9+RuY5BpJPzF1AZPrF9yfQhRvkgf5Ib7nwkJcwpvLpR8IE4NwRPx7d+1VQ4p8IM+8QtrZY+bK/tAezyY3ZKgSH7S3elxZmbwv45+NZ9aSFnMGwEL8J56bSRHnFtCZpitrj7zbBhICexaEcH0gOtg1lgpEI7nsjSYRXa7g4ff7vOgjv1Pqgq9kRp6mFL3P76GdAurxbvpZQPxAzI7JXJHfjStOHv2VjT1Bo1lkLz5g/0uadVttAOqMVOcS2OCwkuRsHlVb/yvSj4B5j65HSXGJ+r0uxChfJ6WJsvDkR5320dvgHp7+HHf55OTYTk2Mxd7oOzkCeQxlJKI57UFCYS2I6AF76vyWlx+m8QknPNf1ymGI2bn+bO9htECpVx2bK/Sd1S1+1z8i/pH4oHkz5+N9Fo7GoiVc7mxOj5Y7/tzbZmgHkN2rVRJ4LX7wYHglOK+44Ep0ka1OKwvv1b68wI1+86CY9JZ/N1nOYkO3UlOV/gaIwodUPbG0q5q5tq9jGMTX6f+NnET/9HasfA4PklkGDLw9h7FUQrYQBbLimkXO/MSlmpkHjQ0gN9ScTxtTbebnPPuttv/rbpR+yl1RE3k/YRjeWXm5urXhpUqHOrhg4+VPjVuzXONxBojkHbLret6h05qsc+p+Eq+WXItzD/++JmC7cYd2f7YvQPOTyBt04j4r36dHC8mqQpHw/yx4tfL24uDo26JDxU3z6A+ZeLDx/3suenbMHzmMbw+/W2NbwIJZOlHZdwr8W5RnJ98evF+Q38ng8dzr2LEmgPbBUNkwkrdI7GmY8OzXa7hN7KbVqcvam/hmmgmMJfQbUTfAyu1ozpRT22XEGIrFw5qwY6N4n2MZzSCFiPevxTaI5gLS87zH5pd1lKaSTEAnHtHUufo2zSJD3Z1OvVMLlUH5NaJ605i7ZMA+yLPcH55vmRjkLwgYv3d3fjmdH7uztQ1oi1Z3H9iNBr2uuMGk/C9tLXz4BMnkL9A3yAHx8l9tOYxH66uwOmsKBwRGIWIzm1KmYmcJyIcRTVIbudmsJpZ1TRVNQ0C43fN7cUa5sjaQ/6q8fmGv8Bx+ibe/wND8vvKuTbjSn1EfNxwrmy7tqdw3ImizU3tyQ7uGdtZ1dc820nS3m4nVdyg/SU9/Ud4Bd+Xev3hY/4VtP1f69f2/B5q4njpCJmnNME51QwqQOeItZ18HemwkjQvt0kamnkgvPutCnoNbQYuuuJL4nSjl6rfVJaAVwd7D7sZjOYtEJ4A8/6jZFWdr7+cj7eu/RrkhzmSZypKtIGI9kd+aqj4nycLAybqR2ntenZ9ASMg5k183JHDuqAHQXUtvJiMLRAu/b0/YxBzGhcoJ2tPgdYF5rGRdYXuNMVKLSWu0j4RyP+P613odr9jl2HWGLMyAeudROr8TENUlXH1aTV3yFTyxrRlnY+XF122hM/0aZx7ka5gL2GBuGK2vpIOupgVjZ/Qp//DwAA//8xd9YQ" } diff --git a/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml b/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml index a5469353f96..45ec464f8a3 100644 --- a/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml +++ b/x-pack/metricbeat/module/aws/s3_request/_meta/fields.yml @@ -36,11 +36,11 @@ type: long description: > The number of Amazon S3 SELECT Object Content requests made for objects in an Amazon S3 bucket. - - name: requests.select.scanned.bytes + - name: requests.select_scanned.bytes type: scaled_float description: > The number of bytes of data scanned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. - - name: requests.select.returned.bytes + - name: requests.select_returned.bytes type: scaled_float description: > The number of bytes of data returned with Amazon S3 SELECT Object Content requests in an Amazon S3 bucket. diff --git a/x-pack/metricbeat/module/aws/s3_request/data.go b/x-pack/metricbeat/module/aws/s3_request/data.go index 06e6f4cb290..0751ef522f1 100644 --- a/x-pack/metricbeat/module/aws/s3_request/data.go +++ b/x-pack/metricbeat/module/aws/s3_request/data.go @@ -19,8 +19,8 @@ var ( "head": c.Int("HeadRequests"), "post": c.Int("PostRequests"), "select": c.Int("SelectRequests"), - "select.scanned.bytes": c.Float("SelectScannedBytes"), - "select.returned.bytes": c.Float("SelectReturnedBytes"), + "select_scanned.bytes": c.Float("SelectScannedBytes"), + "select_returned.bytes": c.Float("SelectReturnedBytes"), "list": c.Int("ListRequests"), }, "downloaded": s.Object{ diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go index 8a3d369216e..e731e3fb4b0 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request_integration_test.go @@ -48,8 +48,8 @@ func TestFetch(t *testing.T) { mtest.CheckEventField("requests.head", "int", event, t) mtest.CheckEventField("requests.post", "int", event, t) mtest.CheckEventField("select.requests", "int", event, t) - mtest.CheckEventField("select.scanned.bytes", "float", event, t) - mtest.CheckEventField("select.returned.bytes", "float", event, t) + mtest.CheckEventField("select_scanned.bytes", "float", event, t) + mtest.CheckEventField("select_returned.bytes", "float", event, t) mtest.CheckEventField("requests.list", "int", event, t) mtest.CheckEventField("downloaded.bytes", "float", event, t) mtest.CheckEventField("uploaded.bytes", "float", event, t) From 675d5238676a917a7ff61928f94b57fe4e22e10b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 28 Feb 2019 17:40:28 -0700 Subject: [PATCH 14/14] run make update --- metricbeat/docs/fields.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5dfc609fadd..a72b37affdf 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1209,7 +1209,7 @@ The number of Amazon S3 SELECT Object Content requests made for objects in an Am -- -*`aws.s3_request.requests.select.scanned.bytes`*:: +*`aws.s3_request.requests.select_scanned.bytes`*:: + -- type: scaled_float @@ -1219,7 +1219,7 @@ The number of bytes of data scanned with Amazon S3 SELECT Object Content request -- -*`aws.s3_request.requests.select.returned.bytes`*:: +*`aws.s3_request.requests.select_returned.bytes`*:: + -- type: scaled_float