From 997870493a0ca9fce96e5f69c1bf8e2616edc07b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 8 May 2020 15:20:23 -0600 Subject: [PATCH 1/4] remove validate region/zone --- .../stackdriver/metadata_services.go | 6 +---- .../googlecloud/stackdriver/metricset.go | 23 +++++++------------ 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go index d628ee4b09d..bee0a2d834a 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go @@ -5,8 +5,6 @@ package stackdriver import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver/compute" ) @@ -17,9 +15,7 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error) switch c.ServiceName { case googlecloud.ServiceCompute: return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.opt...) - case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing, googlecloud.ServiceStorage: - return nil, nil default: - return nil, errors.Errorf("service '%s' not supported", c.ServiceName) + return nil, nil } } diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go index 81fa98751aa..23a787b3014 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go @@ -188,19 +188,6 @@ func validatePeriodForGCP(d time.Duration) (err error) { return nil } -// Validate googlecloud module config -func (c *config) Validate() error { - // storage metricset does not require region or zone config parameter. - if c.ServiceName == "storage" { - return nil - } - - if c.Region == "" && c.Zone == "" { - return errors.New("region and zone in Google Cloud config file cannot both be empty") - } - return nil -} - // Validate stackdriver related config func (mc *stackDriverConfig) Validate() error { gcpAlignerNames := make([]string, 0) @@ -234,10 +221,16 @@ func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, proj return metricsWithMeta, errors.Errorf("Could not make ListMetricDescriptors request: %s: %v", mt, err) } - metricsWithMeta[mt] = metricMeta{ + meta := metricMeta{ samplePeriod: time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second, - ingestDelay: time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second, + ingestDelay: 0 * time.Second, + } + + if out.Metadata.IngestDelay != nil { + meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second } + + metricsWithMeta[mt] = meta } } From 25778f841876dbbad47a53d8d836700ba5ca3f1b Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 8 May 2020 15:59:55 -0600 Subject: [PATCH 2/4] add stackdriver into googlecloud default config --- metricbeat/docs/modules/googlecloud.asciidoc | 16 ++++++++++++++++ x-pack/metricbeat/metricbeat.reference.yml | 16 ++++++++++++++++ .../module/googlecloud/_meta/config.yml | 16 ++++++++++++++++ .../googlecloud/stackdriver/metrics_requester.go | 3 +++ .../stackdriver/metrics_requester_test.go | 6 ++++++ .../modules.d/googlecloud.yml.disabled | 16 ++++++++++++++++ 6 files changed, 73 insertions(+) diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index 19d59a9fe8d..a8fd38f2afe 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -235,6 +235,22 @@ metricbeat.modules: credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 5m + +- module: googlecloud + metricsets: + - stackdriver + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" ---- [float] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 558f7e88f15..6a2582a6c49 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -523,6 +523,22 @@ metricbeat.modules: exclude_labels: false period: 5m +- module: googlecloud + metricsets: + - stackdriver + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" + #------------------------------- Graphite Module ------------------------------- - module: graphite metricsets: ["server"] diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index 35d252c699f..6a56af4e1bd 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -24,3 +24,19 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 5m + +- module: googlecloud + metricsets: + - stackdriver + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index 11b04e5a1b2..a0560769dbe 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -105,6 +105,9 @@ var serviceRegexp = regexp.MustCompile(`^(?P[a-z]+)\.googleapis.com.*`) // if they have a region specified. func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { f = fmt.Sprintf(`metric.type="%s"`, m) + if r.config.Zone == "" && r.config.Region == "" { + return + } service := serviceRegexp.ReplaceAllString(m, "${service}") diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go index f7aff666c0f..42a7fd01d0a 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -88,6 +88,12 @@ func TestGetFilterForMetric(t *testing.T) { stackdriverMetricsRequester{config: config{Zone: "us-west1-*"}, logger: logger}, "metric.type=\"compute.googleapis.com/instance/uptime\" AND resource.labels.zone = starts_with(\"us-west1-\")", }, + { + "compute service with no region/zone in config", + "compute.googleapis.com/firewall/dropped_bytes_count", + stackdriverMetricsRequester{config: config{}}, + "metric.type=\"compute.googleapis.com/firewall/dropped_bytes_count\"", + }, } for _, c := range cases { diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 23e258c0020..2ad466ac15d 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -27,3 +27,19 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 5m + +- module: googlecloud + metricsets: + - stackdriver + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" From 10d1dcb079fb04e58a1941c4ad023b8ab9de1c60 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 11 May 2020 13:57:03 -0600 Subject: [PATCH 3/4] add check if SamplePeriod is nil --- metricbeat/docs/modules/googlecloud.asciidoc | 3 ++- x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc | 3 ++- .../metricbeat/module/googlecloud/stackdriver/metricset.go | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index a8fd38f2afe..bf4ea8404f2 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -29,7 +29,8 @@ a partial region name like `us-east` or `us-east*`, which will monitor all regio `us-east`: `us-east1` and `us-east4`. If both region and zone are configured, only region will be used. Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] -for regions that are available in GCP. +for regions that are available in GCP. If both `region` and `zone` are not +specified, metrics will be collected from all regions/zones. * *project_id*: A single string with your GCP Project ID diff --git a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc index 692db78bb55..c1d05a7e5f5 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc @@ -19,7 +19,8 @@ a partial region name like `us-east` or `us-east*`, which will monitor all regio `us-east`: `us-east1` and `us-east4`. If both region and zone are configured, only region will be used. Please see https://cloud.google.com/compute/docs/regions-zones#available[GCP regions] -for regions that are available in GCP. +for regions that are available in GCP. If both `region` and `zone` are not +specified, metrics will be collected from all regions/zones. * *project_id*: A single string with your GCP Project ID diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go index 23a787b3014..7bc70929686 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go @@ -221,11 +221,16 @@ func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, proj return metricsWithMeta, errors.Errorf("Could not make ListMetricDescriptors request: %s: %v", mt, err) } + // Set samplePeriod default to 60 seconds and ingestDelay default to 0. meta := metricMeta{ - samplePeriod: time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second, + samplePeriod: 60 * time.Second, ingestDelay: 0 * time.Second, } + if out.Metadata.SamplePeriod != nil { + meta.samplePeriod = time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second + } + if out.Metadata.IngestDelay != nil { meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second } From 7064a1628d74bd88f0544a91fe2a9cdd8e4e6bf9 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 11 May 2020 15:14:43 -0600 Subject: [PATCH 4/4] add debug message for sample rate and ingest delay --- metricbeat/docs/modules/googlecloud.asciidoc | 17 +++++++++-------- x-pack/metricbeat/metricbeat.reference.yml | 17 +++++++++-------- .../module/googlecloud/_meta/config.yml | 17 +++++++++-------- .../module/googlecloud/stackdriver/metricset.go | 10 ++++++---- .../modules.d/googlecloud.yml.disabled | 17 +++++++++-------- 5 files changed, 42 insertions(+), 36 deletions(-) diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index bf4ea8404f2..fd0bd772cd4 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -244,14 +244,15 @@ metricbeat.modules: credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 1m - service: compute - metrics: - - aligner: ALIGN_NONE - metric_types: - - "compute.googleapis.com/instance/cpu/reserved_cores" - - "compute.googleapis.com/instance/cpu/usage_time" - - "compute.googleapis.com/instance/cpu/utilization" - - "compute.googleapis.com/instance/uptime" + stackdriver: + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" ---- [float] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 6a2582a6c49..1b5327397a4 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -530,14 +530,15 @@ metricbeat.modules: credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 1m - service: compute - metrics: - - aligner: ALIGN_NONE - metric_types: - - "compute.googleapis.com/instance/cpu/reserved_cores" - - "compute.googleapis.com/instance/cpu/usage_time" - - "compute.googleapis.com/instance/cpu/utilization" - - "compute.googleapis.com/instance/uptime" + stackdriver: + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" #------------------------------- Graphite Module ------------------------------- - module: graphite diff --git a/x-pack/metricbeat/module/googlecloud/_meta/config.yml b/x-pack/metricbeat/module/googlecloud/_meta/config.yml index 6a56af4e1bd..8103b2c111a 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/config.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/config.yml @@ -32,11 +32,12 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 1m - service: compute - metrics: - - aligner: ALIGN_NONE - metric_types: - - "compute.googleapis.com/instance/cpu/reserved_cores" - - "compute.googleapis.com/instance/cpu/usage_time" - - "compute.googleapis.com/instance/cpu/utilization" - - "compute.googleapis.com/instance/uptime" + stackdriver: + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime" diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go index 7bc70929686..e31aa96202c 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metricset.go @@ -106,7 +106,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, errors.Wrap(err, "error creating Stackdriver client") } - m.metricsMeta, err = metricDescriptor(ctx, client, m.config.ProjectID, m.stackDriverConfig) + m.metricsMeta, err = m.metricDescriptor(ctx, client) if err != nil { return nil, errors.Wrap(err, "error calling metricDescriptor function") } @@ -205,13 +205,13 @@ func (mc *stackDriverConfig) Validate() error { // metricDescriptor calls ListMetricDescriptorsRequest API to get metric metadata // (sample period and ingest delay) of each given metric type -func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, projectID string, stackDriverConfigs []stackDriverConfig) (map[string]metricMeta, error) { +func (m *MetricSet) metricDescriptor(ctx context.Context, client *monitoring.MetricClient) (map[string]metricMeta, error) { metricsWithMeta := make(map[string]metricMeta, 0) - for _, sdc := range stackDriverConfigs { + for _, sdc := range m.stackDriverConfig { for _, mt := range sdc.MetricTypes { req := &monitoringpb.ListMetricDescriptorsRequest{ - Name: "projects/" + projectID, + Name: "projects/" + m.config.ProjectID, Filter: fmt.Sprintf(`metric.type = "%s"`, mt), } @@ -228,10 +228,12 @@ func metricDescriptor(ctx context.Context, client *monitoring.MetricClient, proj } if out.Metadata.SamplePeriod != nil { + m.Logger().Debugf("For metric type %s: sample period = %s", mt, out.Metadata.SamplePeriod) meta.samplePeriod = time.Duration(out.Metadata.SamplePeriod.Seconds) * time.Second } if out.Metadata.IngestDelay != nil { + m.Logger().Debugf("For metric type %s: ingest delay = %s", mt, out.Metadata.IngestDelay) meta.ingestDelay = time.Duration(out.Metadata.IngestDelay.Seconds) * time.Second } diff --git a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled index 2ad466ac15d..aecd41c1ee3 100644 --- a/x-pack/metricbeat/modules.d/googlecloud.yml.disabled +++ b/x-pack/metricbeat/modules.d/googlecloud.yml.disabled @@ -35,11 +35,12 @@ credentials_file_path: "your JSON credentials file path" exclude_labels: false period: 1m - service: compute - metrics: - - aligner: ALIGN_NONE - metric_types: - - "compute.googleapis.com/instance/cpu/reserved_cores" - - "compute.googleapis.com/instance/cpu/usage_time" - - "compute.googleapis.com/instance/cpu/utilization" - - "compute.googleapis.com/instance/uptime" + stackdriver: + service: compute + metrics: + - aligner: ALIGN_NONE + metric_types: + - "compute.googleapis.com/instance/cpu/reserved_cores" + - "compute.googleapis.com/instance/cpu/usage_time" + - "compute.googleapis.com/instance/cpu/utilization" + - "compute.googleapis.com/instance/uptime"