From bedc5350dee02c95ee44001a34993bf12637a993 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 29 Jun 2021 00:31:28 +0800 Subject: [PATCH] [Metricbeat] Change Account ID to Project ID in `gcp.billing` module (#26412) (#26505) (cherry picked from commit 076e0a693c18617454388c6759c4010e821c0f59) Co-authored-by: Alex Resnick --- CHANGELOG.next.asciidoc | 1 + .../metricbeat/module/gcp/billing/billing.go | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 77b37738b2d..a6b20dce5d7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -380,6 +380,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix copy-paste error in libbeat docs. {pull}25448[25448] - Fix azure billing dashboard. {pull}25554[25554] - Major refactor of system/cpu and system/core metrics. {pull}25771[25771] +- Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412] *Packetbeat* diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index 13236ff4ae4..a14d263ac6b 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -209,6 +209,8 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, SELECT invoice.month, project.id, + project.name, + billing_account_id, cost_type, (SUM(CAST(cost * 1000000 AS int64)) + SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000 @@ -217,8 +219,8 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, WHERE project.id IS NOT NULL AND invoice.month = '%s' AND cost_type = '%s' - GROUP BY 1, 2, 3 - ORDER BY 1 ASC, 2 ASC, 3 ASC;`, tableMeta.tableFullID, month, costType) + GROUP BY 1, 2, 3, 4, 5 + ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC;`, tableMeta.tableFullID, month, costType) q := client.Query(query) m.logger.Debug("bigquery query = ", query) @@ -261,26 +263,29 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, return events, err } - if len(row) == 4 { + if len(row) == 6 { events = append(events, createEvents(row, m.config.ProjectID)) } } return events, nil } -func createEvents(rowItems []bigquery.Value, accountID string) mb.Event { +func createEvents(rowItems []bigquery.Value, projectID string) mb.Event { event := mb.Event{} event.MetricSetFields = common.MapStr{ - "invoice_month": rowItems[0], - "project_id": rowItems[1], - "cost_type": rowItems[2], - "total": rowItems[3], + "invoice_month": rowItems[0], + "project_id": rowItems[1], + "project_name": rowItems[2], + "billing_account_id": rowItems[3], + "cost_type": rowItems[4], + "total": rowItems[5], } event.RootFields = common.MapStr{ "cloud.provider": "gcp", - "cloud.account.id": accountID, - "cloud.account.name": accountID, + "cloud.project.id": projectID, + "cloud.project.name": rowItems[2], + "cloud.account.id": rowItems[3], } // create eventID for each current_date + invoice_month + project_id + cost_type