diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b466ebcfe61..6f5a7a7baea 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -847,6 +847,7 @@ field. You can revert this change by configuring tags for the module and omittin - Release all kubernetes `state` metricsets as GA {pull}20901[20901] - Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985] - Sanitize `event.host`. {pull}21022[21022] +- Add support for different Azure Cloud environments in the metricbeat azure module. {pull}21044[21044] {issue}20988[20988] - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124] - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255] - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137] diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index 42d4d619c02..8ec43e4be43 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -70,7 +70,26 @@ Required credentials for the `azure` module: `tenant_id`:: The unique identifier of the Azure Active Directory instance -Users can use the azure credentials keys if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` +The azure credentials keys can be used if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` + +`resource_manager_endpoint` :: +_string_ +Optional, by default the azure public environment will be used, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn for azure ChinaCloud +https://management.microsoftazure.de for azure GermanCloud +https://management.azure.com for azure PublicCloud +https://management.usgovcloudapi.net for azure USGovernmentCloud + +`active_directory_endpoint` :: +_string_ +Optional, by default the associated active directory endpoint to the resource manager endpoint will be used, to override, users can provide a specific active directory endpoint in order to use a different azure environment. +Ex: +https://login.microsoftonline.com for azure ChinaCloud +https://login.microsoftonline.us for azure GermanCloud +https://login.chinacloudapi.cn for azure PublicCloud +https://login.microsoftonline.de for azure USGovernmentCloud + [float] == Metricsets diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc index 01f6389ab93..9065f2f283f 100644 --- a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc @@ -62,7 +62,26 @@ Required credentials for the `azure` module: `tenant_id`:: The unique identifier of the Azure Active Directory instance -Users can use the azure credentials keys if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` +The azure credentials keys can be used if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID` + +`resource_manager_endpoint` :: +_string_ +Optional, by default the azure public environment will be used, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn for azure ChinaCloud +https://management.microsoftazure.de for azure GermanCloud +https://management.azure.com for azure PublicCloud +https://management.usgovcloudapi.net for azure USGovernmentCloud + +`active_directory_endpoint` :: +_string_ +Optional, by default the associated active directory endpoint to the resource manager endpoint will be used, to override, users can provide a specific active directory endpoint in order to use a different azure environment. +Ex: +https://login.microsoftonline.com for azure ChinaCloud +https://login.microsoftonline.us for azure GermanCloud +https://login.chinacloudapi.cn for azure PublicCloud +https://login.microsoftonline.de for azure USGovernmentCloud + [float] == Metricsets diff --git a/x-pack/metricbeat/module/azure/billing/billing.go b/x-pack/metricbeat/module/azure/billing/billing.go index 2f6025ef1cf..f43c709a8c0 100644 --- a/x-pack/metricbeat/module/azure/billing/billing.go +++ b/x-pack/metricbeat/module/azure/billing/billing.go @@ -5,10 +5,10 @@ package billing import ( - "time" - "github.com/pkg/errors" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -32,19 +32,10 @@ type MetricSet struct { log *logp.Logger } -// Config options -type Config struct { - ClientId string `config:"client_id" validate:"required"` - ClientSecret string `config:"client_secret" validate:"required"` - TenantId string `config:"tenant_id" validate:"required"` - SubscriptionId string `config:"subscription_id" validate:"required"` - Period time.Duration `config:"period" validate:"nonzero,required"` -} - // New creates a new instance of the MetricSet. New is responsible for unpacking // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - var config Config + var config azure.Config err := base.Module().UnpackConfig(&config) if err != nil { return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig") diff --git a/x-pack/metricbeat/module/azure/billing/client.go b/x-pack/metricbeat/module/azure/billing/client.go index a6e4ba83cb7..0c3bc78d187 100644 --- a/x-pack/metricbeat/module/azure/billing/client.go +++ b/x-pack/metricbeat/module/azure/billing/client.go @@ -8,6 +8,8 @@ import ( "fmt" "time" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" + "github.com/pkg/errors" "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption" @@ -18,7 +20,7 @@ import ( // Client represents the azure client which will make use of the azure sdk go metrics related clients type Client struct { BillingService Service - Config Config + Config azure.Config Log *logp.Logger } @@ -29,8 +31,8 @@ type Usage struct { } // NewClient instantiates the an Azure monitoring client -func NewClient(config Config) (*Client, error) { - usageService, err := NewService(config.ClientId, config.ClientSecret, config.TenantId, config.SubscriptionId) +func NewClient(config azure.Config) (*Client, error) { + usageService, err := NewService(config) if err != nil { return nil, err } diff --git a/x-pack/metricbeat/module/azure/billing/client_test.go b/x-pack/metricbeat/module/azure/billing/client_test.go index a0eb6c0d31c..fd93cedad9d 100644 --- a/x-pack/metricbeat/module/azure/billing/client_test.go +++ b/x-pack/metricbeat/module/azure/billing/client_test.go @@ -11,10 +11,12 @@ import ( "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" ) var ( - config = Config{} + config = azure.Config{} ) func TestClient(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/billing/mock_service.go b/x-pack/metricbeat/module/azure/billing/mock_service.go index 4bbf4a16622..d0499701184 100644 --- a/x-pack/metricbeat/module/azure/billing/mock_service.go +++ b/x-pack/metricbeat/module/azure/billing/mock_service.go @@ -7,6 +7,8 @@ package billing import ( "github.com/stretchr/testify/mock" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption" @@ -27,7 +29,7 @@ type MockService struct { func NewMockClient() *Client { return &Client{ new(MockService), - Config{}, + azure.Config{}, logp.NewLogger("test azure monitor"), } } diff --git a/x-pack/metricbeat/module/azure/billing/service.go b/x-pack/metricbeat/module/azure/billing/service.go index ea7056e6c6f..17d8a9f9fd7 100644 --- a/x-pack/metricbeat/module/azure/billing/service.go +++ b/x-pack/metricbeat/module/azure/billing/service.go @@ -7,6 +7,8 @@ package billing import ( "context" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" + "github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption" "github.com/Azure/go-autorest/autorest/azure/auth" @@ -22,14 +24,16 @@ type UsageService struct { } // NewService instantiates the Azure monitoring service -func NewService(clientId string, clientSecret string, tenantId string, subscriptionId string) (*UsageService, error) { - clientConfig := auth.NewClientCredentialsConfig(clientId, clientSecret, tenantId) +func NewService(config azure.Config) (*UsageService, error) { + clientConfig := auth.NewClientCredentialsConfig(config.ClientId, config.ClientSecret, config.TenantId) + clientConfig.AADEndpoint = config.ActiveDirectoryEndpoint + clientConfig.Resource = config.ResourceManagerEndpoint authorizer, err := clientConfig.Authorizer() if err != nil { return nil, err } - forcastsClient := consumption.NewForecastsClient(subscriptionId) - usageDetailsClient := consumption.NewUsageDetailsClient(subscriptionId) + forcastsClient := consumption.NewForecastsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId) + usageDetailsClient := consumption.NewUsageDetailsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId) forcastsClient.Authorizer = authorizer usageDetailsClient.Authorizer = authorizer service := &UsageService{ diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index dd48f962b59..668356bb724 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -32,7 +32,7 @@ type mapResourceMetrics func(client *Client, resources []resources.GenericResour // NewClient instantiates the an Azure monitoring client func NewClient(config Config) (*Client, error) { - azureMonitorService, err := NewService(config.ClientId, config.ClientSecret, config.TenantId, config.SubscriptionId) + azureMonitorService, err := NewService(config) if err != nil { return nil, err } diff --git a/x-pack/metricbeat/module/azure/config.go b/x-pack/metricbeat/module/azure/config.go index 00f1af56126..3748e89abec 100644 --- a/x-pack/metricbeat/module/azure/config.go +++ b/x-pack/metricbeat/module/azure/config.go @@ -7,20 +7,38 @@ package azure import ( "time" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/pkg/errors" ) +const ( + // DefaultBaseURI is the default URI used for the service Insights + DefaultBaseURI = "https://management.azure.com/" +) + +var ( + AzureEnvs = common.MapStr{ + "https://management.azure.com/": "https://login.microsoftonline.com/", + "https://management.usgovcloudapi.net/": "https://login.microsoftonline.us/", + "https://management.chinacloudapi.cn/": "https://login.chinacloudapi.cn/", + "https://management.microsoftazure.de/": "https://login.microsoftonline.de/", + } +) + // Config options type Config struct { - ClientId string `config:"client_id"` - ClientSecret string `config:"client_secret"` - TenantId string `config:"tenant_id"` - SubscriptionId string `config:"subscription_id"` - Period time.Duration `config:"period" validate:"nonzero,required"` - Resources []ResourceConfig `config:"resources"` - RefreshListInterval time.Duration `config:"refresh_list_interval"` - DefaultResourceType string `config:"default_resource_type"` - AddCloudMetadata bool `config:"add_cloud_metadata"` + ClientId string `config:"client_id" validate:"required"` + ClientSecret string `config:"client_secret" validate:"required"` + TenantId string `config:"tenant_id" validate:"required"` + SubscriptionId string `config:"subscription_id" validate:"required"` + Period time.Duration `config:"period" validate:"nonzero,required"` + Resources []ResourceConfig `config:"resources"` + RefreshListInterval time.Duration `config:"refresh_list_interval"` + DefaultResourceType string `config:"default_resource_type"` + AddCloudMetadata bool `config:"add_cloud_metadata"` + ResourceManagerEndpoint string `config:"resource_manager_endpoint"` + ActiveDirectoryEndpoint string `config:"active_directory_endpoint"` } // ResourceConfig contains resource and metric list specific configuration. @@ -52,17 +70,24 @@ type DimensionConfig struct { } func (conf *Config) Validate() error { - if conf.SubscriptionId == "" { - return errors.New("no subscription ID has been configured") - } - if conf.ClientSecret == "" { - return errors.New("no client secret has been configured") - } - if conf.ClientId == "" { - return errors.New("no client ID has been configured") + if conf.ResourceManagerEndpoint == "" { + conf.ResourceManagerEndpoint = DefaultBaseURI } - if conf.TenantId == "" { - return errors.New("no tenant ID has been configured") + if conf.ActiveDirectoryEndpoint == "" { + ok, err := AzureEnvs.HasKey(conf.ResourceManagerEndpoint) + if err != nil { + return errors.Wrap(err, "No active directory endpoint found for the resource manager endpoint selected.") + } + if ok { + add, err := AzureEnvs.GetValue(conf.ResourceManagerEndpoint) + if err != nil { + return errors.Wrap(err, "No active directory endpoint found for the resource manager endpoint selected.") + } + conf.ActiveDirectoryEndpoint = add.(string) + } + if conf.ActiveDirectoryEndpoint == "" { + return errors.New("no active directory endpoint has been configured") + } } return nil } diff --git a/x-pack/metricbeat/module/azure/monitor_service.go b/x-pack/metricbeat/module/azure/monitor_service.go index c3ed4e2fa43..bb77712b49e 100644 --- a/x-pack/metricbeat/module/azure/monitor_service.go +++ b/x-pack/metricbeat/module/azure/monitor_service.go @@ -29,16 +29,18 @@ type MonitorService struct { const metricNameLimit = 20 // NewService instantiates the Azure monitoring service -func NewService(clientId string, clientSecret string, tenantId string, subscriptionId string) (*MonitorService, error) { - clientConfig := auth.NewClientCredentialsConfig(clientId, clientSecret, tenantId) +func NewService(config Config) (*MonitorService, error) { + clientConfig := auth.NewClientCredentialsConfig(config.ClientId, config.ClientSecret, config.TenantId) + clientConfig.AADEndpoint = config.ActiveDirectoryEndpoint + clientConfig.Resource = config.ResourceManagerEndpoint authorizer, err := clientConfig.Authorizer() if err != nil { return nil, err } - metricsClient := insights.NewMetricsClient(subscriptionId) - metricsDefinitionClient := insights.NewMetricDefinitionsClient(subscriptionId) - resourceClient := resources.NewClient(subscriptionId) - metricNamespaceClient := insights.NewMetricNamespacesClient(subscriptionId) + metricsClient := insights.NewMetricsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId) + metricsDefinitionClient := insights.NewMetricDefinitionsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId) + resourceClient := resources.NewClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId) + metricNamespaceClient := insights.NewMetricNamespacesClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId) metricsClient.Authorizer = authorizer metricsDefinitionClient.Authorizer = authorizer resourceClient.Authorizer = authorizer