Add support for different Azure Cloud environments in the metricbeat …

…azure module (elastic#21044)

* mofidy doc

* work

* changelog

* fix url

* work

* fmt update

* err

(cherry picked from commit 554d564)

CHANGELOG.next.asciidoc

@@ -856,6 +856,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Release all kubernetes `state` metricsets as GA {pull}20901[20901]
 - Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985]
 - Sanitize `event.host`. {pull}21022[21022]
+- Add support for different Azure Cloud environments in the metricbeat azure module. {pull}21044[21044] {issue}20988[20988]
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
 - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
 - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]

metricbeat/docs/modules/azure.asciidoc

@@ -70,7 +70,26 @@ Required credentials for the `azure` module:
 `tenant_id`:: The unique identifier of the Azure Active Directory instance
 
 
-Users can use the azure credentials keys if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID`
+The azure credentials keys can be used if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID`
+
+`resource_manager_endpoint` ::
+_string_
+Optional, by default the azure public environment will be used, to override, users can provide a specific resource manager endpoint in order to use a different azure environment.
+Ex:
+https://management.chinacloudapi.cn for azure ChinaCloud
+https://management.microsoftazure.de for azure GermanCloud
+https://management.azure.com for azure PublicCloud
+https://management.usgovcloudapi.net for azure USGovernmentCloud
+
+`active_directory_endpoint` ::
+_string_
+Optional, by default the associated active directory endpoint to the resource manager endpoint will be used, to override, users can provide a specific active directory endpoint in order to use a different azure environment.
+Ex:
+https://login.microsoftonline.com for azure ChinaCloud
+https://login.microsoftonline.us for azure GermanCloud
+https://login.chinacloudapi.cn for azure PublicCloud
+https://login.microsoftonline.de for azure USGovernmentCloud
+
 
 [float]
 == Metricsets

x-pack/metricbeat/module/azure/_meta/docs.asciidoc

@@ -62,7 +62,26 @@ Required credentials for the `azure` module:
 `tenant_id`:: The unique identifier of the Azure Active Directory instance
 
 
-Users can use the azure credentials keys if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID`
+The azure credentials keys can be used if configured `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID`, `AZURE_SUBSCRIPTION_ID`
+
+`resource_manager_endpoint` ::
+_string_
+Optional, by default the azure public environment will be used, to override, users can provide a specific resource manager endpoint in order to use a different azure environment.
+Ex:
+https://management.chinacloudapi.cn for azure ChinaCloud
+https://management.microsoftazure.de for azure GermanCloud
+https://management.azure.com for azure PublicCloud
+https://management.usgovcloudapi.net for azure USGovernmentCloud
+
+`active_directory_endpoint` ::
+_string_
+Optional, by default the associated active directory endpoint to the resource manager endpoint will be used, to override, users can provide a specific active directory endpoint in order to use a different azure environment.
+Ex:
+https://login.microsoftonline.com for azure ChinaCloud
+https://login.microsoftonline.us for azure GermanCloud
+https://login.chinacloudapi.cn for azure PublicCloud
+https://login.microsoftonline.de for azure USGovernmentCloud
+
 
 [float]
 == Metricsets

x-pack/metricbeat/module/azure/billing/billing.go

@@ -5,10 +5,10 @@
 package billing
 
 import (
-	"time"
-
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
+
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
@@ -32,19 +32,10 @@ type MetricSet struct {
 	log    *logp.Logger
 }
 
-// Config options
-type Config struct {
-	ClientId       string        `config:"client_id"    validate:"required"`
-	ClientSecret   string        `config:"client_secret" validate:"required"`
-	TenantId       string        `config:"tenant_id" validate:"required"`
-	SubscriptionId string        `config:"subscription_id" validate:"required"`
-	Period         time.Duration `config:"period" validate:"nonzero,required"`
-}
-
 // New creates a new instance of the MetricSet. New is responsible for unpacking
 // any MetricSet specific configuration options if there are any.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	var config Config
+	var config azure.Config
 	err := base.Module().UnpackConfig(&config)
 	if err != nil {
 		return nil, errors.Wrap(err, "error unpack raw module config using UnpackConfig")

x-pack/metricbeat/module/azure/billing/client.go

@@ -8,6 +8,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
+
 	"github.com/pkg/errors"
 
 	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
@@ -18,7 +20,7 @@ import (
 // Client represents the azure client which will make use of the azure sdk go metrics related clients
 type Client struct {
 	BillingService Service
-	Config         Config
+	Config         azure.Config
 	Log            *logp.Logger
 }
 
@@ -29,8 +31,8 @@ type Usage struct {
 }
 
 // NewClient instantiates the an Azure monitoring client
-func NewClient(config Config) (*Client, error) {
-	usageService, err := NewService(config.ClientId, config.ClientSecret, config.TenantId, config.SubscriptionId)
+func NewClient(config azure.Config) (*Client, error) {
+	usageService, err := NewService(config)
 	if err != nil {
 		return nil, err
 	}

x-pack/metricbeat/module/azure/billing/client_test.go

@@ -11,10 +11,12 @@ import (
 	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
+
+	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
 )
 
 var (
-	config = Config{}
+	config = azure.Config{}
 )
 
 func TestClient(t *testing.T) {

x-pack/metricbeat/module/azure/billing/mock_service.go

@@ -7,6 +7,8 @@ package billing
 import (
 	"github.com/stretchr/testify/mock"
 
+	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
+
 	"github.com/elastic/beats/v7/libbeat/logp"
 
 	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
@@ -27,7 +29,7 @@ type MockService struct {
 func NewMockClient() *Client {
 	return &Client{
 		new(MockService),
-		Config{},
+		azure.Config{},
 		logp.NewLogger("test azure monitor"),
 	}
 }

x-pack/metricbeat/module/azure/billing/service.go

@@ -7,6 +7,8 @@ package billing
 import (
 	"context"
 
+	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
+
 	"github.com/Azure/azure-sdk-for-go/services/consumption/mgmt/2019-01-01/consumption"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
 
@@ -22,14 +24,16 @@ type UsageService struct {
 }
 
 // NewService instantiates the Azure monitoring service
-func NewService(clientId string, clientSecret string, tenantId string, subscriptionId string) (*UsageService, error) {
-	clientConfig := auth.NewClientCredentialsConfig(clientId, clientSecret, tenantId)
+func NewService(config azure.Config) (*UsageService, error) {
+	clientConfig := auth.NewClientCredentialsConfig(config.ClientId, config.ClientSecret, config.TenantId)
+	clientConfig.AADEndpoint = config.ActiveDirectoryEndpoint
+	clientConfig.Resource = config.ResourceManagerEndpoint
 	authorizer, err := clientConfig.Authorizer()
 	if err != nil {
 		return nil, err
 	}
-	forcastsClient := consumption.NewForecastsClient(subscriptionId)
-	usageDetailsClient := consumption.NewUsageDetailsClient(subscriptionId)
+	forcastsClient := consumption.NewForecastsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId)
+	usageDetailsClient := consumption.NewUsageDetailsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId)
 	forcastsClient.Authorizer = authorizer
 	usageDetailsClient.Authorizer = authorizer
 	service := &UsageService{

x-pack/metricbeat/module/azure/client.go

@@ -32,7 +32,7 @@ type mapResourceMetrics func(client *Client, resources []resources.GenericResour
 
 // NewClient instantiates the an Azure monitoring client
 func NewClient(config Config) (*Client, error) {
-	azureMonitorService, err := NewService(config.ClientId, config.ClientSecret, config.TenantId, config.SubscriptionId)
+	azureMonitorService, err := NewService(config)
 	if err != nil {
 		return nil, err
 	}

x-pack/metricbeat/module/azure/config.go

@@ -7,20 +7,38 @@ package azure
 import (
 	"time"
 
+	"github.com/elastic/beats/v7/libbeat/common"
+
 	"github.com/pkg/errors"
 )
 
+const (
+	// DefaultBaseURI is the default URI used for the service Insights
+	DefaultBaseURI = "https://management.azure.com/"
+)
+
+var (
+	AzureEnvs = common.MapStr{
+		"https://management.azure.com/":         "https://login.microsoftonline.com/",
+		"https://management.usgovcloudapi.net/": "https://login.microsoftonline.us/",
+		"https://management.chinacloudapi.cn/":  "https://login.chinacloudapi.cn/",
+		"https://management.microsoftazure.de/": "https://login.microsoftonline.de/",
+	}
+)
+
 // Config options
 type Config struct {
-	ClientId            string           `config:"client_id"`
-	ClientSecret        string           `config:"client_secret"`
-	TenantId            string           `config:"tenant_id"`
-	SubscriptionId      string           `config:"subscription_id"`
-	Period              time.Duration    `config:"period" validate:"nonzero,required"`
-	Resources           []ResourceConfig `config:"resources"`
-	RefreshListInterval time.Duration    `config:"refresh_list_interval"`
-	DefaultResourceType string           `config:"default_resource_type"`
-	AddCloudMetadata    bool             `config:"add_cloud_metadata"`
+	ClientId                string           `config:"client_id"  validate:"required"`
+	ClientSecret            string           `config:"client_secret"  validate:"required"`
+	TenantId                string           `config:"tenant_id"  validate:"required"`
+	SubscriptionId          string           `config:"subscription_id"  validate:"required"`
+	Period                  time.Duration    `config:"period" validate:"nonzero,required"`
+	Resources               []ResourceConfig `config:"resources"`
+	RefreshListInterval     time.Duration    `config:"refresh_list_interval"`
+	DefaultResourceType     string           `config:"default_resource_type"`
+	AddCloudMetadata        bool             `config:"add_cloud_metadata"`
+	ResourceManagerEndpoint string           `config:"resource_manager_endpoint"`
+	ActiveDirectoryEndpoint string           `config:"active_directory_endpoint"`
 }
 
 // ResourceConfig contains resource and metric list specific configuration.
@@ -52,17 +70,24 @@ type DimensionConfig struct {
 }
 
 func (conf *Config) Validate() error {
-	if conf.SubscriptionId == "" {
-		return errors.New("no subscription ID has been configured")
-	}
-	if conf.ClientSecret == "" {
-		return errors.New("no client secret has been configured")
-	}
-	if conf.ClientId == "" {
-		return errors.New("no client ID has been configured")
+	if conf.ResourceManagerEndpoint == "" {
+		conf.ResourceManagerEndpoint = DefaultBaseURI
 	}
-	if conf.TenantId == "" {
-		return errors.New("no tenant ID has been configured")
+	if conf.ActiveDirectoryEndpoint == "" {
+		ok, err := AzureEnvs.HasKey(conf.ResourceManagerEndpoint)
+		if err != nil {
+			return errors.Wrap(err, "No active directory endpoint found for the resource manager endpoint selected.")
+		}
+		if ok {
+			add, err := AzureEnvs.GetValue(conf.ResourceManagerEndpoint)
+			if err != nil {
+				return errors.Wrap(err, "No active directory endpoint found for the resource manager endpoint selected.")
+			}
+			conf.ActiveDirectoryEndpoint = add.(string)
+		}
+		if conf.ActiveDirectoryEndpoint == "" {
+			return errors.New("no active directory endpoint has been configured")
+		}
 	}
 	return nil
 }

x-pack/metricbeat/module/azure/monitor_service.go

@@ -29,16 +29,18 @@ type MonitorService struct {
 const metricNameLimit = 20
 
 // NewService instantiates the Azure monitoring service
-func NewService(clientId string, clientSecret string, tenantId string, subscriptionId string) (*MonitorService, error) {
-	clientConfig := auth.NewClientCredentialsConfig(clientId, clientSecret, tenantId)
+func NewService(config Config) (*MonitorService, error) {
+	clientConfig := auth.NewClientCredentialsConfig(config.ClientId, config.ClientSecret, config.TenantId)
+	clientConfig.AADEndpoint = config.ActiveDirectoryEndpoint
+	clientConfig.Resource = config.ResourceManagerEndpoint
 	authorizer, err := clientConfig.Authorizer()
 	if err != nil {
 		return nil, err
 	}
-	metricsClient := insights.NewMetricsClient(subscriptionId)
-	metricsDefinitionClient := insights.NewMetricDefinitionsClient(subscriptionId)
-	resourceClient := resources.NewClient(subscriptionId)
-	metricNamespaceClient := insights.NewMetricNamespacesClient(subscriptionId)
+	metricsClient := insights.NewMetricsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId)
+	metricsDefinitionClient := insights.NewMetricDefinitionsClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId)
+	resourceClient := resources.NewClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId)
+	metricNamespaceClient := insights.NewMetricNamespacesClientWithBaseURI(config.ResourceManagerEndpoint, config.SubscriptionId)
 	metricsClient.Authorizer = authorizer
 	metricsDefinitionClient.Authorizer = authorizer
 	resourceClient.Authorizer = authorizer