Azure monitor support for azurerm_hdinsight_*_cluster

azurerm/helpers/azure/hdinsight.go

@@ -164,6 +164,34 @@ func SchemaHDInsightsExternalMetastore() *schema.Schema {
 	}
 }
 
+func SchemaHDInsightsMonitor() *schema.Schema {
+	return &schema.Schema{
+		Type:     schema.TypeList,
+		Optional: true,
+		MaxItems: 1,
+		Elem: &schema.Resource{
+			Schema: map[string]*schema.Schema{
+				"log_analytics_workspace_id": {
+					Type:     schema.TypeString,
+					Required: true,
+					ForceNew: false,
+				},
+				"primary_key": {
+					Type:         schema.TypeString,
+					Required:     true,
+					ForceNew:     false,
+					Sensitive:    true,
+					ValidateFunc: validation.StringIsNotEmpty,
+					// Azure doesn't return the key
+					DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
+						return (new == d.Get(k).(string)) && (old == "*****")
+					},
+				},
+			},
+		},
+	}
+}
+
 func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} {
 	vs := input[0].(map[string]interface{})
 
@@ -261,6 +289,18 @@ func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{}
 	}
 }
 
+func ExpandHDInsightsMonitor(input []interface{}) hdinsight.ClusterMonitoringRequest {
+	vs := input[0].(map[string]interface{})
+
+	workspace := vs["log_analytics_workspace_id"].(string)
+	key := vs["primary_key"].(string)
+
+	return hdinsight.ClusterMonitoringRequest{
+		WorkspaceID: utils.String(workspace),
+		PrimaryKey:  utils.String(key),
+	}
+}
+
 func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
 	enabled := true
 

azurerm/internal/services/hdinsight/client/client.go

@@ -9,6 +9,7 @@ type Client struct {
 	ApplicationsClient   *hdinsight.ApplicationsClient
 	ClustersClient       *hdinsight.ClustersClient
 	ConfigurationsClient *hdinsight.ConfigurationsClient
+	ExtensionsClient     *hdinsight.ExtensionsClient
 }
 
 func NewClient(o *common.ClientOptions) *Client {
@@ -21,9 +22,13 @@ func NewClient(o *common.ClientOptions) *Client {
 	ConfigurationsClient := hdinsight.NewConfigurationsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
 	o.ConfigureClient(&ConfigurationsClient.Client, o.ResourceManagerAuthorizer)
 
+	ExtensionsClient := hdinsight.NewExtensionsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
+	o.ConfigureClient(&ExtensionsClient.Client, o.ResourceManagerAuthorizer)
+
 	return &Client{
 		ApplicationsClient:   &ApplicationsClient,
 		ClustersClient:       &ClustersClient,
 		ConfigurationsClient: &ConfigurationsClient,
+		ExtensionsClient:     &ExtensionsClient,
 	}
 }

azurerm/internal/services/hdinsight/common_hdinsight.go

@@ -106,6 +106,18 @@ func hdinsightClusterUpdate(clusterKind string, readFunc schema.ReadFunc) schema
 			}
 		}
 
+		if d.HasChange("monitor") {
+			log.Printf("[DEBUG] Change Azure Monitor for the HDInsight %q Cluster", clusterKind)
+			extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
+			if v, ok := d.GetOk("monitor"); ok {
+				monitorRaw := v.([]interface{})
+				if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+					return err
+				}
+			} else if err := disableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name); err != nil {
+				return err
+			}
+		}
 		if d.HasChange("gateway") {
 			log.Printf("[DEBUG] Updating the HDInsight %q Cluster gateway", clusterKind)
 			vs := d.Get("gateway").([]interface{})[0].(map[string]interface{})
@@ -348,3 +360,42 @@ func flattenHDInsightsMetastores(d *schema.ResourceData, configurations map[stri
 		})
 	}
 }
+
+func flattenHDInsightMonitoring(monitor hdinsight.ClusterMonitoringResponse) []interface{} {
+	if *monitor.ClusterMonitoringEnabled {
+		return []interface{}{
+			map[string]string{
+				"log_analytics_workspace_id": *monitor.WorkspaceID,
+				"primary_key":                "*****",
+			}}
+	}
+
+	return nil
+}
+
+func enableHDInsightMonitoring(ctx context.Context, client *hdinsight.ExtensionsClient, resourceGroup, name string, input []interface{}) error {
+	monitor := azure.ExpandHDInsightsMonitor(input)
+	future, err := client.EnableMonitoring(ctx, resourceGroup, name, monitor)
+	if err != nil {
+		return err
+	}
+
+	if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
+		return fmt.Errorf("Error waiting for enabling monitor for  HDInsight Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+	}
+
+	return nil
+}
+
+func disableHDInsightMonitoring(ctx context.Context, client *hdinsight.ExtensionsClient, resourceGroup, name string) error {
+	future, err := client.DisableMonitoring(ctx, resourceGroup, name)
+	if err != nil {
+		return err
+	}
+
+	if err := future.WaitForCompletionRef(ctx, client.Client); err != nil {
+		return fmt.Errorf("Error waiting for disabling monitor for  HDInsight Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+	}
+
+	return nil
+}

azurerm/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go

@@ -179,12 +179,15 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
 
 func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interface{}) error {
 	client := meta.(*clients.Client).HDInsight.ClustersClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -308,12 +311,21 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
 		}
 	}
 
+	// We can only enable monitoring after creation
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightHadoopClusterRead(d, meta)
 }
 
 func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interface{}) error {
 	clustersClient := meta.(*clients.Client).HDInsight.ClustersClient
 	configurationsClient := meta.(*clients.Client).HDInsight.ConfigurationsClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForRead(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -399,6 +411,13 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go

@@ -117,12 +117,15 @@ func resourceArmHDInsightHBaseCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
 
 func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interface{}) error {
 	client := meta.(*clients.Client).HDInsight.ClustersClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -213,12 +216,21 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightHBaseClusterRead(d, meta)
 }
 
 func resourceArmHDInsightHBaseClusterRead(d *schema.ResourceData, meta interface{}) error {
 	clustersClient := meta.(*clients.Client).HDInsight.ClustersClient
 	configurationsClient := meta.(*clients.Client).HDInsight.ConfigurationsClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForRead(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -282,6 +294,13 @@ func resourceArmHDInsightHBaseClusterRead(d *schema.ResourceData, meta interface
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go

@@ -117,12 +117,15 @@ func resourceArmHDInsightInteractiveQueryCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
 
 func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, meta interface{}) error {
 	client := meta.(*clients.Client).HDInsight.ClustersClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -213,12 +216,21 @@ func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, m
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightInteractiveQueryClusterRead(d, meta)
 }
 
 func resourceArmHDInsightInteractiveQueryClusterRead(d *schema.ResourceData, meta interface{}) error {
 	clustersClient := meta.(*clients.Client).HDInsight.ClustersClient
 	configurationsClient := meta.(*clients.Client).HDInsight.ConfigurationsClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForRead(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -282,6 +294,13 @@ func resourceArmHDInsightInteractiveQueryClusterRead(d *schema.ResourceData, met
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_kafka_cluster_resource.go

@@ -118,12 +118,15 @@ func resourceArmHDInsightKafkaCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
 
 func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interface{}) error {
 	client := meta.(*clients.Client).HDInsight.ClustersClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -214,12 +217,21 @@ func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interfa
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightKafkaClusterRead(d, meta)
 }
 
 func resourceArmHDInsightKafkaClusterRead(d *schema.ResourceData, meta interface{}) error {
 	clustersClient := meta.(*clients.Client).HDInsight.ClustersClient
 	configurationsClient := meta.(*clients.Client).HDInsight.ConfigurationsClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForRead(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -283,6 +295,13 @@ func resourceArmHDInsightKafkaClusterRead(d *schema.ResourceData, meta interface
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)

azurerm/internal/services/hdinsight/hdinsight_spark_cluster_resource.go

@@ -117,12 +117,15 @@ func resourceArmHDInsightSparkCluster() *schema.Resource {
 				Type:     schema.TypeString,
 				Computed: true,
 			},
+
+			"monitor": azure.SchemaHDInsightsMonitor(),
 		},
 	}
 }
 
 func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interface{}) error {
 	client := meta.(*clients.Client).HDInsight.ClustersClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -213,12 +216,21 @@ func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interfa
 
 	d.SetId(*read.ID)
 
+	// We can only enable monitoring after creation
+	if v, ok := d.GetOk("monitor"); ok {
+		monitorRaw := v.([]interface{})
+		if err := enableHDInsightMonitoring(ctx, extensionsClient, resourceGroup, name, monitorRaw); err != nil {
+			return err
+		}
+	}
+
 	return resourceArmHDInsightSparkClusterRead(d, meta)
 }
 
 func resourceArmHDInsightSparkClusterRead(d *schema.ResourceData, meta interface{}) error {
 	clustersClient := meta.(*clients.Client).HDInsight.ClustersClient
 	configurationsClient := meta.(*clients.Client).HDInsight.ConfigurationsClient
+	extensionsClient := meta.(*clients.Client).HDInsight.ExtensionsClient
 	ctx, cancel := timeouts.ForRead(meta.(*clients.Client).StopContext, d)
 	defer cancel()
 
@@ -282,6 +294,13 @@ func resourceArmHDInsightSparkClusterRead(d *schema.ResourceData, meta interface
 		d.Set("https_endpoint", httpEndpoint)
 		sshEndpoint := azure.FindHDInsightConnectivityEndpoint("SSH", props.ConnectivityEndpoints)
 		d.Set("ssh_endpoint", sshEndpoint)
+
+		monitor, err := extensionsClient.GetMonitoringStatus(ctx, resourceGroup, name)
+		if err != nil {
+			return fmt.Errorf("Error reading monitor configuation for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+		}
+
+		d.Set("monitor", flattenHDInsightMonitoring(monitor))
 	}
 
 	return tags.FlattenAndSet(d, resp.Tags)