Add external metastores to azurerm_hdinsight_hadoop_cluster

azurerm/helpers/azure/hdinsight.go

@@ -39,7 +39,7 @@ func SchemaHDInsightTier() *schema.Schema {
 		ValidateFunc: validation.StringInSlice([]string{
 			string(hdinsight.Standard),
 			string(hdinsight.Premium),
-		}, false),
+		}, true),
 		// TODO: file a bug about this
 		DiffSuppressFunc: SuppressLocationDiff,
 	}
@@ -105,6 +105,43 @@ func SchemaHDInsightsGateway() *schema.Schema {
 	}
 }
 
+func SchemaHDInsightsExternalMetastore() *schema.Schema {
+	return &schema.Schema{
+		Type:     schema.TypeList,
+		Optional: true,
+		MaxItems: 1,
+		Elem: &schema.Resource{
+			Schema: map[string]*schema.Schema{
+				"server": {
+					Type:     schema.TypeString,
+					Required: true,
+					ForceNew: true,
+				},
+				"database_name": {
+					Type:     schema.TypeString,
+					Required: true,
+					ForceNew: true,
+				},
+				"username": {
+					Type:     schema.TypeString,
+					Required: true,
+					ForceNew: true,
+				},
+				"password": {
+					Type:      schema.TypeString,
+					Required:  true,
+					ForceNew:  true,
+					Sensitive: true,
+					// Azure returns the key as *****. We'll suppress that here.
+					DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
+						return (new == d.Get(k).(string)) && (old == "*****")
+					},
+				},
+			},
+		},
+	}
+}
+
 func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} {
 	vs := input[0].(map[string]interface{})
 
@@ -122,6 +159,77 @@ func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{}
 	}
 }
 
+func ExpandHDInsightsHiveMetastore(input []interface{}) map[string]interface{} {
+	vs := input[0].(map[string]interface{})
+
+	server := vs["server"].(string)
+	database := vs["database_name"].(string)
+	username := vs["username"].(string)
+	password := vs["password"].(string)
+
+	return map[string]interface{}{
+		"hive-site": map[string]interface{}{
+			"javax.jdo.option.ConnectionDriverName": "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+			"javax.jdo.option.ConnectionURL":        fmt.Sprintf("jdbc:sqlserver://%s;database=%s;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", server, database),
+			"javax.jdo.option.ConnectionUserName":   username,
+			"javax.jdo.option.ConnectionPassword":   password,
+		},
+		"hive-env": map[string]interface{}{
+			"hive_database":                       "Existing MSSQL Server database with SQL authentication",
+			"hive_database_name":                  database,
+			"hive_database_type":                  "mssql",
+			"hive_existing_mssql_server_database": database,
+			"hive_existing_mssql_server_host":     server,
+			"hive_hostname":                       server,
+		},
+	}
+}
+
+func ExpandHDInsightsOozieMetastore(input []interface{}) map[string]interface{} {
+	vs := input[0].(map[string]interface{})
+
+	server := vs["server"].(string)
+	database := vs["database_name"].(string)
+	username := vs["username"].(string)
+	password := vs["password"].(string)
+
+	return map[string]interface{}{
+		"oozie-site": map[string]interface{}{
+			"oozie.service.JPAService.jdbc.driver":   "com.microsoft.sqlserver.jdbc.SQLServerDriver",
+			"oozie.service.JPAService.jdbc.url":      fmt.Sprintf("jdbc:sqlserver://%s;database=%s;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", server, database),
+			"oozie.service.JPAService.jdbc.username": username,
+			"oozie.service.JPAService.jdbc.password": password,
+			"oozie.db.schema.name":                   "oozie",
+		},
+		"oozie-env": map[string]interface{}{
+			"oozie_database":                       "Existing MSSQL Server database with SQL authentication",
+			"oozie_database_name":                  database,
+			"oozie_database_type":                  "mssql",
+			"oozie_existing_mssql_server_database": database,
+			"oozie_existing_mssql_server_host":     server,
+			"oozie_hostname":                       server,
+		},
+	}
+}
+
+func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{} {
+	vs := input[0].(map[string]interface{})
+
+	server := vs["server"].(string)
+	database := vs["database_name"].(string)
+	username := vs["username"].(string)
+	password := vs["password"].(string)
+
+	return map[string]interface{}{
+		"ambari-conf": map[string]interface{}{
+			"database-server":        server,
+			"database-name":          database,
+			"database-user-name":     username,
+			"database-user-password": password,
+		},
+	}
+}
+
 func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
 	enabled := false
 	if v, exists := input["restAuthCredential.isEnabled"]; exists && v != nil {
@@ -150,6 +258,111 @@ func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
 	}
 }
 
+func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*string) []interface{} {
+	server := ""
+	if v, exists := env["hive_hostname"]; exists && v != nil {
+		server = *v
+	}
+
+	database := ""
+	if v, exists := env["hive_database_name"]; exists && v != nil {
+		database = *v
+	}
+
+	username := ""
+	if v, exists := site["javax.jdo.option.ConnectionUserName"]; exists && v != nil {
+		username = *v
+	}
+
+	password := ""
+	if v, exists := site["javax.jdo.option.ConnectionPassword"]; exists && v != nil {
+		password = *v
+	}
+
+	if server != "" && database != "" {
+		return []interface{}{
+			map[string]interface{}{
+				"server":        server,
+				"database_name": database,
+				"username":      username,
+				"password":      password,
+			},
+		}
+	}
+
+	return nil
+}
+
+func FlattenHDInsightsOozieMetastore(env map[string]*string, site map[string]*string) []interface{} {
+	server := ""
+	if v, exists := env["oozie_hostname"]; exists && v != nil {
+		server = *v
+	}
+
+	database := ""
+	if v, exists := env["oozie_database_name"]; exists && v != nil {
+		database = *v
+	}
+
+	username := ""
+	if v, exists := site["oozie.service.JPAService.jdbc.username"]; exists && v != nil {
+		username = *v
+	}
+
+	password := ""
+	if v, exists := site["oozie.service.JPAService.jdbc.password"]; exists && v != nil {
+		password = *v
+	}
+
+	if server != "" && database != "" {
+		return []interface{}{
+			map[string]interface{}{
+				"server":        server,
+				"database_name": database,
+				"username":      username,
+				"password":      password,
+			},
+		}
+	}
+
+	return nil
+}
+
+func FlattenHDInsightsAmbariMetastore(conf map[string]*string) []interface{} {
+	server := ""
+	if v, exists := conf["database-server"]; exists && v != nil {
+		server = *v
+	}
+
+	database := ""
+	if v, exists := conf["database-name"]; exists && v != nil {
+		database = *v
+	}
+
+	username := ""
+	if v, exists := conf["database-user-name"]; exists && v != nil {
+		username = *v
+	}
+
+	password := ""
+	if v, exists := conf["database-user-password"]; exists && v != nil {
+		password = *v
+	}
+
+	if server != "" && database != "" {
+		return []interface{}{
+			map[string]interface{}{
+				"server":        server,
+				"database_name": database,
+				"username":      username,
+				"password":      password,
+			},
+		}
+	}
+
+	return nil
+}
+
 func SchemaHDInsightsStorageAccounts() *schema.Schema {
 	return &schema.Schema{
 		Type:     schema.TypeList,

azurerm/internal/services/hdinsight/common_hdinsight.go

@@ -272,3 +272,56 @@ func deleteHDInsightEdgeNodes(ctx context.Context, client *hdinsight.Application
 
 	return nil
 }
+
+func expandHDInsightsMetastore(input []interface{}) map[string]interface{} {
+	v := input[0].(map[string]interface{})
+
+	config := map[string]interface{}{}
+
+	if hiveRaw, ok := v["hive"]; ok {
+		for k, val := range azure.ExpandHDInsightsHiveMetastore(hiveRaw.([]interface{})) {
+			config[k] = val
+		}
+	}
+
+	if oozieRaw, ok := v["oozie"]; ok {
+		for k, val := range azure.ExpandHDInsightsOozieMetastore(oozieRaw.([]interface{})) {
+			config[k] = val
+		}
+	}
+
+	if ambariRaw, ok := v["ambari"]; ok {
+		for k, val := range azure.ExpandHDInsightsAmbariMetastore(ambariRaw.([]interface{})) {
+			config[k] = val
+		}
+	}
+
+	return config
+}
+
+func flattenHDInsightsMetastores(d *schema.ResourceData, configurations map[string]map[string]*string) {
+	result := map[string]interface{}{}
+
+	hiveEnv, envExists := configurations["hive-env"]
+	hiveSite, siteExists := configurations["hive-site"]
+	if envExists && siteExists {
+		result["hive"] = azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite)
+	}
+
+	oozieEnv, envExists := configurations["oozie-env"]
+	oozieSite, siteExists := configurations["oozie-site"]
+	if envExists && siteExists {
+		result["oozie"] = azure.FlattenHDInsightsOozieMetastore(oozieEnv, oozieSite)
+	}
+
+	ambari, ambariExists := configurations["ambari-conf"]
+	if ambariExists {
+		result["ambari"] = azure.FlattenHDInsightsAmbariMetastore(ambari)
+	}
+
+	if len(result) > 0 {
+		d.Set("metastores", []interface{}{
+			result,
+		})
+	}
+}

azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go

@@ -91,6 +91,21 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {
 
 			"gateway": azure.SchemaHDInsightsGateway(),
 
+			"metastores": {
+				Type:     schema.TypeList,
+				Optional: true,
+				MaxItems: 1,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"hive": azure.SchemaHDInsightsExternalMetastore(),
+
+						"oozie": azure.SchemaHDInsightsExternalMetastore(),
+
+						"ambari": azure.SchemaHDInsightsExternalMetastore(),
+					},
+				},
+			},
+
 			"storage_account": azure.SchemaHDInsightsStorageAccounts(),
 
 			"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
@@ -183,7 +198,14 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
 	componentVersions := expandHDInsightHadoopComponentVersion(componentVersionsRaw)
 
 	gatewayRaw := d.Get("gateway").([]interface{})
-	gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
+	configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)
+
+	if metastoresRaw, ok := d.GetOkExists("metastores"); ok {
+		metastores := expandHDInsightsMetastore(metastoresRaw.([]interface{}))
+		for k, v := range metastores {
+			configurations[k] = v
+		}
+	}
 
 	storageAccountsRaw := d.Get("storage_account").([]interface{})
 	storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
@@ -225,7 +247,7 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
 			ClusterDefinition: &hdinsight.ClusterDefinition{
 				Kind:             utils.String("Hadoop"),
 				ComponentVersion: componentVersions,
-				Configurations:   gateway,
+				Configurations:   configurations,
 			},
 			StorageProfile: &hdinsight.StorageProfile{
 				Storageaccounts: storageAccounts,
@@ -311,11 +333,17 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
 		return fmt.Errorf("Error retrieving HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
 	}
 
-	configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
+	// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
+	configurations, err := configurationsClient.List(ctx, resourceGroup, name)
 	if err != nil {
 		return fmt.Errorf("Error retrieving Configuration for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
 	}
 
+	gateway, exists := configurations.Configurations["gateway"]
+	if !exists {
+		return fmt.Errorf("Error retrieving gateway for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
+	}
+
 	d.Set("name", name)
 	d.Set("resource_group_name", resourceGroup)
 	if location := resp.Location; location != nil {
@@ -332,9 +360,11 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
 				return fmt.Errorf("Error flattening `component_version`: %+v", err)
 			}
 
-			if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
+			if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
 				return fmt.Errorf("Error flattening `gateway`: %+v", err)
 			}
+
+			flattenHDInsightsMetastores(d, configurations.Configurations)
 		}
 
 		hadoopRoles := hdInsightRoleDefinition{