From 5525a83600d22da62050990943ea8864294d0710 Mon Sep 17 00:00:00 2001 From: Konstantin Kosinsky Date: Tue, 17 Mar 2020 11:09:07 -0700 Subject: [PATCH 01/10] Support for external Hive metastore --- azurerm/helpers/azure/hdinsight.go | 104 +++++++++++++++- .../resource_arm_hdinsight_hadoop_cluster.go | 29 ++++- ...ource_arm_hdinsight_hadoop_cluster_test.go | 113 ++++++++++++++++++ .../r/hdinsight_hadoop_cluster.html.markdown | 13 ++ 4 files changed, 254 insertions(+), 5 deletions(-) diff --git a/azurerm/helpers/azure/hdinsight.go b/azurerm/helpers/azure/hdinsight.go index a61d8bd55af3..a48cb5bf7092 100644 --- a/azurerm/helpers/azure/hdinsight.go +++ b/azurerm/helpers/azure/hdinsight.go @@ -105,6 +105,43 @@ func SchemaHDInsightsGateway() *schema.Schema { } } +func SchemaHDInsightsHiveMetastore() *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "server": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "database_name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "username": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "password": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Sensitive: true, + // Azure returns the key as *****. We'll suppress that here. + DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { + return (new == d.Get(k).(string)) && (old == "*****") + }, + }, + }, + }, + } +} + func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} { vs := input[0].(map[string]interface{}) @@ -113,13 +150,41 @@ func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} username := vs["username"].(string) password := vs["password"].(string) - return map[string]interface{}{ + config := map[string]interface{}{ "gateway": map[string]interface{}{ "restAuthCredential.isEnabled": enabled, "restAuthCredential.username": username, "restAuthCredential.password": password, }, } + return config +} + +func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} { + vs := input[0].(map[string]interface{}) + + server := vs["server"].(string) + database := vs["database_name"].(string) + username := vs["username"].(string) + password := vs["password"].(string) + + return map[string]interface{}{ + "hive-site": map[string]interface{}{ + "javax.jdo.option.ConnectionDriverName": "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "javax.jdo.option.ConnectionURL": fmt.Sprintf("jdbc:sqlserver://%s;database=%s;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", server, database), + "javax.jdo.option.ConnectionUserName": username, + "javax.jdo.option.ConnectionPassword": password, + }, + "hive-env": map[string]interface{}{ + "hive_database": "Existing MSSQL Server database with SQL authentication", + "hive_database_name": database, + "hive_database_type": "mssql", + "hive_existing_mssql_server_database": database, + "hive_existing_mssql_server_host": server, + "hive_hostname": server, + }, + } + } func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} { @@ -150,6 +215,43 @@ func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} { } } +func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*string) []interface{} { + + server := "" + if v, exists := env["hive_hostname"]; exists && v != nil { + server = *v + } + + database := "" + if v, exists := env["hive_database_name"]; exists && v != nil { + database = *v + } + + username := "" + if v, exists := site["javax.jdo.option.ConnectionUserName"]; exists && v != nil { + username = *v + } + + password := "" + if v, exists := site["javax.jdo.option.ConnectionPassword"]; exists && v != nil { + password = *v + } + + if server != "" && database != "" { + + return []interface{}{ + map[string]interface{}{ + "server": server, + "database_name": database, + "username": username, + "password": password, + }, + } + } else { + return nil + } +} + func SchemaHDInsightsStorageAccounts() *schema.Schema { return &schema.Schema{ Type: schema.TypeList, diff --git a/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go b/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go index e9623d8ce35e..53f07d58332d 100644 --- a/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go +++ b/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go @@ -91,6 +91,8 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource { "gateway": azure.SchemaHDInsightsGateway(), + "hive_metastore": azure.SchemaHDInsightsHiveMetastore(), + "storage_account": azure.SchemaHDInsightsStorageAccounts(), "storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(), @@ -183,7 +185,14 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf componentVersions := expandHDInsightHadoopComponentVersion(componentVersionsRaw) gatewayRaw := d.Get("gateway").([]interface{}) - gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw) + configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw) + + if metastoreRaw, ok := d.GetOkExists("hive_metastore"); ok { + metastore := azure.ExpandHDInsightsMetastore(metastoreRaw.([]interface{})) + for k, v := range metastore { + configurations[k] = v + } + } storageAccountsRaw := d.Get("storage_account").([]interface{}) storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{}) @@ -225,7 +234,7 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf ClusterDefinition: &hdinsight.ClusterDefinition{ Kind: utils.String("Hadoop"), ComponentVersion: componentVersions, - Configurations: gateway, + Configurations: configurations, }, StorageProfile: &hdinsight.StorageProfile{ Storageaccounts: storageAccounts, @@ -311,11 +320,17 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac return fmt.Errorf("Error retrieving HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err) } - configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway") + // Each call to configurationsClient methods is HTTP request. Getting all settings in one operation + configurations, err := configurationsClient.List(ctx, resourceGroup, name) if err != nil { return fmt.Errorf("Error retrieving Configuration for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err) } + gateway, exists := configurations.Configurations["gateway"] + if !exists { + return fmt.Errorf("Error retrieving gateway for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err) + } + d.Set("name", name) d.Set("resource_group_name", resourceGroup) if location := resp.Location; location != nil { @@ -332,9 +347,15 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac return fmt.Errorf("Error flattening `component_version`: %+v", err) } - if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil { + if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil { return fmt.Errorf("Error flattening `gateway`: %+v", err) } + + hiveEnv, envExists := configurations.Configurations["hive-env"] + hiveSite, siteExists := configurations.Configurations["hive-site"] + if envExists && siteExists { + d.Set("hive_metastore", azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite)) + } } hadoopRoles := hdInsightRoleDefinition{ diff --git a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go index 6ed62031cf9c..e6d55c99166d 100644 --- a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go +++ b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go @@ -319,6 +319,33 @@ func TestAccAzureRMHDInsightHadoopCluster_gen2AndBlobStorage(t *testing.T) { }) } +func TestAccAzureRMHDInsightHadoopCluster_metastore(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_hdinsight_hadoop_cluster", "test") + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acceptance.PreCheck(t) }, + Providers: acceptance.SupportedProviders, + CheckDestroy: testCheckAzureRMHDInsightClusterDestroy(data.ResourceType), + Steps: []resource.TestStep{ + { + Config: testAccAzureRMHDInsightHadoopCluster_metastore(data), + Check: resource.ComposeTestCheckFunc( + testCheckAzureRMHDInsightClusterExists(data.ResourceName), + resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"), + resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"), + ), + }, + data.ImportStep("roles.0.head_node.0.password", + "roles.0.head_node.0.vm_size", + "roles.0.worker_node.0.password", + "roles.0.worker_node.0.vm_size", + "roles.0.zookeeper_node.0.password", + "roles.0.zookeeper_node.0.vm_size", + "storage_account", + "hive_metastore.0.password"), + }, + }) +} + func testAccAzureRMHDInsightHadoopCluster_basic(data acceptance.TestData) string { template := testAccAzureRMHDInsightHadoopCluster_template(data) return fmt.Sprintf(` @@ -974,3 +1001,89 @@ resource "azurerm_role_assignment" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomString) } + +func testAccAzureRMHDInsightHadoopCluster_metastore(data acceptance.TestData) string { + template := testAccAzureRMHDInsightHadoopCluster_template(data) + return fmt.Sprintf(` +%s + +resource "azurerm_sql_server" "test"{ + name = "acctestsql-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + administrator_login = "sql_admin" + administrator_login_password = "TerrAform123!" + version = "12.0" +} + +resource "azurerm_sql_database" "test" { + name = "metastore" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + server_name = azurerm_sql_server.test.name + collation = "SQL_Latin1_General_CP1_CI_AS" + create_mode = "Default" + requested_service_objective_name = "GP_Gen5_2" +} + +resource "azurerm_sql_firewall_rule" "AzureServices" { + name = "allow-azure-services" + resource_group_name = azurerm_resource_group.test.name + server_name = azurerm_sql_server.test.name + start_ip_address = "0.0.0.0" + end_ip_address = "0.0.0.0" +} + +resource "azurerm_hdinsight_hadoop_cluster" "test" { + name = "acctesthdi-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + cluster_version = "3.6" + tier = "Standard" + + component_version { + hadoop = "2.7" + } + + gateway { + enabled = true + username = "acctestusrgw" + password = "TerrAform123!" + } + + storage_account { + storage_container_id = azurerm_storage_container.test.id + storage_account_key = azurerm_storage_account.test.primary_access_key + is_default = true + } + + roles { + head_node { + vm_size = "Standard_D3_v2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + } + + worker_node { + vm_size = "Standard_D4_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + target_instance_count = 2 + } + + zookeeper_node { + vm_size = "Standard_D3_v2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + } + } + + hive_metastore { + server = azurerm_sql_server.test.fully_qualified_domain_name + database_name = azurerm_sql_database.test.name + username = azurerm_sql_server.test.administrator_login + password = azurerm_sql_server.test.administrator_login_password + } +} +`, template, data.RandomInteger, data.RandomInteger) +} diff --git a/website/docs/r/hdinsight_hadoop_cluster.html.markdown b/website/docs/r/hdinsight_hadoop_cluster.html.markdown index 22b4104afa96..cf96c365a768 100644 --- a/website/docs/r/hdinsight_hadoop_cluster.html.markdown +++ b/website/docs/r/hdinsight_hadoop_cluster.html.markdown @@ -107,6 +107,8 @@ The following arguments are supported: * `tags` - (Optional) A map of Tags which should be assigned to this HDInsight Hadoop Cluster. +* `hive_metastore` - (Optional) A `hive_metastore` block as defined below. + --- A `component_version` block supports the following: @@ -247,6 +249,17 @@ A `install_script_action` block supports the following: * `uri` - (Required) The URI pointing to the script to run during the installation of the edge node. Changing this forces a new resource to be created. +--- +A `hive_metastore` block supports the following: + +* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Hive metastore. + +* `database_name` - (Required) The external Hive metastore's existing SQL database. + +* `username` - (Required) The external Hive metastore's existing SQL server admin username. + +* `password` - (Required) The external Hive metastore's existing SQL server admin password. + ## Attributes Reference The following attributes are exported: From b2309425710144dc087dad92b20af1415ccf5162 Mon Sep 17 00:00:00 2001 From: Konstantin Kosinsky Date: Tue, 17 Mar 2020 11:37:25 -0700 Subject: [PATCH 02/10] Fix formatting --- azurerm/helpers/azure/hdinsight.go | 10 +++------- ...resource_arm_hdinsight_hadoop_cluster_test.go | 16 ++++++++-------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/azurerm/helpers/azure/hdinsight.go b/azurerm/helpers/azure/hdinsight.go index a48cb5bf7092..2863a61fca48 100644 --- a/azurerm/helpers/azure/hdinsight.go +++ b/azurerm/helpers/azure/hdinsight.go @@ -150,14 +150,13 @@ func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} username := vs["username"].(string) password := vs["password"].(string) - config := map[string]interface{}{ + return map[string]interface{}{ "gateway": map[string]interface{}{ "restAuthCredential.isEnabled": enabled, "restAuthCredential.username": username, "restAuthCredential.password": password, }, } - return config } func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} { @@ -184,7 +183,6 @@ func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} { "hive_hostname": server, }, } - } func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} { @@ -216,7 +214,6 @@ func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} { } func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*string) []interface{} { - server := "" if v, exists := env["hive_hostname"]; exists && v != nil { server = *v @@ -238,7 +235,6 @@ func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*str } if server != "" && database != "" { - return []interface{}{ map[string]interface{}{ "server": server, @@ -247,9 +243,9 @@ func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*str "password": password, }, } - } else { - return nil } + + return nil } func SchemaHDInsightsStorageAccounts() *schema.Schema { diff --git a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go index e6d55c99166d..21be14a5b223 100644 --- a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go +++ b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go @@ -1007,13 +1007,13 @@ func testAccAzureRMHDInsightHadoopCluster_metastore(data acceptance.TestData) st return fmt.Sprintf(` %s -resource "azurerm_sql_server" "test"{ - name = "acctestsql-%d" - resource_group_name = azurerm_resource_group.test.name - location = azurerm_resource_group.test.location - administrator_login = "sql_admin" - administrator_login_password = "TerrAform123!" - version = "12.0" +resource "azurerm_sql_server" "test" { + name = "acctestsql-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + administrator_login = "sql_admin" + administrator_login_password = "TerrAform123!" + version = "12.0" } resource "azurerm_sql_database" "test" { @@ -1079,7 +1079,7 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" { } hive_metastore { - server = azurerm_sql_server.test.fully_qualified_domain_name + server = azurerm_sql_server.test.fully_qualified_domain_name database_name = azurerm_sql_database.test.name username = azurerm_sql_server.test.administrator_login password = azurerm_sql_server.test.administrator_login_password From ce341b97fe95786d4812849b5e6c7ad5a6c18ee0 Mon Sep 17 00:00:00 2001 From: Konstantin Kosinsky Date: Mon, 23 Mar 2020 13:45:02 -0700 Subject: [PATCH 03/10] Support for Oozie and Ambari --- azurerm/helpers/azure/hdinsight.go | 119 +++++++++++++++++- .../services/hdinsight/common_hdinsight.go | 53 ++++++++ .../resource_arm_hdinsight_hadoop_cluster.go | 30 +++-- ...ource_arm_hdinsight_hadoop_cluster_test.go | 54 ++++++-- .../r/hdinsight_hadoop_cluster.html.markdown | 41 +++++- 5 files changed, 274 insertions(+), 23 deletions(-) diff --git a/azurerm/helpers/azure/hdinsight.go b/azurerm/helpers/azure/hdinsight.go index 2863a61fca48..aa4e4c567b49 100644 --- a/azurerm/helpers/azure/hdinsight.go +++ b/azurerm/helpers/azure/hdinsight.go @@ -105,7 +105,7 @@ func SchemaHDInsightsGateway() *schema.Schema { } } -func SchemaHDInsightsHiveMetastore() *schema.Schema { +func SchemaHDInsightsExternalMetastore() *schema.Schema { return &schema.Schema{ Type: schema.TypeList, Optional: true, @@ -159,7 +159,7 @@ func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} } } -func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} { +func ExpandHDInsightsHiveMetastore(input []interface{}) map[string]interface{} { vs := input[0].(map[string]interface{}) server := vs["server"].(string) @@ -185,6 +185,51 @@ func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} { } } +func ExpandHDInsightsOozieMetastore(input []interface{}) map[string]interface{} { + vs := input[0].(map[string]interface{}) + + server := vs["server"].(string) + database := vs["database_name"].(string) + username := vs["username"].(string) + password := vs["password"].(string) + + return map[string]interface{}{ + "oozie-site": map[string]interface{}{ + "oozie.service.JPAService.jdbc.driver": "com.microsoft.sqlserver.jdbc.SQLServerDriver", + "oozie.service.JPAService.jdbc.url": fmt.Sprintf("jdbc:sqlserver://%s;database=%s;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", server, database), + "oozie.service.JPAService.jdbc.username": username, + "oozie.service.JPAService.jdbc.password": password, + "oozie.db.schema.name": "oozie", + }, + "oozie-env": map[string]interface{}{ + "oozie_database": "Existing MSSQL Server database with SQL authentication", + "oozie_database_name": database, + "oozie_database_type": "mssql", + "oozie_existing_mssql_server_database": database, + "oozie_existing_mssql_server_host": server, + "oozie_hostname": server, + }, + } +} + +func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{} { + vs := input[0].(map[string]interface{}) + + server := vs["server"].(string) + database := vs["database_name"].(string) + username := vs["username"].(string) + password := vs["password"].(string) + + return map[string]interface{}{ + "ambari-conf": map[string]interface{}{ + "database-server": server, + "database-name": database, + "database-user-name": username, + "database-user-password": password, + }, + } +} + func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} { enabled := false if v, exists := input["restAuthCredential.isEnabled"]; exists && v != nil { @@ -248,6 +293,76 @@ func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*str return nil } +func FlattenHDInsightsOozieMetastore(env map[string]*string, site map[string]*string) []interface{} { + server := "" + if v, exists := env["oozie_hostname"]; exists && v != nil { + server = *v + } + + database := "" + if v, exists := env["oozie_database_name"]; exists && v != nil { + database = *v + } + + username := "" + if v, exists := site["oozie.service.JPAService.jdbc.username"]; exists && v != nil { + username = *v + } + + password := "" + if v, exists := site["oozie.service.JPAService.jdbc.password"]; exists && v != nil { + password = *v + } + + if server != "" && database != "" { + return []interface{}{ + map[string]interface{}{ + "server": server, + "database_name": database, + "username": username, + "password": password, + }, + } + } + + return nil +} + +func FlattenHDInsightsAmbariMetastore(conf map[string]*string) []interface{} { + server := "" + if v, exists := conf["database-server"]; exists && v != nil { + server = *v + } + + database := "" + if v, exists := conf["database-name"]; exists && v != nil { + database = *v + } + + username := "" + if v, exists := conf["database-user-name"]; exists && v != nil { + username = *v + } + + password := "" + if v, exists := conf["database-user-password"]; exists && v != nil { + password = *v + } + + if server != "" && database != "" { + return []interface{}{ + map[string]interface{}{ + "server": server, + "database_name": database, + "username": username, + "password": password, + }, + } + } + + return nil +} + func SchemaHDInsightsStorageAccounts() *schema.Schema { return &schema.Schema{ Type: schema.TypeList, diff --git a/azurerm/internal/services/hdinsight/common_hdinsight.go b/azurerm/internal/services/hdinsight/common_hdinsight.go index 98aed1c185cb..265c68e56dc0 100644 --- a/azurerm/internal/services/hdinsight/common_hdinsight.go +++ b/azurerm/internal/services/hdinsight/common_hdinsight.go @@ -272,3 +272,56 @@ func deleteHDInsightEdgeNodes(ctx context.Context, client *hdinsight.Application return nil } + +func expandHDInsightsMetastore(input []interface{}) map[string]interface{} { + v := input[0].(map[string]interface{}) + + config := map[string]interface{}{} + + if hiveRaw, ok := v["hive"]; ok { + for k, val := range azure.ExpandHDInsightsHiveMetastore(hiveRaw.([]interface{})) { + config[k] = val + } + } + + if oozieRaw, ok := v["oozie"]; ok { + for k, val := range azure.ExpandHDInsightsOozieMetastore(oozieRaw.([]interface{})) { + config[k] = val + } + } + + if ambariRaw, ok := v["ambari"]; ok { + for k, val := range azure.ExpandHDInsightsAmbariMetastore(ambariRaw.([]interface{})) { + config[k] = val + } + } + + return config +} + +func flattenHDInsightsMetastores(d *schema.ResourceData, configurations map[string]map[string]*string) { + result := map[string]interface{}{} + + hiveEnv, envExists := configurations["hive-env"] + hiveSite, siteExists := configurations["hive-site"] + if envExists && siteExists { + result["hive"] = azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite) + } + + oozieEnv, envExists := configurations["oozie-env"] + oozieSite, siteExists := configurations["oozie-site"] + if envExists && siteExists { + result["oozie"] = azure.FlattenHDInsightsOozieMetastore(oozieEnv, oozieSite) + } + + ambari, ambariExists := configurations["ambari-conf"] + if ambariExists { + result["ambari"] = azure.FlattenHDInsightsAmbariMetastore(ambari) + } + + if len(result) > 0 { + d.Set("metastores", []interface{}{ + result, + }) + } +} diff --git a/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go b/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go index 53f07d58332d..d1c0802be35e 100644 --- a/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go +++ b/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go @@ -91,7 +91,20 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource { "gateway": azure.SchemaHDInsightsGateway(), - "hive_metastore": azure.SchemaHDInsightsHiveMetastore(), + "metastores": { + Type: schema.TypeList, + Required: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "hive": azure.SchemaHDInsightsExternalMetastore(), + + "oozie": azure.SchemaHDInsightsExternalMetastore(), + + "ambari": azure.SchemaHDInsightsExternalMetastore(), + }, + }, + }, "storage_account": azure.SchemaHDInsightsStorageAccounts(), @@ -187,11 +200,10 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf gatewayRaw := d.Get("gateway").([]interface{}) configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw) - if metastoreRaw, ok := d.GetOkExists("hive_metastore"); ok { - metastore := azure.ExpandHDInsightsMetastore(metastoreRaw.([]interface{})) - for k, v := range metastore { - configurations[k] = v - } + metastoresRaw := d.Get("metastores").([]interface{}) + metastores := expandHDInsightsMetastore(metastoresRaw) + for k, v := range metastores { + configurations[k] = v } storageAccountsRaw := d.Get("storage_account").([]interface{}) @@ -351,11 +363,7 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac return fmt.Errorf("Error flattening `gateway`: %+v", err) } - hiveEnv, envExists := configurations.Configurations["hive-env"] - hiveSite, siteExists := configurations.Configurations["hive-site"] - if envExists && siteExists { - d.Set("hive_metastore", azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite)) - } + flattenHDInsightsMetastores(d, configurations.Configurations) } hadoopRoles := hdInsightRoleDefinition{ diff --git a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go index 21be14a5b223..205b4259cc85 100644 --- a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go +++ b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go @@ -341,7 +341,9 @@ func TestAccAzureRMHDInsightHadoopCluster_metastore(t *testing.T) { "roles.0.zookeeper_node.0.password", "roles.0.zookeeper_node.0.vm_size", "storage_account", - "hive_metastore.0.password"), + "metastores.0.hive.0.password", + "metastores.0.oozie.0.password", + "metastores.0.ambari.0.password"), }, }) } @@ -1016,8 +1018,28 @@ resource "azurerm_sql_server" "test" { version = "12.0" } -resource "azurerm_sql_database" "test" { - name = "metastore" +resource "azurerm_sql_database" "hive" { + name = "hive" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + server_name = azurerm_sql_server.test.name + collation = "SQL_Latin1_General_CP1_CI_AS" + create_mode = "Default" + requested_service_objective_name = "GP_Gen5_2" +} + +resource "azurerm_sql_database" "oozie" { + name = "oozie" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + server_name = azurerm_sql_server.test.name + collation = "SQL_Latin1_General_CP1_CI_AS" + create_mode = "Default" + requested_service_objective_name = "GP_Gen5_2" +} + +resource "azurerm_sql_database" "ambari" { + name = "ambari" resource_group_name = azurerm_resource_group.test.name location = azurerm_resource_group.test.location server_name = azurerm_sql_server.test.name @@ -1078,11 +1100,27 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" { } } - hive_metastore { - server = azurerm_sql_server.test.fully_qualified_domain_name - database_name = azurerm_sql_database.test.name - username = azurerm_sql_server.test.administrator_login - password = azurerm_sql_server.test.administrator_login_password + metastores { + hive { + server = azurerm_sql_server.test.fully_qualified_domain_name + database_name = azurerm_sql_database.hive.name + username = azurerm_sql_server.test.administrator_login + password = azurerm_sql_server.test.administrator_login_password + } + + oozie { + server = azurerm_sql_server.test.fully_qualified_domain_name + database_name = azurerm_sql_database.oozie.name + username = azurerm_sql_server.test.administrator_login + password = azurerm_sql_server.test.administrator_login_password + } + + ambari { + server = azurerm_sql_server.test.fully_qualified_domain_name + database_name = azurerm_sql_database.ambari.name + username = azurerm_sql_server.test.administrator_login + password = azurerm_sql_server.test.administrator_login_password + } } } `, template, data.RandomInteger, data.RandomInteger) diff --git a/website/docs/r/hdinsight_hadoop_cluster.html.markdown b/website/docs/r/hdinsight_hadoop_cluster.html.markdown index cf96c365a768..9ebcd93d9e2c 100644 --- a/website/docs/r/hdinsight_hadoop_cluster.html.markdown +++ b/website/docs/r/hdinsight_hadoop_cluster.html.markdown @@ -107,7 +107,7 @@ The following arguments are supported: * `tags` - (Optional) A map of Tags which should be assigned to this HDInsight Hadoop Cluster. -* `hive_metastore` - (Optional) A `hive_metastore` block as defined below. +* `metastores` - (Optional) A `metastores` block as defined below. --- @@ -249,8 +249,19 @@ A `install_script_action` block supports the following: * `uri` - (Required) The URI pointing to the script to run during the installation of the edge node. Changing this forces a new resource to be created. +--- + +A `metastores` block supports the following: + +* `hive` - (Required) A `hive` block as defined below. + +* `oozie` - (Required) A `oozie` block as defined below. + +* `ambari` - (Required) A `amabari` block as defined below. + --- -A `hive_metastore` block supports the following: + +A `hive` block supports the following: * `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Hive metastore. @@ -260,6 +271,32 @@ A `hive_metastore` block supports the following: * `password` - (Required) The external Hive metastore's existing SQL server admin password. + +--- + +A `oozie` block supports the following: + +* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Oozie metastore. + +* `database_name` - (Required) The external Oozie metastore's existing SQL database. + +* `username` - (Required) The external Oozie metastore's existing SQL server admin username. + +* `password` - (Required) The external Oozie metastore's existing SQL server admin password. + +--- + +A `ambari` block supports the following: + +* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Ambari metastore. + +* `database_name` - (Required) The external Hive metastore's existing SQL database. + +* `username` - (Required) The external Ambari metastore's existing SQL server admin username. + +* `password` - (Required) The external Ambari metastore's existing SQL server admin password. + + ## Attributes Reference The following attributes are exported: From 56d4233fdd3870247e97e7ed2443ae907cb8900f Mon Sep 17 00:00:00 2001 From: Konstantin Kosinsky Date: Tue, 24 Mar 2020 04:46:16 -0700 Subject: [PATCH 04/10] Make metastores optional --- .../resource_arm_hdinsight_hadoop_cluster.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go b/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go index d1c0802be35e..fffecc245347 100644 --- a/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go +++ b/azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go @@ -93,7 +93,7 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource { "metastores": { Type: schema.TypeList, - Required: true, + Optional: true, MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -200,10 +200,11 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf gatewayRaw := d.Get("gateway").([]interface{}) configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw) - metastoresRaw := d.Get("metastores").([]interface{}) - metastores := expandHDInsightsMetastore(metastoresRaw) - for k, v := range metastores { - configurations[k] = v + if metastoresRaw, ok := d.GetOkExists("metastores"); ok { + metastores := expandHDInsightsMetastore(metastoresRaw.([]interface{})) + for k, v := range metastores { + configurations[k] = v + } } storageAccountsRaw := d.Get("storage_account").([]interface{}) From 2d6b93ffa88f7ae44b7777e756dbdcfb1ad1e6f4 Mon Sep 17 00:00:00 2001 From: Konstantin Kosinsky Date: Wed, 25 Mar 2020 16:52:34 -0700 Subject: [PATCH 05/10] Fix AzureRMHDInsightHadoopCluster_requiresImport --- azurerm/helpers/azure/hdinsight.go | 2 +- .../tests/resource_arm_hdinsight_hadoop_cluster_test.go | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/azurerm/helpers/azure/hdinsight.go b/azurerm/helpers/azure/hdinsight.go index aa4e4c567b49..53c0b1acfe94 100644 --- a/azurerm/helpers/azure/hdinsight.go +++ b/azurerm/helpers/azure/hdinsight.go @@ -39,7 +39,7 @@ func SchemaHDInsightTier() *schema.Schema { ValidateFunc: validation.StringInSlice([]string{ string(hdinsight.Standard), string(hdinsight.Premium), - }, false), + }, true), // TODO: file a bug about this DiffSuppressFunc: SuppressLocationDiff, } diff --git a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go index 205b4259cc85..65b153ba2a28 100644 --- a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go +++ b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go @@ -456,7 +456,6 @@ resource "azurerm_hdinsight_hadoop_cluster" "import" { for_each = lookup(roles.value, "head_node", []) content { password = lookup(head_node.value, "password", null) - ssh_keys = lookup(head_node.value, "ssh_keys", null) subnet_id = lookup(head_node.value, "subnet_id", null) username = head_node.value.username virtual_network_id = lookup(head_node.value, "virtual_network_id", null) @@ -467,9 +466,7 @@ resource "azurerm_hdinsight_hadoop_cluster" "import" { dynamic "worker_node" { for_each = lookup(roles.value, "worker_node", []) content { - min_instance_count = lookup(worker_node.value, "min_instance_count", null) password = lookup(worker_node.value, "password", null) - ssh_keys = lookup(worker_node.value, "ssh_keys", null) subnet_id = lookup(worker_node.value, "subnet_id", null) target_instance_count = worker_node.value.target_instance_count username = worker_node.value.username @@ -482,7 +479,6 @@ resource "azurerm_hdinsight_hadoop_cluster" "import" { for_each = lookup(roles.value, "zookeeper_node", []) content { password = lookup(zookeeper_node.value, "password", null) - ssh_keys = lookup(zookeeper_node.value, "ssh_keys", null) subnet_id = lookup(zookeeper_node.value, "subnet_id", null) username = zookeeper_node.value.username virtual_network_id = lookup(zookeeper_node.value, "virtual_network_id", null) From 713ccd34ae74dcf323c1418f8fc1f4c390554162 Mon Sep 17 00:00:00 2001 From: Konstantin Kosinsky Date: Mon, 30 Mar 2020 07:22:18 -0700 Subject: [PATCH 06/10] Fix doc --- website/docs/r/hdinsight_hadoop_cluster.html.markdown | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/docs/r/hdinsight_hadoop_cluster.html.markdown b/website/docs/r/hdinsight_hadoop_cluster.html.markdown index 9ebcd93d9e2c..321dc78963d0 100644 --- a/website/docs/r/hdinsight_hadoop_cluster.html.markdown +++ b/website/docs/r/hdinsight_hadoop_cluster.html.markdown @@ -253,11 +253,11 @@ A `install_script_action` block supports the following: A `metastores` block supports the following: -* `hive` - (Required) A `hive` block as defined below. +* `hive` - (Optional) A `hive` block as defined below. -* `oozie` - (Required) A `oozie` block as defined below. +* `oozie` - (Optional) A `oozie` block as defined below. -* `ambari` - (Required) A `amabari` block as defined below. +* `ambari` - (Optional) A `amabari` block as defined below. --- From 0b228b555d649e3a579d9a1a2267a7194f846efd Mon Sep 17 00:00:00 2001 From: kosinsky Date: Fri, 24 Apr 2020 07:49:57 -0700 Subject: [PATCH 07/10] Fix typo Co-Authored-By: Steve <11830746+jackofallops@users.noreply.github.com> --- website/docs/r/hdinsight_hadoop_cluster.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/r/hdinsight_hadoop_cluster.html.markdown b/website/docs/r/hdinsight_hadoop_cluster.html.markdown index 6513a1133107..bc7e707932a2 100644 --- a/website/docs/r/hdinsight_hadoop_cluster.html.markdown +++ b/website/docs/r/hdinsight_hadoop_cluster.html.markdown @@ -261,7 +261,7 @@ A `metastores` block supports the following: * `oozie` - (Optional) A `oozie` block as defined below. -* `ambari` - (Optional) A `amabari` block as defined below. +* `ambari` - (Optional) A `ambari` block as defined below. --- From 72652caf5e27e0aa5e135fd7dfa25f0ad017595f Mon Sep 17 00:00:00 2001 From: Konstantin Kosinsky Date: Fri, 24 Apr 2020 09:55:24 -0700 Subject: [PATCH 08/10] Added test for hive only and check in reading code --- azurerm/helpers/azure/hdinsight.go | 9 ++ ...ource_arm_hdinsight_hadoop_cluster_test.go | 106 ++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/azurerm/helpers/azure/hdinsight.go b/azurerm/helpers/azure/hdinsight.go index 4bef5695452c..ca129004101e 100644 --- a/azurerm/helpers/azure/hdinsight.go +++ b/azurerm/helpers/azure/hdinsight.go @@ -174,6 +174,9 @@ func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} } func ExpandHDInsightsHiveMetastore(input []interface{}) map[string]interface{} { + if len(input) == 0 { + return nil + } vs := input[0].(map[string]interface{}) server := vs["server"].(string) @@ -200,6 +203,9 @@ func ExpandHDInsightsHiveMetastore(input []interface{}) map[string]interface{} { } func ExpandHDInsightsOozieMetastore(input []interface{}) map[string]interface{} { + if len(input) == 0 { + return nil + } vs := input[0].(map[string]interface{}) server := vs["server"].(string) @@ -227,6 +233,9 @@ func ExpandHDInsightsOozieMetastore(input []interface{}) map[string]interface{} } func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{} { + if len(input) == 0 { + return nil + } vs := input[0].(map[string]interface{}) server := vs["server"].(string) diff --git a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go index 6b07d2d6b1e5..d3ee75312882 100644 --- a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go +++ b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go @@ -374,6 +374,35 @@ func TestAccAzureRMHDInsightHadoopCluster_metastore(t *testing.T) { }) } +func TestAccAzureRMHDInsightHadoopCluster_hiveMetastore(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_hdinsight_hadoop_cluster", "test") + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acceptance.PreCheck(t) }, + Providers: acceptance.SupportedProviders, + CheckDestroy: testCheckAzureRMHDInsightClusterDestroy(data.ResourceType), + Steps: []resource.TestStep{ + { + Config: testAccAzureRMHDInsightHadoopCluster_hiveMetastore(data), + Check: resource.ComposeTestCheckFunc( + testCheckAzureRMHDInsightClusterExists(data.ResourceName), + resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"), + resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"), + ), + }, + data.ImportStep("roles.0.head_node.0.password", + "roles.0.head_node.0.vm_size", + "roles.0.worker_node.0.password", + "roles.0.worker_node.0.vm_size", + "roles.0.zookeeper_node.0.password", + "roles.0.zookeeper_node.0.vm_size", + "storage_account", + "metastores.0.hive.0.password", + "metastores.0.oozie.0.password", + "metastores.0.ambari.0.password"), + }, + }) +} + func testAccAzureRMHDInsightHadoopCluster_basic(data acceptance.TestData) string { template := testAccAzureRMHDInsightHadoopCluster_template(data) return fmt.Sprintf(` @@ -1178,3 +1207,80 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" { } `, template, data.RandomInteger, data.RandomInteger) } + +func testAccAzureRMHDInsightHadoopCluster_hiveMetastore(data acceptance.TestData) string { + template := testAccAzureRMHDInsightHadoopCluster_template(data) + return fmt.Sprintf(` +%s +resource "azurerm_sql_server" "test" { + name = "acctestsql-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + administrator_login = "sql_admin" + administrator_login_password = "TerrAform123!" + version = "12.0" +} +resource "azurerm_sql_database" "hive" { + name = "hive" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + server_name = azurerm_sql_server.test.name + collation = "SQL_Latin1_General_CP1_CI_AS" + create_mode = "Default" + requested_service_objective_name = "GP_Gen5_2" +} +resource "azurerm_sql_firewall_rule" "AzureServices" { + name = "allow-azure-services" + resource_group_name = azurerm_resource_group.test.name + server_name = azurerm_sql_server.test.name + start_ip_address = "0.0.0.0" + end_ip_address = "0.0.0.0" +} +resource "azurerm_hdinsight_hadoop_cluster" "test" { + name = "acctesthdi-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + cluster_version = "3.6" + tier = "Standard" + component_version { + hadoop = "2.7" + } + gateway { + enabled = true + username = "acctestusrgw" + password = "TerrAform123!" + } + storage_account { + storage_container_id = azurerm_storage_container.test.id + storage_account_key = azurerm_storage_account.test.primary_access_key + is_default = true + } + roles { + head_node { + vm_size = "Standard_D3_v2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + } + worker_node { + vm_size = "Standard_D4_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + target_instance_count = 2 + } + zookeeper_node { + vm_size = "Standard_D3_v2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + } + } + metastores { + hive { + server = azurerm_sql_server.test.fully_qualified_domain_name + database_name = azurerm_sql_database.hive.name + username = azurerm_sql_server.test.administrator_login + password = azurerm_sql_server.test.administrator_login_password + } + } +} +`, template, data.RandomInteger, data.RandomInteger) +} From 189a4599e83e233afb59505f0a458bbf80285a6a Mon Sep 17 00:00:00 2001 From: Konstantin Kosinsky Date: Fri, 24 Apr 2020 17:54:18 -0700 Subject: [PATCH 09/10] Update tests --- ...ource_arm_hdinsight_hadoop_cluster_test.go | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go index d3ee75312882..313981e4b5c2 100644 --- a/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go +++ b/azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go @@ -345,7 +345,7 @@ func TestAccAzureRMHDInsightHadoopCluster_tls(t *testing.T) { }) } -func TestAccAzureRMHDInsightHadoopCluster_metastore(t *testing.T) { +func TestAccAzureRMHDInsightHadoopCluster_allMetastores(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_hdinsight_hadoop_cluster", "test") resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { acceptance.PreCheck(t) }, @@ -353,7 +353,7 @@ func TestAccAzureRMHDInsightHadoopCluster_metastore(t *testing.T) { CheckDestroy: testCheckAzureRMHDInsightClusterDestroy(data.ResourceType), Steps: []resource.TestStep{ { - Config: testAccAzureRMHDInsightHadoopCluster_metastore(data), + Config: testAccAzureRMHDInsightHadoopCluster_allMetastores(data), Check: resource.ComposeTestCheckFunc( testCheckAzureRMHDInsightClusterExists(data.ResourceName), resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"), @@ -389,6 +389,43 @@ func TestAccAzureRMHDInsightHadoopCluster_hiveMetastore(t *testing.T) { resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"), ), }, + }, + }) +} + +func TestAccAzureRMHDInsightHadoopCluster_updateMetastore(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_hdinsight_hadoop_cluster", "test") + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acceptance.PreCheck(t) }, + Providers: acceptance.SupportedProviders, + CheckDestroy: testCheckAzureRMHDInsightClusterDestroy(data.ResourceType), + Steps: []resource.TestStep{ + { + Config: testAccAzureRMHDInsightHadoopCluster_hiveMetastore(data), + Check: resource.ComposeTestCheckFunc( + testCheckAzureRMHDInsightClusterExists(data.ResourceName), + resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"), + resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"), + ), + }, + data.ImportStep("roles.0.head_node.0.password", + "roles.0.head_node.0.vm_size", + "roles.0.worker_node.0.password", + "roles.0.worker_node.0.vm_size", + "roles.0.zookeeper_node.0.password", + "roles.0.zookeeper_node.0.vm_size", + "storage_account", + "metastores.0.hive.0.password", + "metastores.0.oozie.0.password", + "metastores.0.ambari.0.password"), + { + Config: testAccAzureRMHDInsightHadoopCluster_allMetastores(data), + Check: resource.ComposeTestCheckFunc( + testCheckAzureRMHDInsightClusterExists(data.ResourceName), + resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"), + resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"), + ), + }, data.ImportStep("roles.0.head_node.0.password", "roles.0.head_node.0.vm_size", "roles.0.worker_node.0.password", @@ -1101,7 +1138,7 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" { `, template, data.RandomInteger) } -func testAccAzureRMHDInsightHadoopCluster_metastore(data acceptance.TestData) string { +func testAccAzureRMHDInsightHadoopCluster_allMetastores(data acceptance.TestData) string { template := testAccAzureRMHDInsightHadoopCluster_template(data) return fmt.Sprintf(` %s From 0cc4b04ce8b19749973bd7baa7794e4ba9b76fbc Mon Sep 17 00:00:00 2001 From: jackofallops Date: Mon, 4 May 2020 09:12:24 +0100 Subject: [PATCH 10/10] minor docs update --- .../r/hdinsight_hadoop_cluster.html.markdown | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/website/docs/r/hdinsight_hadoop_cluster.html.markdown b/website/docs/r/hdinsight_hadoop_cluster.html.markdown index bc7e707932a2..bccdc6d1b00b 100644 --- a/website/docs/r/hdinsight_hadoop_cluster.html.markdown +++ b/website/docs/r/hdinsight_hadoop_cluster.html.markdown @@ -259,46 +259,46 @@ A `metastores` block supports the following: * `hive` - (Optional) A `hive` block as defined below. -* `oozie` - (Optional) A `oozie` block as defined below. +* `oozie` - (Optional) An `oozie` block as defined below. -* `ambari` - (Optional) A `ambari` block as defined below. +* `ambari` - (Optional) An `ambari` block as defined below. --- A `hive` block supports the following: -* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Hive metastore. +* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Hive metastore. Changing this forces a new resource to be created. -* `database_name` - (Required) The external Hive metastore's existing SQL database. +* `database_name` - (Required) The external Hive metastore's existing SQL database. Changing this forces a new resource to be created. -* `username` - (Required) The external Hive metastore's existing SQL server admin username. +* `username` - (Required) The external Hive metastore's existing SQL server admin username. Changing this forces a new resource to be created. -* `password` - (Required) The external Hive metastore's existing SQL server admin password. +* `password` - (Required) The external Hive metastore's existing SQL server admin password. Changing this forces a new resource to be created. --- -A `oozie` block supports the following: +An `oozie` block supports the following: -* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Oozie metastore. +* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Oozie metastore. Changing this forces a new resource to be created. -* `database_name` - (Required) The external Oozie metastore's existing SQL database. +* `database_name` - (Required) The external Oozie metastore's existing SQL database. Changing this forces a new resource to be created. -* `username` - (Required) The external Oozie metastore's existing SQL server admin username. +* `username` - (Required) The external Oozie metastore's existing SQL server admin username. Changing this forces a new resource to be created. -* `password` - (Required) The external Oozie metastore's existing SQL server admin password. +* `password` - (Required) The external Oozie metastore's existing SQL server admin password. Changing this forces a new resource to be created. --- -A `ambari` block supports the following: +An `ambari` block supports the following: -* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Ambari metastore. +* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Ambari metastore. Changing this forces a new resource to be created. -* `database_name` - (Required) The external Hive metastore's existing SQL database. +* `database_name` - (Required) The external Hive metastore's existing SQL database. Changing this forces a new resource to be created. -* `username` - (Required) The external Ambari metastore's existing SQL server admin username. +* `username` - (Required) The external Ambari metastore's existing SQL server admin username. Changing this forces a new resource to be created. -* `password` - (Required) The external Ambari metastore's existing SQL server admin password. +* `password` - (Required) The external Ambari metastore's existing SQL server admin password. Changing this forces a new resource to be created. ## Attributes Reference