Skip to content

Commit

Permalink
Merge pull request #6145 from kosinsky/hdi/external_metastore
Browse files Browse the repository at this point in the history
  • Loading branch information
@jackofallops
jackofallops authored May 4, 2020
2 parents 5ad0f7b + 0cc4b04 commit b16f7e5
Show file tree
Hide file tree
Showing 5 changed files with 638 additions and 15 deletions.
224 changes: 223 additions & 1 deletion azurerm/helpers/azure/hdinsight.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func SchemaHDInsightTier() *schema.Schema {
ValidateFunc: validation.StringInSlice([]string{
string(hdinsight.Standard),
string(hdinsight.Premium),
}, false),
}, true),
// TODO: file a bug about this
DiffSuppressFunc: location.DiffSuppressFunc,
}
Expand Down Expand Up @@ -119,6 +119,43 @@ func SchemaHDInsightsGateway() *schema.Schema {
}
}

func SchemaHDInsightsExternalMetastore() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"server": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"database_name": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"username": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"password": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
Sensitive: true,
// Azure returns the key as *****. We'll suppress that here.
DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
return (new == d.Get(k).(string)) && (old == "*****")
},
},
},
},
}
}

func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

Expand All @@ -136,6 +173,86 @@ func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{}
}
}

func ExpandHDInsightsHiveMetastore(input []interface{}) map[string]interface{} {
if len(input) == 0 {
return nil
}
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
database := vs["database_name"].(string)
username := vs["username"].(string)
password := vs["password"].(string)

return map[string]interface{}{
"hive-site": map[string]interface{}{
"javax.jdo.option.ConnectionDriverName": "com.microsoft.sqlserver.jdbc.SQLServerDriver",
"javax.jdo.option.ConnectionURL": fmt.Sprintf("jdbc:sqlserver://%s;database=%s;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", server, database),
"javax.jdo.option.ConnectionUserName": username,
"javax.jdo.option.ConnectionPassword": password,
},
"hive-env": map[string]interface{}{
"hive_database": "Existing MSSQL Server database with SQL authentication",
"hive_database_name": database,
"hive_database_type": "mssql",
"hive_existing_mssql_server_database": database,
"hive_existing_mssql_server_host": server,
"hive_hostname": server,
},
}
}

func ExpandHDInsightsOozieMetastore(input []interface{}) map[string]interface{} {
if len(input) == 0 {
return nil
}
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
database := vs["database_name"].(string)
username := vs["username"].(string)
password := vs["password"].(string)

return map[string]interface{}{
"oozie-site": map[string]interface{}{
"oozie.service.JPAService.jdbc.driver": "com.microsoft.sqlserver.jdbc.SQLServerDriver",
"oozie.service.JPAService.jdbc.url": fmt.Sprintf("jdbc:sqlserver://%s;database=%s;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", server, database),
"oozie.service.JPAService.jdbc.username": username,
"oozie.service.JPAService.jdbc.password": password,
"oozie.db.schema.name": "oozie",
},
"oozie-env": map[string]interface{}{
"oozie_database": "Existing MSSQL Server database with SQL authentication",
"oozie_database_name": database,
"oozie_database_type": "mssql",
"oozie_existing_mssql_server_database": database,
"oozie_existing_mssql_server_host": server,
"oozie_hostname": server,
},
}
}

func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{} {
if len(input) == 0 {
return nil
}
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
database := vs["database_name"].(string)
username := vs["username"].(string)
password := vs["password"].(string)

return map[string]interface{}{
"ambari-conf": map[string]interface{}{
"database-server": server,
"database-name": database,
"database-user-name": username,
"database-user-password": password,
},
}
}

func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
enabled := false
if v, exists := input["restAuthCredential.isEnabled"]; exists && v != nil {
Expand Down Expand Up @@ -164,6 +281,111 @@ func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
}
}

func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*string) []interface{} {
server := ""
if v, exists := env["hive_hostname"]; exists && v != nil {
server = *v
}

database := ""
if v, exists := env["hive_database_name"]; exists && v != nil {
database = *v
}

username := ""
if v, exists := site["javax.jdo.option.ConnectionUserName"]; exists && v != nil {
username = *v
}

password := ""
if v, exists := site["javax.jdo.option.ConnectionPassword"]; exists && v != nil {
password = *v
}

if server != "" && database != "" {
return []interface{}{
map[string]interface{}{
"server": server,
"database_name": database,
"username": username,
"password": password,
},
}
}

return nil
}

func FlattenHDInsightsOozieMetastore(env map[string]*string, site map[string]*string) []interface{} {
server := ""
if v, exists := env["oozie_hostname"]; exists && v != nil {
server = *v
}

database := ""
if v, exists := env["oozie_database_name"]; exists && v != nil {
database = *v
}

username := ""
if v, exists := site["oozie.service.JPAService.jdbc.username"]; exists && v != nil {
username = *v
}

password := ""
if v, exists := site["oozie.service.JPAService.jdbc.password"]; exists && v != nil {
password = *v
}

if server != "" && database != "" {
return []interface{}{
map[string]interface{}{
"server": server,
"database_name": database,
"username": username,
"password": password,
},
}
}

return nil
}

func FlattenHDInsightsAmbariMetastore(conf map[string]*string) []interface{} {
server := ""
if v, exists := conf["database-server"]; exists && v != nil {
server = *v
}

database := ""
if v, exists := conf["database-name"]; exists && v != nil {
database = *v
}

username := ""
if v, exists := conf["database-user-name"]; exists && v != nil {
username = *v
}

password := ""
if v, exists := conf["database-user-password"]; exists && v != nil {
password = *v
}

if server != "" && database != "" {
return []interface{}{
map[string]interface{}{
"server": server,
"database_name": database,
"username": username,
"password": password,
},
}
}

return nil
}

func SchemaHDInsightsStorageAccounts() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Expand Down
53 changes: 53 additions & 0 deletions azurerm/internal/services/hdinsight/common_hdinsight.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,3 +272,56 @@ func deleteHDInsightEdgeNodes(ctx context.Context, client *hdinsight.Application

return nil
}

func expandHDInsightsMetastore(input []interface{}) map[string]interface{} {
v := input[0].(map[string]interface{})

config := map[string]interface{}{}

if hiveRaw, ok := v["hive"]; ok {
for k, val := range azure.ExpandHDInsightsHiveMetastore(hiveRaw.([]interface{})) {
config[k] = val
}
}

if oozieRaw, ok := v["oozie"]; ok {
for k, val := range azure.ExpandHDInsightsOozieMetastore(oozieRaw.([]interface{})) {
config[k] = val
}
}

if ambariRaw, ok := v["ambari"]; ok {
for k, val := range azure.ExpandHDInsightsAmbariMetastore(ambariRaw.([]interface{})) {
config[k] = val
}
}

return config
}

func flattenHDInsightsMetastores(d *schema.ResourceData, configurations map[string]map[string]*string) {
result := map[string]interface{}{}

hiveEnv, envExists := configurations["hive-env"]
hiveSite, siteExists := configurations["hive-site"]
if envExists && siteExists {
result["hive"] = azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite)
}

oozieEnv, envExists := configurations["oozie-env"]
oozieSite, siteExists := configurations["oozie-site"]
if envExists && siteExists {
result["oozie"] = azure.FlattenHDInsightsOozieMetastore(oozieEnv, oozieSite)
}

ambari, ambariExists := configurations["ambari-conf"]
if ambariExists {
result["ambari"] = azure.FlattenHDInsightsAmbariMetastore(ambari)
}

if len(result) > 0 {
d.Set("metastores", []interface{}{
result,
})
}
}
38 changes: 34 additions & 4 deletions azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,21 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"hive": azure.SchemaHDInsightsExternalMetastore(),

"oozie": azure.SchemaHDInsightsExternalMetastore(),

"ambari": azure.SchemaHDInsightsExternalMetastore(),
},
},
},

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
Expand Down Expand Up @@ -186,7 +201,14 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
componentVersions := expandHDInsightHadoopComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

if metastoresRaw, ok := d.GetOkExists("metastores"); ok {
metastores := expandHDInsightsMetastore(metastoresRaw.([]interface{}))
for k, v := range metastores {
configurations[k] = v
}
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -229,7 +251,7 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("Hadoop"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -315,11 +337,17 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
return fmt.Errorf("Error retrieving HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -337,9 +365,11 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

hadoopRoles := hdInsightRoleDefinition{
Expand Down
Loading

0 comments on commit b16f7e5

Please sign in to comment.