From 08378906bd3c44a15463e331d73c362a073a36b1 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Wed, 23 Feb 2022 21:09:57 +0200 Subject: [PATCH 01/14] dataproc-job-log-level --- .../resources/resource_dataproc_job.go.erb | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb index ac2f35512b7b..3e64449ebd97 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb @@ -16,6 +16,18 @@ import ( <% end -%> ) +loggingLevels := []string{ + "LEVEL_UNSPECIFIED", + "ALL", + "TRACE", + "DEBUG", + "INFO", + "WARN", + "ERROR", + "FATAL", + "OFF", +} + func resourceDataprocJob() *schema.Resource { return &schema.Resource{ Create: resourceDataprocJobCreate, @@ -425,11 +437,12 @@ var loggingConfig = &schema.Schema{ Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "driver_log_levels": { - Type: schema.TypeMap, - Description: "Optional. The per-package log levels for the driver. This may include 'root' package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'.", - Required: true, - ForceNew: true, - Elem: &schema.Schema{Type: schema.TypeString}, + Type: schema.TypeMap, + Description: "Optional. The per-package log levels for the driver. This may include 'root' package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'.", + Required: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + ValidateFunc: validation.StringInSlice(loggingLevels, false), }, }, }, From c5200f7f0c5d7391a13d2352cd2ed4cf2cc167c2 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Wed, 23 Feb 2022 21:45:39 +0200 Subject: [PATCH 02/14] add presto job --- .../resources/resource_dataproc_job.go.erb | 130 +++++++++++++++--- 1 file changed, 112 insertions(+), 18 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb index 3e64449ebd97..33197c273963 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb @@ -16,17 +16,7 @@ import ( <% end -%> ) -loggingLevels := []string{ - "LEVEL_UNSPECIFIED", - "ALL", - "TRACE", - "DEBUG", - "INFO", - "WARN", - "ERROR", - "FATAL", - "OFF", -} +jobTypes := []string{"pyspark_config", "spark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config", "presto_config"} func resourceDataprocJob() *schema.Resource { return &schema.Resource{ @@ -442,7 +432,6 @@ var loggingConfig = &schema.Schema{ Required: true, ForceNew: true, Elem: &schema.Schema{Type: schema.TypeString}, - ValidateFunc: validation.StringInSlice(loggingLevels, false), }, }, }, @@ -454,7 +443,7 @@ var pySparkSchema = &schema.Schema{ ForceNew: true, MaxItems: 1, Description: `The config of pySpark job.`, - ExactlyOneOf: []string{"pyspark_config", "spark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config"}, + ExactlyOneOf: jobTypes, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "main_python_file_uri": { @@ -583,7 +572,7 @@ var sparkSchema = &schema.Schema{ ForceNew: true, MaxItems: 1, Description: `The config of the Spark job.`, - ExactlyOneOf: []string{"pyspark_config", "spark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config"}, + ExactlyOneOf: jobTypes, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ // main driver: can be only one of the class | jar_file @@ -704,7 +693,7 @@ var hadoopSchema = &schema.Schema{ ForceNew: true, MaxItems: 1, Description: `The config of Hadoop job`, - ExactlyOneOf: []string{"spark_config", "pyspark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config"}, + ExactlyOneOf: jobTypes, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ // main driver: can be only one of the main_class | main_jar_file_uri @@ -825,7 +814,7 @@ var hiveSchema = &schema.Schema{ ForceNew: true, MaxItems: 1, Description: `The config of hive job`, - ExactlyOneOf: []string{"spark_config", "pyspark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config"}, + ExactlyOneOf: jobTypes, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ // main query: can be only one of query_list | query_file_uri @@ -931,7 +920,7 @@ var pigSchema = &schema.Schema{ ForceNew: true, MaxItems: 1, Description: `The config of pag job.`, - ExactlyOneOf: []string{"spark_config", "pyspark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config"}, + ExactlyOneOf: jobTypes, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ // main query: can be only one of query_list | query_file_uri @@ -1040,7 +1029,7 @@ var sparkSqlSchema = &schema.Schema{ ForceNew: true, MaxItems: 1, Description: `The config of SparkSql job`, - ExactlyOneOf: []string{"spark_config", "pyspark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config"}, + ExactlyOneOf: jobTypes, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ // main query: can be only one of query_list | query_file_uri @@ -1130,6 +1119,111 @@ func expandSparkSqlJob(config map[string]interface{}) *dataproc.SparkSqlJob { } +// ---- Presto Job ---- + +var prestoSchema = &schema.Schema{ + Type: schema.TypeList, + Optional: true, + ForceNew: true, + MaxItems: 1, + Description: `The config of presto job`, + ExactlyOneOf: jobTypes, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "client_tags": { + Type: schema.TypeList, + Description: `Presto client tags to attach to this query.`, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "continue_on_failure": { + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + Description: `Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries. Defaults to false.`, + }, + // main query: can be only one of query_list | query_file_uri + "query_list": { + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Description: `The list of SQL queries or statements to execute as part of the job. Conflicts with query_file_uri`, + Elem: &schema.Schema{Type: schema.TypeString}, + ExactlyOneOf: []string{"presto_config.0.query_file_uri", "presto_config.0.query_list"}, + }, + + "query_file_uri": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: `The HCFS URI of the script that contains SQL queries. Conflicts with query_list`, + ExactlyOneOf: []string{"presto_config.0.query_file_uri", "presto_config.0.query_list"}, + }, + + "properties": { + Type: schema.TypeMap, + Optional: true, + ForceNew: true, + Description: `A mapping of property names to values, used to configure Spark SQL's SparkConf. Properties that conflict with values set by the Cloud Dataproc API may be overwritten.`, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "output_format": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: `The format in which query output will be displayed. See the Presto documentation for supported output formats.`, + }, + + "logging_config": loggingConfig, + }, + }, +} + +func flattenPrestoJob(job *dataproc.PrestoJob) []map[string]interface{} { + queries := []string{} + if job.QueryList != nil { + queries = job.QueryList.Queries + } + return []map[string]interface{}{ + { + "client_tags": job.ClientTags, + "continue_on_failure": job.ContinueOnFailure, + "query_list": queries, + "query_file_uri": job.QueryFileUri, + "properties": job.Properties, + "output_format": job.OutputFormat, + }, + } +} + +func expandPrestoJob(config map[string]interface{}) *dataproc.PrestoJob { + job := &dataproc.PrestoJob{} + if v, ok := config["client_tags"]; ok { + job.ClientTags = convertStringArr(v.([]interface{})) + } + if v, ok := config["continue_on_failure"]; ok { + job.ContinueOnFailure = v.(bool) + } + if v, ok := config["query_file_uri"]; ok { + job.QueryFileUri = v.(string) + } + if v, ok := config["query_list"]; ok { + job.QueryList = &dataproc.QueryList{ + Queries: convertStringArr(v.([]interface{})), + } + } + if v, ok := config["properties"]; ok { + job.Properties = convertStringMap(v.(map[string]interface{})) + } + if v, ok := config["output_format"]; ok { + job.OutputFormat = v.(string) + } + + return job + +} + // ---- Other flatten / expand methods ---- func expandLoggingConfig(config map[string]interface{}) *dataproc.LoggingConfig { From cbf770b5c8bf6d950c2c47fddfa2e27069272e77 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Wed, 23 Feb 2022 21:50:37 +0200 Subject: [PATCH 03/14] fmt --- .../terraform/resources/resource_dataproc_job.go.erb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb index 33197c273963..30cb0451a5ca 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb @@ -427,11 +427,11 @@ var loggingConfig = &schema.Schema{ Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "driver_log_levels": { - Type: schema.TypeMap, - Description: "Optional. The per-package log levels for the driver. This may include 'root' package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'.", - Required: true, - ForceNew: true, - Elem: &schema.Schema{Type: schema.TypeString}, + Type: schema.TypeMap, + Description: "Optional. The per-package log levels for the driver. This may include 'root' package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG'.", + Required: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, }, }, }, From 0a494960d72dd4ae364641140d75ed71071e68c4 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Wed, 23 Feb 2022 21:52:00 +0200 Subject: [PATCH 04/14] desc for presto props --- .../terraform/resources/resource_dataproc_job.go.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb index 30cb0451a5ca..da90298c7eff 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb @@ -1165,7 +1165,7 @@ var prestoSchema = &schema.Schema{ Type: schema.TypeMap, Optional: true, ForceNew: true, - Description: `A mapping of property names to values, used to configure Spark SQL's SparkConf. Properties that conflict with values set by the Cloud Dataproc API may be overwritten.`, + Description: `A mapping of property names to values. Used to set Presto session properties Equivalent to using the --session flag in the Presto CLI.`, Elem: &schema.Schema{Type: schema.TypeString}, }, "output_format": { From be62d3d8b4008ce7c4c0edceed2506fc7e51facb Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Wed, 23 Feb 2022 22:10:33 +0200 Subject: [PATCH 05/14] docs --- .../resources/resource_dataproc_job.go.erb | 2 +- .../website/docs/r/dataproc_job.html.markdown | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb index da90298c7eff..04be2dc44ef0 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb @@ -1141,7 +1141,7 @@ var prestoSchema = &schema.Schema{ Type: schema.TypeBool, Optional: true, ForceNew: true, - Description: `Whether to continue executing queries if a query fails. The default value is false. Setting to true can be useful when executing independent parallel queries. Defaults to false.`, + Description: `Whether to continue executing queries if a query fails. Setting to true can be useful when executing independent parallel queries. Defaults to false.`, }, // main query: can be only one of query_list | query_file_uri "query_list": { diff --git a/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown index 1c7ef3dbed78..7e9fad780126 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown @@ -90,6 +90,7 @@ output "pyspark_status" { * [hive_config](#nested_hive_config) - Submits a Hive job to the cluster * [hpig_config](#nested_hpig_config) - Submits a Pig job to the cluster * [sparksql_config](#nested_sparksql_config) - Submits a Spark SQL job to the cluster + * [presto_config](#nested_presto_config) - Submits a Spark SQL job to the cluster - - - @@ -320,6 +321,37 @@ resource "google_dataproc_job" "sparksql" { * `logging_config.driver_log_levels`- (Required) The per-package log levels for the driver. This may include 'root' package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG' +The `presto_config` block supports: + +```hcl +# Submit a spark SQL job to the cluster +resource "google_dataproc_job" "presto" { + ... + presto_config { + query_list = [ + "DROP TABLE IF EXISTS dprocjob_test", + "CREATE TABLE dprocjob_test(bar int)", + "SELECT * FROM dprocjob_test WHERE bar > 2", + ] + } +} +``` + +* `client_tags` - (Optional) Presto client tags to attach to this query. + +* `continue_on_failure` - (Optional) Whether to continue executing queries if a query fails. Setting to true can be useful when executing independent parallel queries. Defaults to false. + +* `query_list`- (Optional) The list of SQL queries or statements to execute as part of the job. + Conflicts with `query_file_uri` + +* `query_file_uri` - (Optional) The HCFS URI of the script that contains SQL queries. + Conflicts with `query_list` + +* `properties` - (Optional) A mapping of property names to values. Used to set Presto session properties Equivalent to using the --session flag in the Presto CLI. + +* `output_format` - (Optional) The format in which query output will be displayed. See the Presto documentation for supported output formats. + +* `logging_config.driver_log_levels`- (Required) The per-package log levels for the driver. This may include 'root' package name to configure rootLogger. Examples: 'com.google = FATAL', 'root = INFO', 'org.apache = DEBUG' ## Attributes Reference From 13d83fc10c7584ff0e667a6ade2459344ada3c80 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Wed, 23 Feb 2022 22:11:47 +0200 Subject: [PATCH 06/14] docs --- .../terraform/website/docs/r/dataproc_job.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown index 7e9fad780126..9f2d49f63721 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown @@ -324,7 +324,7 @@ resource "google_dataproc_job" "sparksql" { The `presto_config` block supports: ```hcl -# Submit a spark SQL job to the cluster +# Submit a Presto job to the cluster resource "google_dataproc_job" "presto" { ... presto_config { From 59cc908107a765a65824f2f234f76d132e5352aa Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 24 Feb 2022 00:43:08 +0200 Subject: [PATCH 07/14] actually add presto config --- .../third_party/terraform/resources/resource_dataproc_job.go.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb index 04be2dc44ef0..7d09f5e66cda 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb @@ -182,6 +182,7 @@ func resourceDataprocJob() *schema.Resource { "hive_config": hiveSchema, "pig_config": pigSchema, "sparksql_config": sparkSqlSchema, + "presto_config": prestoSchema, }, UseJSONNumber: true, } From a0127c80b5b535c4935b4a04368ad7da01e9fc3a Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 24 Feb 2022 00:45:17 +0200 Subject: [PATCH 08/14] presto test --- .../tests/resource_dataproc_job_test.go.erb | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb b/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb index bc7d6f0ba760..bfbb465b42a1 100644 --- a/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb @@ -272,6 +272,38 @@ func TestAccDataprocJob_SparkSql(t *testing.T) { }) } +func TestAccDataprocJob_Presto(t *testing.T) { + t.Parallel() + + var job dataproc.Job + rnd := randString(t, 10) + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataprocJobDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataprocJob_presto(rnd), + Check: resource.ComposeTestCheckFunc( + testAccCheckDataprocJobExists(t, "google_dataproc_job.presto", &job), + + // Autogenerated / computed values + resource.TestCheckResourceAttrSet("google_dataproc_job.presto", "reference.0.job_id"), + resource.TestCheckResourceAttrSet("google_dataproc_job.presto", "status.0.state"), + resource.TestCheckResourceAttrSet("google_dataproc_job.presto", "status.0.state_start_time"), + + // Unique job config + testAccCheckDataprocJobAttrMatch( + "google_dataproc_job.presto", "presto_config", &job), + + // Wait until job completes successfully + testAccCheckDataprocJobCompletesSuccessfully(t, "google_dataproc_job.presto", &job), + ), + }, + }, + }) +} + func testAccCheckDataprocJobDestroyProducer(t *testing.T) func(s *terraform.State) error { return func(s *terraform.State) error { config := googleProviderConfig(t) @@ -693,3 +725,25 @@ resource "google_dataproc_job" "sparksql" { `, rnd) } + +func testAccDataprocJob_sparksql(rnd string) string { + return fmt.Sprintf( + singleNodeClusterConfig+` +resource "google_dataproc_job" "presto" { + region = google_dataproc_cluster.basic.region + force_delete = true + placement { + cluster_name = google_dataproc_cluster.basic.name + } + + presto_config { + query_list = [ + "DROP TABLE IF EXISTS dprocjob_test", + "CREATE TABLE dprocjob_test(bar int)", + "SELECT * FROM dprocjob_test WHERE bar > 2", + ] + } +} +`, rnd) + +} From 5a9d09b96a154f4c49dd9933d26c26109577f8ee Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 24 Feb 2022 00:47:18 +0200 Subject: [PATCH 09/14] presto test actually --- .../tests/resource_dataproc_job_test.go.erb | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb b/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb index bfbb465b42a1..edca46938dff 100644 --- a/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb @@ -727,8 +727,30 @@ resource "google_dataproc_job" "sparksql" { } func testAccDataprocJob_sparksql(rnd string) string { - return fmt.Sprintf( - singleNodeClusterConfig+` + return fmt.Sprintf(` +resource "google_dataproc_cluster" "basic" { + name = "dproc-job-test-%s" + region = "us-central1" + + cluster_config { + # Keep the costs down with smallest config we can get away with + software_config { + override_properties = { + "dataproc:dataproc.allow.zero.workers" = "true" + } + optional_components = ["PRESTO"] + } + + master_config { + num_instances = 1 + machine_type = "e2-standard-2" + disk_config { + boot_disk_size_gb = 35 + } + } + } +} + resource "google_dataproc_job" "presto" { region = google_dataproc_cluster.basic.region force_delete = true From 3d95027a11826d393e8ea26fca88a3930a412b04 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 24 Feb 2022 00:50:55 +0200 Subject: [PATCH 10/14] docs --- .../terraform/website/docs/r/dataproc_job.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown index 9f2d49f63721..fb2e436aadf3 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataproc_job.html.markdown @@ -90,7 +90,7 @@ output "pyspark_status" { * [hive_config](#nested_hive_config) - Submits a Hive job to the cluster * [hpig_config](#nested_hpig_config) - Submits a Pig job to the cluster * [sparksql_config](#nested_sparksql_config) - Submits a Spark SQL job to the cluster - * [presto_config](#nested_presto_config) - Submits a Spark SQL job to the cluster + * [presto_config](#nested_presto_config) - Submits a Presto job to the cluster - - - From 749b95aaf5e6a857edc9c680362a87534e159d22 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 24 Feb 2022 08:57:52 +0200 Subject: [PATCH 11/14] presto expand/flatten --- .../terraform/resources/resource_dataproc_job.go.erb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb index 7d09f5e66cda..d6e015508b0d 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb @@ -264,6 +264,11 @@ func resourceDataprocJobCreate(d *schema.ResourceData, meta interface{}) error { submitReq.Job.SparkSqlJob = expandSparkSqlJob(config) } + if v, ok := d.GetOk("presto_config"); ok { + config := extractFirstMapConfig(v.([]interface{})) + submitReq.Job.PrestoJob = expandPrestoJob(config) + } + // Submit the job job, err := config.NewDataprocClient(userAgent).Projects.Regions.Jobs.Submit( project, region, submitReq).Do() @@ -362,6 +367,12 @@ func resourceDataprocJobRead(d *schema.ResourceData, meta interface{}) error { return fmt.Errorf("Error setting sparksql_config: %s", err) } } + + if job.PrestoJob != nil { + if err := d.Set("presto_config", flattenPrestoJob(job.PrestoJob)); err != nil { + return fmt.Errorf("Error setting presto_config: %s", err) + } + } return nil } From d5b483225057564e40affda12f5c80d5b6a28649 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 24 Feb 2022 09:28:16 +0200 Subject: [PATCH 12/14] presto expand/flatten --- .../terraform/resources/resource_dataproc_job.go.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb index d6e015508b0d..83e2059dd5fa 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_job.go.erb @@ -16,7 +16,7 @@ import ( <% end -%> ) -jobTypes := []string{"pyspark_config", "spark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config", "presto_config"} +var jobTypes = []string{"pyspark_config", "spark_config", "hadoop_config", "hive_config", "pig_config", "sparksql_config", "presto_config"} func resourceDataprocJob() *schema.Resource { return &schema.Resource{ From 6e4228c82609af6083d04cf9b3a901b1a38b6bd0 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 24 Feb 2022 10:41:02 +0200 Subject: [PATCH 13/14] fix test --- .../terraform/tests/resource_dataproc_job_test.go.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb b/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb index edca46938dff..318402e9c5ad 100644 --- a/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb @@ -726,7 +726,7 @@ resource "google_dataproc_job" "sparksql" { } -func testAccDataprocJob_sparksql(rnd string) string { +func testAccDataprocJob_presto(rnd string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "basic" { name = "dproc-job-test-%s" From c561f41f6aa0c18c666495f64fba68fa05b32954 Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Mon, 28 Feb 2022 00:21:38 +0200 Subject: [PATCH 14/14] Update resource_dataproc_job_test.go.erb --- .../terraform/tests/resource_dataproc_job_test.go.erb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb b/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb index 318402e9c5ad..c1424014185b 100644 --- a/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_dataproc_job_test.go.erb @@ -760,9 +760,7 @@ resource "google_dataproc_job" "presto" { presto_config { query_list = [ - "DROP TABLE IF EXISTS dprocjob_test", - "CREATE TABLE dprocjob_test(bar int)", - "SELECT * FROM dprocjob_test WHERE bar > 2", + "SELECT * FROM system.metadata.schema_properties" ] } }