From 60d74e28d85fb6f3794f5234ef164822339eca9e Mon Sep 17 00:00:00 2001 From: The Magician Date: Mon, 7 Mar 2022 12:21:25 -0600 Subject: [PATCH] Add support for non-preemptible secondary dataproc workers (#5686) (#11230) Co-authored-by: Matthew Barnes Co-authored-by: Stephen Lewis (Burrows) Co-authored-by: Riley Karson Co-authored-by: Scott Suarez Co-authored-by: Sampath Kumar Co-authored-by: Sam Levenick Co-authored-by: John Pellman Co-authored-by: Jacek Kikiewicz Co-authored-by: Alex Ellis Co-authored-by: megan07 Co-authored-by: Daniel Randell Co-authored-by: Iris Chen <10179943+iyabchen@users.noreply.github.com> Co-authored-by: prateek2408 Signed-off-by: Modular Magician Co-authored-by: Matthew Barnes Co-authored-by: Stephen Lewis (Burrows) Co-authored-by: Riley Karson Co-authored-by: Scott Suarez Co-authored-by: Sampath Kumar Co-authored-by: Sam Levenick Co-authored-by: John Pellman Co-authored-by: Jacek Kikiewicz Co-authored-by: Alex Ellis Co-authored-by: megan07 Co-authored-by: Daniel Randell Co-authored-by: Iris Chen <10179943+iyabchen@users.noreply.github.com> Co-authored-by: prateek2408 --- .changelog/5686.txt | 4 ++ google/resource_dataproc_cluster.go | 23 +++++++- google/resource_dataproc_cluster_test.go | 56 +++++++++++++++++++ website/docs/r/dataproc_cluster.html.markdown | 6 ++ 4 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 .changelog/5686.txt diff --git a/.changelog/5686.txt b/.changelog/5686.txt new file mode 100644 index 00000000000..93e76aa0994 --- /dev/null +++ b/.changelog/5686.txt @@ -0,0 +1,4 @@ +```release-note:enhancement +dataproc:added `preemptibility` field to the `preemptible_worker_config` of `google_dataproc_cluster` + +``` diff --git a/google/resource_dataproc_cluster.go b/google/resource_dataproc_cluster.go index 5fac09e18cd..d17d23fc959 100644 --- a/google/resource_dataproc_cluster.go +++ b/google/resource_dataproc_cluster.go @@ -340,6 +340,7 @@ func resourceDataprocCluster() *schema.Resource { Description: `Specifies the number of preemptible nodes to create. Defaults to 0.`, AtLeastOneOf: []string{ "cluster_config.0.preemptible_worker_config.0.num_instances", + "cluster_config.0.preemptible_worker_config.0.preemptibility", "cluster_config.0.preemptible_worker_config.0.disk_config", }, }, @@ -348,6 +349,20 @@ func resourceDataprocCluster() *schema.Resource { // It always uses whatever is specified for the worker_config // "machine_type": { ... } // "min_cpu_platform": { ... } + "preemptibility": { + Type: schema.TypeString, + Optional: true, + Description: `Specifies the preemptibility of the secondary nodes. Defaults to PREEMPTIBLE.`, + AtLeastOneOf: []string{ + "cluster_config.0.preemptible_worker_config.0.num_instances", + "cluster_config.0.preemptible_worker_config.0.preemptibility", + "cluster_config.0.preemptible_worker_config.0.disk_config", + }, + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"PREEMPTIBILITY_UNSPECIFIED", "NON_PREEMPTIBLE", "PREEMPTIBLE"}, false), + Default: "PREEMPTIBLE", + }, + "disk_config": { Type: schema.TypeList, Optional: true, @@ -355,6 +370,7 @@ func resourceDataprocCluster() *schema.Resource { Description: `Disk Config`, AtLeastOneOf: []string{ "cluster_config.0.preemptible_worker_config.0.num_instances", + "cluster_config.0.preemptible_worker_config.0.preemptibility", "cluster_config.0.preemptible_worker_config.0.disk_config", }, MaxItems: 1, @@ -891,9 +907,6 @@ func expandClusterConfig(d *schema.ResourceData, config *Config) (*dataproc.Clus if cfg, ok := configOptions(d, "cluster_config.0.preemptible_worker_config"); ok { log.Println("[INFO] got preemptible worker config") conf.SecondaryWorkerConfig = expandPreemptibleInstanceGroupConfig(cfg) - if conf.SecondaryWorkerConfig.NumInstances > 0 { - conf.SecondaryWorkerConfig.IsPreemptible = true - } } return conf, nil } @@ -1101,6 +1114,9 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc. } } } + if p, ok := cfg["preemptibility"]; ok { + icg.Preemptibility = p.(string) + } return icg } @@ -1473,6 +1489,7 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc if icg != nil { data["num_instances"] = icg.NumInstances data["instance_names"] = icg.InstanceNames + data["preemptibility"] = icg.Preemptibility if icg.DiskConfig != nil { disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds diff --git a/google/resource_dataproc_cluster_test.go b/google/resource_dataproc_cluster_test.go index d31c1114e7b..1fe4b43a223 100644 --- a/google/resource_dataproc_cluster_test.go +++ b/google/resource_dataproc_cluster_test.go @@ -391,6 +391,27 @@ func TestAccDataprocCluster_updatable(t *testing.T) { }) } +func TestAccDataprocCluster_nonPreemptibleSecondary(t *testing.T) { + t.Parallel() + + rnd := randString(t, 10) + var cluster dataproc.Cluster + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataprocClusterDestroy(t), + Steps: []resource.TestStep{ + { + Config: testAccDataprocCluster_nonPreemptibleSecondary(rnd), + Check: resource.ComposeTestCheckFunc( + testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.non_preemptible_secondary", &cluster), + resource.TestCheckResourceAttr("google_dataproc_cluster.non_preemptible_secondary", "cluster_config.0.preemptible_worker_config.0.preemptibility", "NON_PREEMPTIBLE"), + ), + }, + }, + }) +} + func TestAccDataprocCluster_withStagingBucket(t *testing.T) { t.Parallel() @@ -1220,6 +1241,41 @@ resource "google_dataproc_cluster" "updatable" { `, rnd, w, p) } +func testAccDataprocCluster_nonPreemptibleSecondary(rnd string) string { + return fmt.Sprintf(` +resource "google_dataproc_cluster" "non_preemptible_secondary" { + name = "tf-test-dproc-%s" + region = "us-central1" + + cluster_config { + master_config { + num_instances = "1" + machine_type = "e2-medium" + disk_config { + boot_disk_size_gb = 35 + } + } + + worker_config { + num_instances = "2" + machine_type = "e2-medium" + disk_config { + boot_disk_size_gb = 35 + } + } + + preemptible_worker_config { + num_instances = "1" + preemptibility = "NON_PREEMPTIBLE" + disk_config { + boot_disk_size_gb = 35 + } + } + } +} + `, rnd) +} + func testAccDataprocCluster_withStagingBucketOnly(bucketName string) string { return fmt.Sprintf(` resource "google_storage_bucket" "bucket" { diff --git a/website/docs/r/dataproc_cluster.html.markdown b/website/docs/r/dataproc_cluster.html.markdown index 071d281e791..f339b5eec0c 100644 --- a/website/docs/r/dataproc_cluster.html.markdown +++ b/website/docs/r/dataproc_cluster.html.markdown @@ -443,6 +443,12 @@ will be set for you based on whatever was set for the `worker_config.machine_typ * `num_instances`- (Optional) Specifies the number of preemptible nodes to create. Defaults to 0. +* `preemptibility`- (Optional) Specifies the preemptibility of the secondary workers. The default value is `PREEMPTIBLE` + Accepted values are: + * PREEMPTIBILITY_UNSPECIFIED + * NON_PREEMPTIBLE + * PREEMPTIBLE + * `disk_config` (Optional) Disk Config * `boot_disk_type` - (Optional) The disk type of the primary disk attached to each preemptible worker node.