Add support for non-preemptible secondary dataproc workers (#5686) (#…

…11230) Co-authored-by: Matthew Barnes <matthew.simon.barnes@gmail.com> Co-authored-by: Stephen Lewis (Burrows) <stephenrlewis@google.com> Co-authored-by: Riley Karson <rileykarson@google.com> Co-authored-by: Scott Suarez <ScottMSuarez@gmail.com> Co-authored-by: Sampath Kumar <sampathm@google.com> Co-authored-by: Sam Levenick <slevenick@google.com> Co-authored-by: John Pellman <pellman.john@gmail.com> Co-authored-by: Jacek Kikiewicz <jaceq@users.noreply.github.com> Co-authored-by: Alex Ellis <alexellis@google.com> Co-authored-by: megan07 <mbang@hashicorp.com> Co-authored-by: Daniel Randell <drandell@users.noreply.github.com> Co-authored-by: Iris Chen <10179943+iyabchen@users.noreply.github.com> Co-authored-by: prateek2408 <prateek.khushalani@gmail.com> Signed-off-by: Modular Magician <magic-modules@google.com> Co-authored-by: Matthew Barnes <matthew.simon.barnes@gmail.com> Co-authored-by: Stephen Lewis (Burrows) <stephenrlewis@google.com> Co-authored-by: Riley Karson <rileykarson@google.com> Co-authored-by: Scott Suarez <ScottMSuarez@gmail.com> Co-authored-by: Sampath Kumar <sampathm@google.com> Co-authored-by: Sam Levenick <slevenick@google.com> Co-authored-by: John Pellman <pellman.john@gmail.com> Co-authored-by: Jacek Kikiewicz <jaceq@users.noreply.github.com> Co-authored-by: Alex Ellis <alexellis@google.com> Co-authored-by: megan07 <mbang@hashicorp.com> Co-authored-by: Daniel Randell <drandell@users.noreply.github.com> Co-authored-by: Iris Chen <10179943+iyabchen@users.noreply.github.com> Co-authored-by: prateek2408 <prateek.khushalani@gmail.com>
hashicorp · Mar 7, 2022 · 60d74e2 · 60d74e2
1 parent e0be584
commit 60d74e2
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 3 deletions.
diff --git a/.changelog/5686.txt b/.changelog/5686.txt
@@ -0,0 +1,4 @@
+```release-note:enhancement
+dataproc:added `preemptibility` field to the `preemptible_worker_config` of `google_dataproc_cluster`
+
+```
diff --git a/google/resource_dataproc_cluster.go b/google/resource_dataproc_cluster.go
@@ -340,6 +340,7 @@ func resourceDataprocCluster() *schema.Resource {
 										Description: `Specifies the number of preemptible nodes to create. Defaults to 0.`,
 										AtLeastOneOf: []string{
 											"cluster_config.0.preemptible_worker_config.0.num_instances",
+											"cluster_config.0.preemptible_worker_config.0.preemptibility",
 											"cluster_config.0.preemptible_worker_config.0.disk_config",
 										},
 									},
@@ -348,13 +349,28 @@ func resourceDataprocCluster() *schema.Resource {
 									// It always uses whatever is specified for the worker_config
 									// "machine_type": { ... }
 									// "min_cpu_platform": { ... }
+									"preemptibility": {
+										Type:        schema.TypeString,
+										Optional:    true,
+										Description: `Specifies the preemptibility of the secondary nodes. Defaults to PREEMPTIBLE.`,
+										AtLeastOneOf: []string{
+											"cluster_config.0.preemptible_worker_config.0.num_instances",
+											"cluster_config.0.preemptible_worker_config.0.preemptibility",
+											"cluster_config.0.preemptible_worker_config.0.disk_config",
+										},
+										ForceNew:     true,
+										ValidateFunc: validation.StringInSlice([]string{"PREEMPTIBILITY_UNSPECIFIED", "NON_PREEMPTIBLE", "PREEMPTIBLE"}, false),
+										Default:      "PREEMPTIBLE",
+									},
+
 									"disk_config": {
 										Type:        schema.TypeList,
 										Optional:    true,
 										Computed:    true,
 										Description: `Disk Config`,
 										AtLeastOneOf: []string{
 											"cluster_config.0.preemptible_worker_config.0.num_instances",
+											"cluster_config.0.preemptible_worker_config.0.preemptibility",
 											"cluster_config.0.preemptible_worker_config.0.disk_config",
 										},
 										MaxItems: 1,
@@ -891,9 +907,6 @@ func expandClusterConfig(d *schema.ResourceData, config *Config) (*dataproc.Clus
 	if cfg, ok := configOptions(d, "cluster_config.0.preemptible_worker_config"); ok {
 		log.Println("[INFO] got preemptible worker config")
 		conf.SecondaryWorkerConfig = expandPreemptibleInstanceGroupConfig(cfg)
-		if conf.SecondaryWorkerConfig.NumInstances > 0 {
-			conf.SecondaryWorkerConfig.IsPreemptible = true
-		}
 	}
 	return conf, nil
 }
@@ -1101,6 +1114,9 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc.
 			}
 		}
 	}
+	if p, ok := cfg["preemptibility"]; ok {
+		icg.Preemptibility = p.(string)
+	}
 	return icg
 }
 
@@ -1473,6 +1489,7 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc
 	if icg != nil {
 		data["num_instances"] = icg.NumInstances
 		data["instance_names"] = icg.InstanceNames
+		data["preemptibility"] = icg.Preemptibility
 		if icg.DiskConfig != nil {
 			disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
 			disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds

diff --git a/google/resource_dataproc_cluster_test.go b/google/resource_dataproc_cluster_test.go
@@ -391,6 +391,27 @@ func TestAccDataprocCluster_updatable(t *testing.T) {
 	})
 }
 
+func TestAccDataprocCluster_nonPreemptibleSecondary(t *testing.T) {
+	t.Parallel()
+
+	rnd := randString(t, 10)
+	var cluster dataproc.Cluster
+	vcrTest(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckDataprocClusterDestroy(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataprocCluster_nonPreemptibleSecondary(rnd),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.non_preemptible_secondary", &cluster),
+					resource.TestCheckResourceAttr("google_dataproc_cluster.non_preemptible_secondary", "cluster_config.0.preemptible_worker_config.0.preemptibility", "NON_PREEMPTIBLE"),
+				),
+			},
+		},
+	})
+}
+
 func TestAccDataprocCluster_withStagingBucket(t *testing.T) {
 	t.Parallel()
 
@@ -1220,6 +1241,41 @@ resource "google_dataproc_cluster" "updatable" {
 `, rnd, w, p)
 }
 
+func testAccDataprocCluster_nonPreemptibleSecondary(rnd string) string {
+	return fmt.Sprintf(`
+resource "google_dataproc_cluster" "non_preemptible_secondary" {
+  name   = "tf-test-dproc-%s"
+  region = "us-central1"
+
+  cluster_config {
+    master_config {
+	  num_instances = "1"
+	  machine_type  = "e2-medium"
+	  disk_config {
+		boot_disk_size_gb = 35
+	  }
+	}
+  
+	worker_config {
+	  num_instances = "2"
+	  machine_type  = "e2-medium"
+	  disk_config {
+		boot_disk_size_gb = 35
+	  }
+	}
+  
+	preemptible_worker_config {
+	  num_instances = "1"
+	  preemptibility = "NON_PREEMPTIBLE"
+	  disk_config {
+		boot_disk_size_gb = 35
+	  }
+	}
+  }
+}
+	`, rnd)
+}
+
 func testAccDataprocCluster_withStagingBucketOnly(bucketName string) string {
 	return fmt.Sprintf(`
 resource "google_storage_bucket" "bucket" {

diff --git a/website/docs/r/dataproc_cluster.html.markdown b/website/docs/r/dataproc_cluster.html.markdown
@@ -443,6 +443,12 @@ will be set for you based on whatever was set for the `worker_config.machine_typ
 * `num_instances`- (Optional) Specifies the number of preemptible nodes to create.
    Defaults to 0.
 
+* `preemptibility`- (Optional) Specifies the preemptibility of the secondary workers. The default value is `PREEMPTIBLE`
+  Accepted values are:
+  * PREEMPTIBILITY_UNSPECIFIED
+  * NON_PREEMPTIBLE
+  * PREEMPTIBLE
+
 * `disk_config` (Optional) Disk Config
 
     * `boot_disk_type` - (Optional) The disk type of the primary disk attached to each preemptible worker node.