From 7a1de3349a66f90015220c504ea87dbaeaef4e8a Mon Sep 17 00:00:00 2001 From: Tom Samaras Date: Mon, 7 Mar 2022 13:12:11 -0500 Subject: [PATCH] Add support for non-preemptible secondary dataproc workers (#5686) Co-authored-by: Matthew Barnes Co-authored-by: Stephen Lewis (Burrows) Co-authored-by: Riley Karson Co-authored-by: Scott Suarez Co-authored-by: Sampath Kumar Co-authored-by: Sam Levenick Co-authored-by: John Pellman Co-authored-by: Jacek Kikiewicz Co-authored-by: Alex Ellis Co-authored-by: megan07 Co-authored-by: Daniel Randell Co-authored-by: Iris Chen <10179943+iyabchen@users.noreply.github.com> Co-authored-by: prateek2408 --- mmv1/products/dataproc/api.yaml | 33 +++++++++++ .../resource_dataproc_cluster.go.erb | 23 +++++++- .../resource_dataproc_cluster_test.go.erb | 56 +++++++++++++++++++ .../docs/r/dataproc_cluster.html.markdown | 6 ++ 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/mmv1/products/dataproc/api.yaml b/mmv1/products/dataproc/api.yaml index 4ed026314f1c..ed575636ecc3 100644 --- a/mmv1/products/dataproc/api.yaml +++ b/mmv1/products/dataproc/api.yaml @@ -326,8 +326,19 @@ objects: Number of attached SSDs, from 0 to 4. - !ruby/object:Api::Type::Boolean name: 'isPreemptible' + output: true description: | Specifies if this instance group contains preemptible instances. + - !ruby/object:Api::Type::Enum + name: 'preemptibility' + description: | + Specifies the preemptibility of the instance group. + The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. + The default value for secondary instances is PREEMPTIBLE. + values: + - :PREEMPTIBILITY_UNSPECIFIED + - :NON_PREEMPTIBLE + - :PREEMPTIBLE - !ruby/object:Api::Type::NestedObject name: 'managedGroupConfig' output: true @@ -387,8 +398,19 @@ objects: Number of attached SSDs, from 0 to 4. - !ruby/object:Api::Type::Boolean name: 'isPreemptible' + output: true description: | Specifies if this instance group contains preemptible instances. + - !ruby/object:Api::Type::Enum + name: 'preemptibility' + description: | + Specifies the preemptibility of the instance group. + The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. + The default value for secondary instances is PREEMPTIBLE. + values: + - :PREEMPTIBILITY_UNSPECIFIED + - :NON_PREEMPTIBLE + - :PREEMPTIBLE - !ruby/object:Api::Type::NestedObject name: 'managedGroupConfig' output: true @@ -448,8 +470,19 @@ objects: Number of attached SSDs, from 0 to 4. - !ruby/object:Api::Type::Boolean name: 'isPreemptible' + output: true description: | Specifies if this instance group contains preemptible instances. + - !ruby/object:Api::Type::Enum + name: 'preemptibility' + description: | + Specifies the preemptibility of the instance group. + The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed. + The default value for secondary instances is PREEMPTIBLE. + values: + - :PREEMPTIBILITY_UNSPECIFIED + - :NON_PREEMPTIBLE + - :PREEMPTIBLE - !ruby/object:Api::Type::NestedObject name: 'managedGroupConfig' output: true diff --git a/mmv1/third_party/terraform/resources/resource_dataproc_cluster.go.erb b/mmv1/third_party/terraform/resources/resource_dataproc_cluster.go.erb index 1c2865888f53..7a47afc50837 100644 --- a/mmv1/third_party/terraform/resources/resource_dataproc_cluster.go.erb +++ b/mmv1/third_party/terraform/resources/resource_dataproc_cluster.go.erb @@ -350,6 +350,7 @@ func resourceDataprocCluster() *schema.Resource { Description: `Specifies the number of preemptible nodes to create. Defaults to 0.`, AtLeastOneOf: []string{ "cluster_config.0.preemptible_worker_config.0.num_instances", + "cluster_config.0.preemptible_worker_config.0.preemptibility", "cluster_config.0.preemptible_worker_config.0.disk_config", }, }, @@ -358,6 +359,20 @@ func resourceDataprocCluster() *schema.Resource { // It always uses whatever is specified for the worker_config // "machine_type": { ... } // "min_cpu_platform": { ... } + "preemptibility": { + Type: schema.TypeString, + Optional: true, + Description: `Specifies the preemptibility of the secondary nodes. Defaults to PREEMPTIBLE.`, + AtLeastOneOf: []string{ + "cluster_config.0.preemptible_worker_config.0.num_instances", + "cluster_config.0.preemptible_worker_config.0.preemptibility", + "cluster_config.0.preemptible_worker_config.0.disk_config", + }, + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"PREEMPTIBILITY_UNSPECIFIED", "NON_PREEMPTIBLE", "PREEMPTIBLE"}, false), + Default: "PREEMPTIBLE", + }, + "disk_config": { Type: schema.TypeList, Optional: true, @@ -365,6 +380,7 @@ func resourceDataprocCluster() *schema.Resource { Description: `Disk Config`, AtLeastOneOf: []string{ "cluster_config.0.preemptible_worker_config.0.num_instances", + "cluster_config.0.preemptible_worker_config.0.preemptibility", "cluster_config.0.preemptible_worker_config.0.disk_config", }, MaxItems: 1, @@ -996,9 +1012,6 @@ func expandClusterConfig(d *schema.ResourceData, config *Config) (*dataproc.Clus if cfg, ok := configOptions(d, "cluster_config.0.preemptible_worker_config"); ok { log.Println("[INFO] got preemptible worker config") conf.SecondaryWorkerConfig = expandPreemptibleInstanceGroupConfig(cfg) - if conf.SecondaryWorkerConfig.NumInstances > 0 { - conf.SecondaryWorkerConfig.IsPreemptible = true - } } return conf, nil } @@ -1235,6 +1248,9 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc. } } } + if p, ok := cfg["preemptibility"]; ok { + icg.Preemptibility = p.(string) + } return icg } @@ -1676,6 +1692,7 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc if icg != nil { data["num_instances"] = icg.NumInstances data["instance_names"] = icg.InstanceNames + data["preemptibility"] = icg.Preemptibility if icg.DiskConfig != nil { disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds diff --git a/mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb b/mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb index cfe20777def9..0f2e69c153bf 100644 --- a/mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb +++ b/mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb @@ -399,6 +399,27 @@ func TestAccDataprocCluster_updatable(t *testing.T) { }) } +func TestAccDataprocCluster_nonPreemptibleSecondary(t *testing.T) { + t.Parallel() + + rnd := randString(t, 10) + var cluster dataproc.Cluster + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataprocClusterDestroy(t), + Steps: []resource.TestStep{ + { + Config: testAccDataprocCluster_nonPreemptibleSecondary(rnd), + Check: resource.ComposeTestCheckFunc( + testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.non_preemptible_secondary", &cluster), + resource.TestCheckResourceAttr("google_dataproc_cluster.non_preemptible_secondary", "cluster_config.0.preemptible_worker_config.0.preemptibility", "NON_PREEMPTIBLE"), + ), + }, + }, + }) +} + func TestAccDataprocCluster_withStagingBucket(t *testing.T) { t.Parallel() @@ -1346,6 +1367,41 @@ resource "google_dataproc_cluster" "updatable" { `, rnd, w, p) } +func testAccDataprocCluster_nonPreemptibleSecondary(rnd string) string { + return fmt.Sprintf(` +resource "google_dataproc_cluster" "non_preemptible_secondary" { + name = "tf-test-dproc-%s" + region = "us-central1" + + cluster_config { + master_config { + num_instances = "1" + machine_type = "e2-medium" + disk_config { + boot_disk_size_gb = 35 + } + } + + worker_config { + num_instances = "2" + machine_type = "e2-medium" + disk_config { + boot_disk_size_gb = 35 + } + } + + preemptible_worker_config { + num_instances = "1" + preemptibility = "NON_PREEMPTIBLE" + disk_config { + boot_disk_size_gb = 35 + } + } + } +} + `, rnd) +} + func testAccDataprocCluster_withStagingBucketOnly(bucketName string) string { return fmt.Sprintf(` resource "google_storage_bucket" "bucket" { diff --git a/mmv1/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown index 071d281e791c..f339b5eec0cf 100644 --- a/mmv1/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown @@ -443,6 +443,12 @@ will be set for you based on whatever was set for the `worker_config.machine_typ * `num_instances`- (Optional) Specifies the number of preemptible nodes to create. Defaults to 0. +* `preemptibility`- (Optional) Specifies the preemptibility of the secondary workers. The default value is `PREEMPTIBLE` + Accepted values are: + * PREEMPTIBILITY_UNSPECIFIED + * NON_PREEMPTIBLE + * PREEMPTIBLE + * `disk_config` (Optional) Disk Config * `boot_disk_type` - (Optional) The disk type of the primary disk attached to each preemptible worker node.