Skip to content

Commit

Permalink
Add support for non-preemptible secondary dataproc workers (GoogleClo…
Browse files Browse the repository at this point in the history 
…udPlatform#5686)

Co-authored-by: Matthew Barnes <matthew.simon.barnes@gmail.com>
Co-authored-by: Stephen Lewis (Burrows) <stephenrlewis@google.com>
Co-authored-by: Riley Karson <rileykarson@google.com>
Co-authored-by: Scott Suarez <ScottMSuarez@gmail.com>
Co-authored-by: Sampath Kumar <sampathm@google.com>
Co-authored-by: Sam Levenick <slevenick@google.com>
Co-authored-by: John Pellman <pellman.john@gmail.com>
Co-authored-by: Jacek Kikiewicz <jaceq@users.noreply.github.com>
Co-authored-by: Alex Ellis <alexellis@google.com>
Co-authored-by: megan07 <mbang@hashicorp.com>
Co-authored-by: Daniel Randell <drandell@users.noreply.github.com>
Co-authored-by: Iris Chen <10179943+iyabchen@users.noreply.github.com>
Co-authored-by: prateek2408 <prateek.khushalani@gmail.com>
  • Loading branch information
@tsamaras @MatthewSBarnes
@melinath @rileykarson @ScottSuarez @msampathkumar @slevenick @jpellman @jaceq @megan07 @drandell @iyabchen @prateek2408 @lcaggio
14 people authored and lcaggio committed Mar 17, 2022
1 parent ed78ae9 commit 7a1de33
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 3 deletions.
33 changes: 33 additions & 0 deletions mmv1/products/dataproc/api.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,8 +326,19 @@ objects:
Number of attached SSDs, from 0 to 4.
- !ruby/object:Api::Type::Boolean
name: 'isPreemptible'
output: true
description: |
Specifies if this instance group contains preemptible instances.
- !ruby/object:Api::Type::Enum
name: 'preemptibility'
description: |
Specifies the preemptibility of the instance group.
The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed.
The default value for secondary instances is PREEMPTIBLE.
values:
- :PREEMPTIBILITY_UNSPECIFIED
- :NON_PREEMPTIBLE
- :PREEMPTIBLE
- !ruby/object:Api::Type::NestedObject
name: 'managedGroupConfig'
output: true
Expand Down Expand Up @@ -387,8 +398,19 @@ objects:
Number of attached SSDs, from 0 to 4.
- !ruby/object:Api::Type::Boolean
name: 'isPreemptible'
output: true
description: |
Specifies if this instance group contains preemptible instances.
- !ruby/object:Api::Type::Enum
name: 'preemptibility'
description: |
Specifies the preemptibility of the instance group.
The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed.
The default value for secondary instances is PREEMPTIBLE.
values:
- :PREEMPTIBILITY_UNSPECIFIED
- :NON_PREEMPTIBLE
- :PREEMPTIBLE
- !ruby/object:Api::Type::NestedObject
name: 'managedGroupConfig'
output: true
Expand Down Expand Up @@ -448,8 +470,19 @@ objects:
Number of attached SSDs, from 0 to 4.
- !ruby/object:Api::Type::Boolean
name: 'isPreemptible'
output: true
description: |
Specifies if this instance group contains preemptible instances.
- !ruby/object:Api::Type::Enum
name: 'preemptibility'
description: |
Specifies the preemptibility of the instance group.
The default value for master and worker groups is NON_PREEMPTIBLE. This default cannot be changed.
The default value for secondary instances is PREEMPTIBLE.
values:
- :PREEMPTIBILITY_UNSPECIFIED
- :NON_PREEMPTIBLE
- :PREEMPTIBLE
- !ruby/object:Api::Type::NestedObject
name: 'managedGroupConfig'
output: true
Expand Down
23 changes: 20 additions & 3 deletions mmv1/third_party/terraform/resources/resource_dataproc_cluster.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ func resourceDataprocCluster() *schema.Resource {
Description: `Specifies the number of preemptible nodes to create. Defaults to 0.`,
AtLeastOneOf: []string{
"cluster_config.0.preemptible_worker_config.0.num_instances",
"cluster_config.0.preemptible_worker_config.0.preemptibility",
"cluster_config.0.preemptible_worker_config.0.disk_config",
},
},
Expand All @@ -358,13 +359,28 @@ func resourceDataprocCluster() *schema.Resource {
// It always uses whatever is specified for the worker_config
// "machine_type": { ... }
// "min_cpu_platform": { ... }
"preemptibility": {
Type: schema.TypeString,
Optional: true,
Description: `Specifies the preemptibility of the secondary nodes. Defaults to PREEMPTIBLE.`,
AtLeastOneOf: []string{
"cluster_config.0.preemptible_worker_config.0.num_instances",
"cluster_config.0.preemptible_worker_config.0.preemptibility",
"cluster_config.0.preemptible_worker_config.0.disk_config",
},
ForceNew: true,
ValidateFunc: validation.StringInSlice([]string{"PREEMPTIBILITY_UNSPECIFIED", "NON_PREEMPTIBLE", "PREEMPTIBLE"}, false),
Default: "PREEMPTIBLE",
},

"disk_config": {
Type: schema.TypeList,
Optional: true,
Computed: true,
Description: `Disk Config`,
AtLeastOneOf: []string{
"cluster_config.0.preemptible_worker_config.0.num_instances",
"cluster_config.0.preemptible_worker_config.0.preemptibility",
"cluster_config.0.preemptible_worker_config.0.disk_config",
},
MaxItems: 1,
Expand Down Expand Up @@ -996,9 +1012,6 @@ func expandClusterConfig(d *schema.ResourceData, config *Config) (*dataproc.Clus
if cfg, ok := configOptions(d, "cluster_config.0.preemptible_worker_config"); ok {
log.Println("[INFO] got preemptible worker config")
conf.SecondaryWorkerConfig = expandPreemptibleInstanceGroupConfig(cfg)
if conf.SecondaryWorkerConfig.NumInstances > 0 {
conf.SecondaryWorkerConfig.IsPreemptible = true
}
}
return conf, nil
}
Expand Down Expand Up @@ -1235,6 +1248,9 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc.
}
}
}
if p, ok := cfg["preemptibility"]; ok {
icg.Preemptibility = p.(string)
}
return icg
}

Expand Down Expand Up @@ -1676,6 +1692,7 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc
if icg != nil {
data["num_instances"] = icg.NumInstances
data["instance_names"] = icg.InstanceNames
data["preemptibility"] = icg.Preemptibility
if icg.DiskConfig != nil {
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
Expand Down
56 changes: 56 additions & 0 deletions mmv1/third_party/terraform/tests/resource_dataproc_cluster_test.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,27 @@ func TestAccDataprocCluster_updatable(t *testing.T) {
})
}

func TestAccDataprocCluster_nonPreemptibleSecondary(t *testing.T) {
t.Parallel()

rnd := randString(t, 10)
var cluster dataproc.Cluster
vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckDataprocClusterDestroy(t),
Steps: []resource.TestStep{
{
Config: testAccDataprocCluster_nonPreemptibleSecondary(rnd),
Check: resource.ComposeTestCheckFunc(
testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.non_preemptible_secondary", &cluster),
resource.TestCheckResourceAttr("google_dataproc_cluster.non_preemptible_secondary", "cluster_config.0.preemptible_worker_config.0.preemptibility", "NON_PREEMPTIBLE"),
),
},
},
})
}

func TestAccDataprocCluster_withStagingBucket(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -1346,6 +1367,41 @@ resource "google_dataproc_cluster" "updatable" {
`, rnd, w, p)
}

func testAccDataprocCluster_nonPreemptibleSecondary(rnd string) string {
return fmt.Sprintf(`
resource "google_dataproc_cluster" "non_preemptible_secondary" {
name = "tf-test-dproc-%s"
region = "us-central1"

cluster_config {
master_config {
num_instances = "1"
machine_type = "e2-medium"
disk_config {
boot_disk_size_gb = 35
}
}

worker_config {
num_instances = "2"
machine_type = "e2-medium"
disk_config {
boot_disk_size_gb = 35
}
}

preemptible_worker_config {
num_instances = "1"
preemptibility = "NON_PREEMPTIBLE"
disk_config {
boot_disk_size_gb = 35
}
}
}
}
`, rnd)
}

func testAccDataprocCluster_withStagingBucketOnly(bucketName string) string {
return fmt.Sprintf(`
resource "google_storage_bucket" "bucket" {
Expand Down
6 changes: 6 additions & 0 deletions mmv1/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,12 @@ will be set for you based on whatever was set for the `worker_config.machine_typ
* `num_instances`- (Optional) Specifies the number of preemptible nodes to create.
Defaults to 0.

* `preemptibility`- (Optional) Specifies the preemptibility of the secondary workers. The default value is `PREEMPTIBLE`
Accepted values are:
* PREEMPTIBILITY_UNSPECIFIED
* NON_PREEMPTIBLE
* PREEMPTIBLE

* `disk_config` (Optional) Disk Config

* `boot_disk_type` - (Optional) The disk type of the primary disk attached to each preemptible worker node.
Expand Down

0 comments on commit 7a1de33

Please sign in to comment.