Add Scheduled snapshots for CC

.changelog/7065.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+composer: added `recovery_config` in `google_composer_environment` resource
+```

google/resource_composer_environment.go

@@ -50,6 +50,7 @@ var (
 		"config.0.node_count",
 		"config.0.node_config",
 		"config.0.software_config",
+		"config.0.recovery_config",
 		"config.0.private_environment_config",
 		"config.0.web_server_network_access_control",
 		"config.0.database_config",
@@ -61,6 +62,10 @@ var (
 		"config.0.master_authorized_networks_config",
 	}
 
+	recoveryConfigKeys = []string{
+		"config.0.recovery_config.0.scheduled_snapshots_config",
+	}
+
 	workloadsConfigKeys = []string{
 		"config.0.workloads_config.0.scheduler",
 		"config.0.workloads_config.0.web_server",
@@ -314,6 +319,48 @@ func resourceComposerEnvironment() *schema.Resource {
 								},
 							},
 						},
+						"recovery_config": {
+							Type:         schema.TypeList,
+							Optional:     true,
+							AtLeastOneOf: composerConfigKeys,
+							MaxItems:     1,
+							Description:  `The recovery configuration settings for the Cloud Composer environment`,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"scheduled_snapshots_config": {
+										Type:         schema.TypeList,
+										Optional:     true,
+										AtLeastOneOf: recoveryConfigKeys,
+										Description:  `The configuration settings for scheduled snapshots.`,
+										MaxItems:     1,
+										Elem: &schema.Resource{
+											Schema: map[string]*schema.Schema{
+												"enabled": {
+													Type:        schema.TypeBool,
+													Required:    true,
+													Description: `When enabled, Cloud Composer periodically saves snapshots of your environment to a Cloud Storage bucket.`,
+												},
+												"snapshot_location": {
+													Type:        schema.TypeString,
+													Optional:    true,
+													Description: `the URI of a bucket folder where to save the snapshot.`,
+												},
+												"snapshot_creation_schedule": {
+													Type:        schema.TypeString,
+													Optional:    true,
+													Description: `Snapshot schedule, in the unix-cron format.`,
+												},
+												"time_zone": {
+													Type:        schema.TypeString,
+													Optional:    true,
+													Description: `A time zone for the schedule. This value is a time offset and does not take into account daylight saving time changes. Valid values are from UTC-12 to UTC+12. Examples: UTC, UTC-01, UTC+03.`,
+												},
+											},
+										},
+									},
+								},
+							},
+						},
 						"software_config": {
 							Type:         schema.TypeList,
 							Optional:     true,
@@ -996,6 +1043,18 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{})
 				return err
 			}
 		}
+
+		if d.HasChange("config.0.recovery_config.0.scheduled_snapshots_config") {
+			patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
+			if config != nil {
+				patchObj.Config.RecoveryConfig = config.RecoveryConfig
+			}
+			err = resourceComposerEnvironmentPatchField("config.RecoveryConfig.ScheduledSnapshotsConfig", userAgent, patchObj, d, tfConfig)
+			if err != nil {
+				return err
+			}
+		}
+
 		if d.HasChange("config.0.environment_size") {
 			patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
 			if config != nil {
@@ -1140,6 +1199,7 @@ func flattenComposerEnvironmentConfig(envCfg *composer.EnvironmentConfig) interf
 	transformed["encryption_config"] = flattenComposerEnvironmentConfigEncryptionConfig(envCfg.EncryptionConfig)
 	transformed["maintenance_window"] = flattenComposerEnvironmentConfigMaintenanceWindow(envCfg.MaintenanceWindow)
 	transformed["workloads_config"] = flattenComposerEnvironmentConfigWorkloadsConfig(envCfg.WorkloadsConfig)
+	transformed["recovery_config"] = flattenComposerEnvironmentConfigRecoveryConfig(envCfg.RecoveryConfig)
 	transformed["environment_size"] = envCfg.EnvironmentSize
 	transformed["master_authorized_networks_config"] = flattenComposerEnvironmentConfigMasterAuthorizedNetworksConfig(envCfg.MasterAuthorizedNetworksConfig)
 	return []interface{}{transformed}
@@ -1199,6 +1259,30 @@ func flattenComposerEnvironmentConfigEncryptionConfig(encryptionCfg *composer.En
 	return []interface{}{transformed}
 }
 
+func flattenComposerEnvironmentConfigRecoveryConfig(recoveryCfg *composer.RecoveryConfig) interface{} {
+	if recoveryCfg == nil {
+		return nil
+	}
+
+	transformed := make(map[string]interface{})
+	transformedScheduledSnapshotsConfig := make(map[string]interface{})
+
+	scheduledSnapshotsConfig := recoveryCfg.ScheduledSnapshotsConfig
+
+	if scheduledSnapshotsConfig == nil {
+		transformedScheduledSnapshotsConfig = nil
+	} else {
+		transformedScheduledSnapshotsConfig["enabled"] = scheduledSnapshotsConfig.Enabled
+		transformedScheduledSnapshotsConfig["snapshot_location"] = scheduledSnapshotsConfig.SnapshotLocation
+		transformedScheduledSnapshotsConfig["time_zone"] = scheduledSnapshotsConfig.TimeZone
+		transformedScheduledSnapshotsConfig["snapshot_creation_schedule"] = scheduledSnapshotsConfig.SnapshotCreationSchedule
+	}
+
+	transformed["scheduled_snapshots_config"] = []interface{}{transformedScheduledSnapshotsConfig}
+
+	return []interface{}{transformed}
+}
+
 func flattenComposerEnvironmentConfigMaintenanceWindow(maintenanceWindow *composer.MaintenanceWindow) interface{} {
 	if maintenanceWindow == nil {
 		return nil
@@ -1437,6 +1521,13 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf
 		return nil, err
 	}
 	transformed.MasterAuthorizedNetworksConfig = transformedMasterAuthorizedNetworksConfig
+
+	transformedRecoveryConfig, err := expandComposerEnvironmentConfigRecoveryConfig(original["recovery_config"], d, config)
+	if err != nil {
+		return nil, err
+	}
+	transformed.RecoveryConfig = transformedRecoveryConfig
+
 	return transformed, nil
 }
 
@@ -1617,6 +1708,30 @@ func expandComposerEnvironmentConfigWorkloadsConfig(v interface{}, d *schema.Res
 	return transformed, nil
 }
 
+func expandComposerEnvironmentConfigRecoveryConfig(v interface{}, d *schema.ResourceData, config *Config) (*composer.RecoveryConfig, error) {
+	l := v.([]interface{})
+	if len(l) == 0 {
+		return nil, nil
+	}
+	raw := l[0]
+	original := raw.(map[string]interface{})
+	transformed := &composer.RecoveryConfig{}
+
+	if v, ok := original["scheduled_snapshots_config"]; ok {
+		if len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil {
+			transformedScheduledSnapshotsConfig := &composer.ScheduledSnapshotsConfig{}
+			originalScheduledSnapshotsConfigRaw := v.([]interface{})[0].(map[string]interface{})
+			transformedScheduledSnapshotsConfig.Enabled = originalScheduledSnapshotsConfigRaw["enabled"].(bool)
+			transformedScheduledSnapshotsConfig.SnapshotLocation = originalScheduledSnapshotsConfigRaw["snapshot_location"].(string)
+			transformedScheduledSnapshotsConfig.TimeZone = originalScheduledSnapshotsConfigRaw["time_zone"].(string)
+			transformedScheduledSnapshotsConfig.SnapshotCreationSchedule = originalScheduledSnapshotsConfigRaw["snapshot_creation_schedule"].(string)
+			transformed.ScheduledSnapshotsConfig = transformedScheduledSnapshotsConfig
+		}
+	}
+
+	return transformed, nil
+}
+
 func expandComposerEnvironmentConfigEnvironmentSize(v interface{}, d *schema.ResourceData, config *Config) (string, error) {
 	if v == nil {
 		return "", nil

google/resource_composer_environment_test.go

@@ -744,6 +744,46 @@ func TestAccComposerEnvironment_withNodeConfig(t *testing.T) {
 	})
 }
 
+func TestAccComposerEnvironmentAirflow2_withRecoveryConfig(t *testing.T) {
+	t.Parallel()
+	envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, randInt(t))
+	network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, randInt(t))
+	subnetwork := network + "-1"
+
+	vcrTest(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccComposerEnvironmentDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccComposerEnvironment_airflow2RecoveryCfg(envName, network, subnetwork),
+			},
+			{
+				ResourceName:      "google_composer_environment.test",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config: testAccComposerEnvironmentUpdate_airflow2RecoveryCfg(envName, network, subnetwork),
+			},
+			{
+				ResourceName:      "google_composer_environment.test",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			// This is a terrible clean-up step in order to get destroy to succeed,
+			// due to dangling firewall rules left by the Composer Environment blocking network deletion.
+			// TODO: Remove this check if firewall rules bug gets fixed by Composer.
+			{
+				PlanOnly:           true,
+				ExpectNonEmptyPlan: false,
+				Config:             testAccComposerEnvironmentUpdate_airflow2RecoveryCfg(envName, network, subnetwork),
+				Check:              testAccCheckClearComposerEnvironmentFirewalls(t, network),
+			},
+		},
+	})
+}
+
 func TestAccComposerEnvironment_withSoftwareConfig(t *testing.T) {
 	t.Parallel()
 	envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, randInt(t))
@@ -1792,6 +1832,98 @@ resource "google_project_iam_member" "composer-worker" {
 `, environment, network, subnetwork, serviceAccount)
 }
 
+func testAccComposerEnvironment_airflow2RecoveryCfg(name, network, subnetwork string) string {
+	return fmt.Sprintf(`
+resource "google_composer_environment" "test" {
+  name   = "%s"
+  region = "us-central1"
+
+  config {
+    node_config {
+      network          = google_compute_network.test.self_link
+      subnetwork       = google_compute_subnetwork.test.self_link
+      ip_allocation_policy {
+        cluster_ipv4_cidr_block = "10.0.0.0/16"
+      }
+    }
+
+    software_config {
+      image_version = "composer-2-airflow-2"
+    }
+
+    recovery_config {
+      scheduled_snapshots_config {
+        enabled =                    true
+        snapshot_location =          "gs://example-bucket/environment_snapshots"
+        snapshot_creation_schedule = "0 4 * * *"
+        time_zone =                  "UTC+01"
+      }
+    }
+  }
+
+}
+
+resource "google_compute_network" "test" {
+  name                    = "%s"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "test" {
+  name          = "%s"
+  ip_cidr_range = "10.2.0.0/16"
+  region        = "us-central1"
+  network       = google_compute_network.test.self_link
+  private_ip_google_access = true
+}
+`, name, network, subnetwork)
+}
+
+func testAccComposerEnvironmentUpdate_airflow2RecoveryCfg(name, network, subnetwork string) string {
+	return fmt.Sprintf(`
+resource "google_composer_environment" "test" {
+  name   = "%s"
+  region = "us-central1"
+
+  config {
+    node_config {
+      network          = google_compute_network.test.self_link
+      subnetwork       = google_compute_subnetwork.test.self_link
+      ip_allocation_policy {
+        cluster_ipv4_cidr_block = "10.0.0.0/16"
+      }
+    }
+
+    software_config {
+      image_version = "composer-2-airflow-2"
+    }
+
+    recovery_config {
+		  scheduled_snapshots_config {
+			  enabled =                    true
+			  snapshot_location =          "gs://example-bucket/environment_snapshots2"
+			  snapshot_creation_schedule = "1 2 * * *"
+			  time_zone =                  "UTC+02"
+      }
+    }
+  }
+
+}
+
+resource "google_compute_network" "test" {
+  name                    = "%s"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "test" {
+  name          = "%s"
+  ip_cidr_range = "10.2.0.0/16"
+  region        = "us-central1"
+  network       = google_compute_network.test.self_link
+  private_ip_google_access = true
+}
+`, name, network, subnetwork)
+}
+
 func testAccComposerEnvironment_softwareCfg(name, network, subnetwork string) string {
 	return fmt.Sprintf(`
 resource "google_composer_environment" "test" {

website/docs/r/composer_environment.html.markdown

@@ -281,6 +281,10 @@ The following arguments are supported:
   (Optional)
   The configuration used for the Kubernetes Engine cluster.  Structure is [documented below](#nested_node_config).
 
+* `recovery_config` -
+  (Optional, Cloud Composer 2 only)
+  The configuration settings for recovery. Structure is [documented below](#nested_recovery_config).
+
 * `software_config` -
   (Optional)
   The configuration settings for software inside the environment.  Structure is [documented below](#nested_software_config).
@@ -877,6 +881,30 @@ The `ip_allocation_policy` block supports:
   The only allowed values for 'FREQ' field are 'FREQ=DAILY' and 'FREQ=WEEKLY;BYDAY=...'.
   Example values: 'FREQ=WEEKLY;BYDAY=TU,WE', 'FREQ=DAILY'.
 
+The `recovery_config` block supports:
+
+* `scheduled_snapshots_config` -
+  (Optional)
+  The recovery configuration settings for the Cloud Composer environment.
+
+The `scheduled_snapshots_config` block supports:
+
+* `enabled` -
+  (Optional)
+  When enabled, Cloud Composer periodically saves snapshots of your environment to a Cloud Storage bucket.
+
+* `snapshot_location` -
+  (Optional)
+  The URI of a bucket folder where to save the snapshot.
+
+* `snapshot_creation_schedule` -
+  (Optional)
+  Snapshot schedule, in the unix-cron format.
+
+* `time_zone` -
+  (Optional)
+  A time zone for the schedule. This value is a time offset and does not take into account daylight saving time changes. Valid values are from UTC-12 to UTC+12. Examples: UTC, UTC-01, UTC+03.
+
 The `workloads_config` block supports:
 
 * `scheduler` -