Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding resilience_mode field support to Cloud Composer #8058

Merged
merged 21 commits into from
Jun 20, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ var (
"config.0.workloads_config",
"config.0.environment_size",
"config.0.master_authorized_networks_config",
"config.0.resilience_mode",
}

recoveryConfigKeys = []string{
Expand Down Expand Up @@ -802,14 +803,23 @@ func ResourceComposerEnvironment() *schema.Resource {
},
},
"environment_size": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: false,
AtLeastOneOf: composerConfigKeys,
ValidateFunc: validation.StringInSlice([]string{"ENVIRONMENT_SIZE_SMALL", "ENVIRONMENT_SIZE_MEDIUM", "ENVIRONMENT_SIZE_LARGE"}, false),
Description: `The size of the Cloud Composer environment. This field is supported for Cloud Composer environments in versions composer-2.*.*-airflow-*.*.* and newer.`,
},
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: false,
AtLeastOneOf: composerConfigKeys,
ValidateFunc: validation.StringInSlice([]string{"ENVIRONMENT_SIZE_SMALL", "ENVIRONMENT_SIZE_MEDIUM", "ENVIRONMENT_SIZE_LARGE"}, false),
Description: `The size of the Cloud Composer environment. This field is supported for Cloud Composer environments in versions composer-2.*.*-airflow-*.*.* and newer.`,
},
"resilience_mode": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
AtLeastOneOf: composerConfigKeys,
ValidateFunc: validation.StringInSlice([]string{"HIGH_RESILIENCE"}, false),
Description: `Whether high resilience is enabled or not. This field is supported for Cloud Composer environments in versions composer-2.1.15-airflow-*.*.* and newer.`,
},
"master_authorized_networks_config": {
Type: schema.TypeList,
Optional: true,
Expand Down Expand Up @@ -1310,6 +1320,7 @@ func flattenComposerEnvironmentConfig(envCfg *composer.EnvironmentConfig) interf
transformed["workloads_config"] = flattenComposerEnvironmentConfigWorkloadsConfig(envCfg.WorkloadsConfig)
transformed["recovery_config"] = flattenComposerEnvironmentConfigRecoveryConfig(envCfg.RecoveryConfig)
transformed["environment_size"] = envCfg.EnvironmentSize
transformed["resilience_mode"] = envCfg.ResilienceMode
transformed["master_authorized_networks_config"] = flattenComposerEnvironmentConfigMasterAuthorizedNetworksConfig(envCfg.MasterAuthorizedNetworksConfig)
return []interface{}{transformed}
}
Expand Down Expand Up @@ -1666,6 +1677,13 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf
return nil, err
}
transformed.EnvironmentSize = transformedEnvironmentSize

transformedResilienceMode, err := expandComposerEnvironmentConfigResilienceMode(original["resilience_mode"], d, config)
if err != nil {
return nil, err
}
transformed.ResilienceMode = transformedResilienceMode

transformedMasterAuthorizedNetworksConfig, err := expandComposerEnvironmentConfigMasterAuthorizedNetworksConfig(original["master_authorized_networks_config"], d, config)
if err != nil {
return nil, err
Expand Down Expand Up @@ -1903,6 +1921,13 @@ func expandComposerEnvironmentConfigEnvironmentSize(v interface{}, d *schema.Res
return v.(string), nil
}

func expandComposerEnvironmentConfigResilienceMode(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) (string, error) {
if v == nil {
return "", nil
}
return v.(string), nil
}

func expandComposerEnvironmentConfigPrivateEnvironmentConfig(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) (*composer.PrivateEnvironmentConfig, error) {
l := v.([]interface{})
if len(l) == 0 {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,40 @@ func TestAccComposerEnvironment_ComposerV2(t *testing.T) {
})
}


func TestAccComposerEnvironment_ComposerV2HighResilience(t *testing.T) {
Khaledmohamedrefaat marked this conversation as resolved.
Show resolved Hide resolved
t.Parallel()

envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, RandInt(t))
network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, RandInt(t))
subnetwork := network + "-1"

VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: ProtoV5ProviderFactories(t),
CheckDestroy: testAccComposerEnvironmentDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccComposerEnvironment_composerV2HighResilience(envName, network, subnetwork),
},
{
ResourceName: "google_composer_environment.test",
ImportState: true,
ImportStateVerify: true,
},
// This is a terrible clean-up step in order to get destroy to succeed,
// due to dangling firewall rules left by the Composer Environment blocking network deletion.
// TODO(dzarmola): Remove this check if firewall rules bug gets fixed by Composer.
{
PlanOnly: true,
Khaledmohamedrefaat marked this conversation as resolved.
Show resolved Hide resolved
ExpectNonEmptyPlan: false,
Config: testAccComposerEnvironment_composerV2HighResilience(envName, network, subnetwork),
Check: testAccCheckClearComposerEnvironmentFirewalls(t, network),
},
},
})
}

<% unless version == "ga" -%>
func TestAccComposerEnvironment_UpdateComposerV2WithTriggerer(t *testing.T) {
// TODO: This test was seemingly working, but then started to re-run on every gcbrun https://github.com/hashicorp/terraform-provider-google/issues/14160
Expand Down Expand Up @@ -1648,6 +1682,69 @@ resource "google_compute_subnetwork" "test" {
`, envName, network, subnetwork)
}

func testAccComposerEnvironment_composerV2HighResilience(envName, network, subnetwork string) string {
return fmt.Sprintf(`
resource "google_composer_environment" "test" {
name = "%s"
region = "us-east1"
Khaledmohamedrefaat marked this conversation as resolved.
Show resolved Hide resolved

config {
node_config {
network = google_compute_network.test.self_link
subnetwork = google_compute_subnetwork.test.self_link
}

software_config {
image_version = "composer-2-airflow-2"
}

workloads_config {
scheduler {
cpu = 1.25
memory_gb = 2.5
storage_gb = 5.4
count = 2
}
web_server {
cpu = 1.75
memory_gb = 3.0
storage_gb = 4.4
}
worker {
cpu = 0.5
memory_gb = 2.0
storage_gb = 3.4
min_count = 2
max_count = 5
}
}
environment_size = "ENVIRONMENT_SIZE_MEDIUM"
resilience_mode = "HIGH_RESILIENCE"
private_environment_config {
enable_private_endpoint = true
cloud_composer_network_ipv4_cidr_block = "10.3.192.0/24"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm a little out of my depth so want to make sure: is there any risk of these cidr ranges conflicting with other resources in our various tests environments? (for example, if this test runs at the same time as the test above, testAccComposerEnvironment_composerV2)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if there is a risk. Nonetheless, these IP ranges have been duplicated several times already in these tests. I expect that duplicating them once more shouldn't change much.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh alright, I hadn't noticed that, but agreed it shouldn't be an issue then.

master_ipv4_cidr_block = "172.16.194.0/23"
cloud_sql_ipv4_cidr_block = "10.3.224.0/20"
}
}
}

resource "google_compute_network" "test" {
name = "%s"
auto_create_subnetworks = false
}

resource "google_compute_subnetwork" "test" {
name = "%s"
ip_cidr_range = "10.2.0.0/16"
region = "us-east1"
network = google_compute_network.test.self_link
private_ip_google_access = true
}

`, envName, network, subnetwork)
}

func testAccComposerEnvironment_composerV2PrivateServiceConnect(envName, network, subnetwork string) string {
return fmt.Sprintf(`
resource "google_composer_environment" "test" {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -679,6 +679,12 @@ The `config` block supports:
environment size are `ENVIRONMENT_SIZE_SMALL`, `ENVIRONMENT_SIZE_MEDIUM`,
and `ENVIRONMENT_SIZE_LARGE`.

* `resilience_mode` -
(Optional, Cloud Composer 2.1.15 or newer only)
The resilience mode states whether high resilience is enabled for
the environment or not. Value for resilience mode is `HIGH_RESILIENCE`.
If unspecified, defaults to standard resilience.

* `master_authorized_networks_config` -
(Optional)
Configuration options for the master authorized networks feature. Enabled
Expand Down