Skip to content

Commit

Permalink
Composer v2 GA (#5470)
Browse files Browse the repository at this point in the history 
* work in progress, assure if everything works

* added support for composer v2 ga. Also some minor bugs in tests are fixed

* moved tests to us-east1 for some time
  • Loading branch information
@kubasieron
kubasieron authored Dec 23, 2021
1 parent 836bbbe commit cd44d20
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 36 deletions.
46 changes: 28 additions & 18 deletions mmv1/third_party/terraform/resources/resource_composer_environment.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,9 @@ var (
"config.0.web_server_config",
"config.0.encryption_config",
"config.0.maintenance_window",
<% end -%>
"config.0.workloads_config",
"config.0.environment_size",
<% end -%>
<% unless version == "ga" -%>
"config.0.master_authorized_networks_config",
<% end -%>
Expand All @@ -75,8 +75,8 @@ var (
"config.0.private_environment_config.0.master_ipv4_cidr_block",
"config.0.private_environment_config.0.cloud_sql_ipv4_cidr_block",
"config.0.private_environment_config.0.web_server_ipv4_cidr_block",
<% unless version == "ga" -%>
"config.0.private_environment_config.0.cloud_composer_network_ipv4_cidr_block",
<% unless version == "ga" -%>
"config.0.private_environment_config.0.enable_privately_used_public_ips",
"config.0.private_environment_config.0.cloud_composer_connection_subnetwork",
<% end -%>
Expand Down Expand Up @@ -433,7 +433,6 @@ func resourceComposerEnvironment() *schema.Resource {
ForceNew: true,
Description: `The CIDR block from which IP range in tenant project will be reserved for Cloud SQL. Needs to be disjoint from web_server_ipv4_cidr_block.`,
},
<% unless version == "ga" -%>
"cloud_composer_network_ipv4_cidr_block": {
Type: schema.TypeString,
Optional: true,
Expand All @@ -442,6 +441,7 @@ func resourceComposerEnvironment() *schema.Resource {
ForceNew: true,
Description: `The CIDR block from which IP range for Cloud Composer Network in tenant project will be reserved. Needs to be disjoint from private_cluster_config.master_ipv4_cidr_block and cloud_sql_ipv4_cidr_block. This field is supported for Cloud Composer environments in versions composer-2.*.*-airflow-*.*.* and newer.`,
},
<% unless version == "ga" -%>
"enable_privately_used_public_ips": {
Type: schema.TypeBool,
Optional: true,
Expand All @@ -458,7 +458,7 @@ func resourceComposerEnvironment() *schema.Resource {
ForceNew: true,
Description: `When specified, the environment will use Private Service Connect instead of VPC peerings to connect to Cloud SQL in the Tenant Project, and the PSC endpoint in the Customer Project will use an IP address from this subnetwork. This field is supported for Cloud Composer environments in versions composer-2.*.*-airflow-*.*.* and newer.`,
},
<% end -%>
<% end -%>
},
},
},
Expand Down Expand Up @@ -566,6 +566,8 @@ func resourceComposerEnvironment() *schema.Resource {
},
},
},
<% end -%>

"workloads_config": {
Type: schema.TypeList,
Optional: true,
Expand Down Expand Up @@ -704,7 +706,6 @@ func resourceComposerEnvironment() *schema.Resource {
ValidateFunc: validation.StringInSlice([]string{"ENVIRONMENT_SIZE_SMALL", "ENVIRONMENT_SIZE_MEDIUM", "ENVIRONMENT_SIZE_LARGE"}, false),
Description: `The size of the Cloud Composer environment. This field is supported for Cloud Composer environments in versions composer-2.*.*-airflow-*.*.* and newer.`,
},
<% end -%>
<% unless version == "ga" -%>
"master_authorized_networks_config": {
Type: schema.TypeList,
Expand Down Expand Up @@ -1007,7 +1008,7 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{})
return err
}
}

<% end -%>
if d.HasChange("config.0.workloads_config") {
patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
if config != nil {
Expand All @@ -1018,7 +1019,17 @@ func resourceComposerEnvironmentUpdate(d *schema.ResourceData, meta interface{})
return err
}
}

if d.HasChange("config.0.environment_size") {
patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
if config != nil {
patchObj.Config.EnvironmentSize = config.EnvironmentSize
}
err = resourceComposerEnvironmentPatchField("config.EnvironmentSize", userAgent, patchObj, d, tfConfig)
if err != nil {
return err
}
}
<% unless version == "ga" -%>
if d.HasChange("config.0.master_authorized_networks_config") {
patchObj := &composer.Environment{Config: &composer.EnvironmentConfig{}}
if config != nil {
Expand Down Expand Up @@ -1154,11 +1165,12 @@ func flattenComposerEnvironmentConfig(envCfg *composer.EnvironmentConfig) interf
transformed["web_server_config"] = flattenComposerEnvironmentConfigWebServerConfig(envCfg.WebServerConfig)
transformed["encryption_config"] = flattenComposerEnvironmentConfigEncryptionConfig(envCfg.EncryptionConfig)
transformed["maintenance_window"] = flattenComposerEnvironmentConfigMaintenanceWindow(envCfg.MaintenanceWindow)
<% end -%>
transformed["workloads_config"] = flattenComposerEnvironmentConfigWorkloadsConfig(envCfg.WorkloadsConfig)
transformed["environment_size"] = envCfg.EnvironmentSize
<% unless version == "ga" -%>
transformed["master_authorized_networks_config"] = flattenComposerEnvironmentConfigMasterAuthorizedNetworksConfig(envCfg.MasterAuthorizedNetworksConfig)
<% end -%>

return []interface{}{transformed}
}

Expand Down Expand Up @@ -1232,6 +1244,8 @@ func flattenComposerEnvironmentConfigMaintenanceWindow(maintenanceWindow *compos
return []interface{}{transformed}
}

<% end -%>

func flattenComposerEnvironmentConfigWorkloadsConfig(workloadsConfig *composer.WorkloadsConfig) interface{} {
if workloadsConfig == nil {
return nil
Expand Down Expand Up @@ -1281,8 +1295,6 @@ func flattenComposerEnvironmentConfigWorkloadsConfig(workloadsConfig *composer.W
return []interface{}{transformed}
}

<% end -%>

func flattenComposerEnvironmentConfigPrivateEnvironmentConfig(envCfg *composer.PrivateEnvironmentConfig) interface{} {
if envCfg == nil {
return nil
Expand All @@ -1293,8 +1305,8 @@ func flattenComposerEnvironmentConfigPrivateEnvironmentConfig(envCfg *composer.P
transformed["master_ipv4_cidr_block"] = envCfg.PrivateClusterConfig.MasterIpv4CidrBlock
transformed["cloud_sql_ipv4_cidr_block"] = envCfg.CloudSqlIpv4CidrBlock
transformed["web_server_ipv4_cidr_block"] = envCfg.WebServerIpv4CidrBlock
<% unless version == "ga" -%>
transformed["cloud_composer_network_ipv4_cidr_block"] = envCfg.CloudComposerNetworkIpv4CidrBlock
<% unless version == "ga" -%>
transformed["enable_privately_used_public_ips"] = envCfg.EnablePrivatelyUsedPublicIps
transformed["cloud_composer_connection_subnetwork"] = envCfg.CloudComposerConnectionSubnetwork

Expand Down Expand Up @@ -1456,7 +1468,7 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf
return nil, err
}
transformed.MaintenanceWindow = transformedMaintenanceWindow

<% end -%>
transformedWorkloadsConfig, err := expandComposerEnvironmentConfigWorkloadsConfig(original["workloads_config"], d, config)
if err != nil {
return nil, err
Expand All @@ -1468,15 +1480,12 @@ func expandComposerEnvironmentConfig(v interface{}, d *schema.ResourceData, conf
return nil, err
}
transformed.EnvironmentSize = transformedEnvironmentSize
<% end -%>
<% unless version == "ga" -%>
transformedMasterAuthorizedNetworksConfig, err := expandComposerEnvironmentConfigMasterAuthorizedNetworksConfig(original["master_authorized_networks_config"], d, config)
if err != nil {
return nil, err
}
transformed.MasterAuthorizedNetworksConfig = transformedMasterAuthorizedNetworksConfig

<% end -%>
return transformed, nil
}
Expand Down Expand Up @@ -1620,6 +1629,8 @@ func expandComposerEnvironmentConfigMaintenanceWindow(v interface{}, d *schema.R
return transformed, nil
}

<% end -%>

func expandComposerEnvironmentConfigWorkloadsConfig(v interface{}, d *schema.ResourceData, config *Config) (*composer.WorkloadsConfig, error) {
l := v.([]interface{})
if len(l) == 0 {
Expand Down Expand Up @@ -1674,7 +1685,6 @@ func expandComposerEnvironmentConfigEnvironmentSize(v interface{}, d *schema.Res
}
return v.(string), nil
}
<% end -%>

func expandComposerEnvironmentConfigPrivateEnvironmentConfig(v interface{}, d *schema.ResourceData, config *Config) (*composer.PrivateEnvironmentConfig, error) {
l := v.([]interface{})
Expand Down Expand Up @@ -1704,11 +1714,11 @@ func expandComposerEnvironmentConfigPrivateEnvironmentConfig(v interface{}, d *s
if v, ok := original["web_server_ipv4_cidr_block"]; ok {
transformed.WebServerIpv4CidrBlock = v.(string)
}
<% unless version == "ga" -%>

if v, ok := original["cloud_composer_network_ipv4_cidr_block"]; ok {
transformed.CloudComposerNetworkIpv4CidrBlock = v.(string)
}

<% unless version == "ga" -%>
if v, ok := original["enable_privately_used_public_ips"]; ok {
transformed.EnablePrivatelyUsedPublicIps = v.(bool)
}
Expand Down
139 changes: 126 additions & 13 deletions mmv1/third_party/terraform/tests/resource_composer_environment_test.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,21 +363,22 @@ func TestAccComposerEnvironment_withMaintenanceWindow(t *testing.T) {
},
})
}
<% end -%>

func TestAccComposerEnvironment_ComposerV2(t *testing.T) {
t.Parallel()

pid := getTestProjectFromEnv()
envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, randInt(t))
network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, randInt(t))
subnetwork := network + "-1"

vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccComposerEnvironmentDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccComposerEnvironment_composerV2(pid, envName, network, subnetwork),
Config: testAccComposerEnvironment_composerV2(envName, network, subnetwork),
},
{
ResourceName: "google_composer_environment.test",
Expand All @@ -390,13 +391,51 @@ func TestAccComposerEnvironment_ComposerV2(t *testing.T) {
{
PlanOnly: true,
ExpectNonEmptyPlan: false,
Config: testAccComposerEnvironment_composerV2(pid, envName, network, subnetwork),
Config: testAccComposerEnvironment_composerV2(envName, network, subnetwork),
Check: testAccCheckClearComposerEnvironmentFirewalls(t, network),
},
},
})
}


func TestAccComposerEnvironment_UpdateComposerV2(t *testing.T) {
t.Parallel()

envName := fmt.Sprintf("%s-%d", testComposerEnvironmentPrefix, randInt(t))
network := fmt.Sprintf("%s-%d", testComposerNetworkPrefix, randInt(t))
subnetwork := network + "-1"

vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccComposerEnvironmentDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccComposerEnvironment_composerV2(envName, network, subnetwork),
},
{
Config: testAccComposerEnvironment_updateComposerV2(envName, network, subnetwork),
},
{
ResourceName: "google_composer_environment.test",
ImportState: true,
ImportStateVerify: true,
},
// This is a terrible clean-up step in order to get destroy to succeed,
// due to dangling firewall rules left by the Composer Environment blocking network deletion.
// TODO(dzarmola): Remove this check if firewall rules bug gets fixed by Composer.
{
PlanOnly: true,
ExpectNonEmptyPlan: false,
Config: testAccComposerEnvironment_updateComposerV2(envName, network, subnetwork),
Check: testAccCheckClearComposerEnvironmentFirewalls(t, network),
},
},
})
}

<% unless version == "ga" -%>
func TestAccComposerEnvironment_composerV2PrivateServiceConnect(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -1139,8 +1178,9 @@ resource "google_compute_subnetwork" "test" {

`, envName, network, subnetwork)
}
<% end -%>

func testAccComposerEnvironment_composerV2(pid, envName, network, subnetwork string) string {
func testAccComposerEnvironment_composerV2(envName, network, subnetwork string) string {
return fmt.Sprintf(`
data "google_composer_image_versions" "all" {
}
Expand All @@ -1165,14 +1205,7 @@ resource "google_composer_environment" "test" {
}

software_config {
# image_version = local.matching_images[0]
image_version = "composer-2.0.0-preview.6-airflow-2.1.4"
}

maintenance_window {
start_time = "2019-08-01T01:00:00Z"
end_time = "2019-08-01T07:00:00Z"
recurrence = "FREQ=WEEKLY;BYDAY=TU,WE"
image_version = local.matching_images[0]
}

workloads_config {
Expand Down Expand Up @@ -1222,7 +1255,7 @@ resource "google_compute_subnetwork" "test" {
`, envName, network, subnetwork)
}


<% unless version == "ga" -%>
func testAccComposerEnvironment_composerV2PrivateServiceConnect(envName, network, subnetwork string) string {
return fmt.Sprintf(`
data "google_composer_image_versions" "all" {
Expand Down Expand Up @@ -1398,6 +1431,11 @@ resource "google_composer_environment" "test" {
network = google_compute_network.test.self_link
subnetwork = google_compute_subnetwork.test.self_link
zone = "us-central1-a"
machine_type = "n1-standard-1"
ip_allocation_policy {
use_ip_aliases = true
cluster_ipv4_cidr_block = "10.0.0.0/16"
}
}

software_config {
Expand Down Expand Up @@ -1448,6 +1486,81 @@ resource "google_compute_subnetwork" "test" {
`, name, network, subnetwork)
}


func testAccComposerEnvironment_updateComposerV2(name, network, subnetwork string) string {
return fmt.Sprintf(`
data "google_composer_image_versions" "all" {
}

locals {
composer_version = "2" # both composer_version and airflow_version are parts of regex, so if either 1 or 2 version is ok "[12]" should be used,
airflow_version = "2" # if sub-version is needed remember to escape "." with "\\." for example 1.2 should be written as "1\\.2"
reg_ex = join("", ["composer-", local.composer_version, "\\.[\\d+\\.]*\\d+.*-airflow-", local.airflow_version, "\\.[\\d+\\.]*\\d+"])
matching_images = [for v in data.google_composer_image_versions.all.image_versions[*].image_version_id: v if length(regexall(local.reg_ex, v)) > 0]
}
resource "google_composer_environment" "test" {
name = "%s"
region = "us-east1"

config {
node_config {
network = google_compute_network.test.self_link
subnetwork = google_compute_subnetwork.test.self_link
ip_allocation_policy {
cluster_ipv4_cidr_block = "10.0.0.0/16"
}
}

software_config {
image_version = local.matching_images[0]
}

workloads_config {
scheduler {
cpu = 2.25
memory_gb = 3.5
storage_gb = 6.4
count = 3
}
web_server {
cpu = 2.75
memory_gb = 4.0
storage_gb = 5.4
}
worker {
cpu = 1.5
memory_gb = 3.0
storage_gb = 4.4
min_count = 3
max_count = 6
}
}
environment_size = "ENVIRONMENT_SIZE_LARGE"
private_environment_config {
enable_private_endpoint = true
cloud_composer_network_ipv4_cidr_block = "10.3.192.0/24"
master_ipv4_cidr_block = "172.16.194.0/23"
cloud_sql_ipv4_cidr_block = "10.3.224.0/20"
}
}

}

resource "google_compute_network" "test" {
name = "%s"
auto_create_subnetworks = false
}

resource "google_compute_subnetwork" "test" {
name = "%s"
ip_cidr_range = "10.2.0.0/16"
region = "us-east1"
network = google_compute_network.test.self_link
private_ip_google_access = true
}
`, name, network, subnetwork)
}

func testAccComposerEnvironment_nodeCfg(environment, network, subnetwork, serviceAccount string) string {
return fmt.Sprintf(`
data "google_project" "project" {}
Expand Down
Loading

0 comments on commit cd44d20

Please sign in to comment.