From 212c948ef84aeb019dba1b1ad8316baa4a521dd3 Mon Sep 17 00:00:00 2001 From: Modular Magician Date: Tue, 25 Aug 2020 15:58:09 +0000 Subject: [PATCH] Advanced logging config options in google_compute_subnetwork (#3603) Co-authored-by: Dana Hoffman Signed-off-by: Modular Magician --- .changelog/3603.txt | 3 + google-beta/resource_compute_subnetwork.go | 44 +++++++++-- .../resource_compute_subnetwork_test.go | 75 ++++++++++++++++++- .../docs/r/compute_subnetwork.html.markdown | 14 +++- 4 files changed, 127 insertions(+), 9 deletions(-) create mode 100644 .changelog/3603.txt diff --git a/.changelog/3603.txt b/.changelog/3603.txt new file mode 100644 index 0000000000..e73c0867e1 --- /dev/null +++ b/.changelog/3603.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +compute: Added custom metadata fields and filter expressions to `google_compute_subnetwork` flow log configuration +``` diff --git a/google-beta/resource_compute_subnetwork.go b/google-beta/resource_compute_subnetwork.go index 7a85bd93ff..688d35a345 100644 --- a/google-beta/resource_compute_subnetwork.go +++ b/google-beta/resource_compute_subnetwork.go @@ -126,7 +126,15 @@ Toggles the aggregation interval for collecting flow logs. Increasing the interval time will reduce the amount of generated flow logs for long lasting connections. Default is an interval of 5 seconds per connection. Default value: "INTERVAL_5_SEC" Possible values: ["INTERVAL_5_SEC", "INTERVAL_30_SEC", "INTERVAL_1_MIN", "INTERVAL_5_MIN", "INTERVAL_10_MIN", "INTERVAL_15_MIN"]`, Default: "INTERVAL_5_SEC", - AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata"}, + AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata", "log_config.0.filter_expr"}, + }, + "filter_expr": { + Type: schema.TypeString, + Optional: true, + Description: `Export filter used to define which VPC flow logs should be logged, as as CEL expression. See +https://cloud.google.com/vpc/docs/flow-logs#filtering for details on how to format this field.`, + Default: "true", + AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata", "log_config.0.filter_expr"}, }, "flow_sampling": { Type: schema.TypeFloat, @@ -137,17 +145,27 @@ flow logs within the subnetwork where 1.0 means all collected logs are reported and 0.0 means no logs are reported. Default is 0.5 which means half of all collected logs are reported.`, Default: 0.5, - AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata"}, + AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata", "log_config.0.filter_expr"}, }, "metadata": { Type: schema.TypeString, Optional: true, - ValidateFunc: validation.StringInSlice([]string{"EXCLUDE_ALL_METADATA", "INCLUDE_ALL_METADATA", ""}, false), + ValidateFunc: validation.StringInSlice([]string{"EXCLUDE_ALL_METADATA", "INCLUDE_ALL_METADATA", "CUSTOM_METADATA", ""}, false), Description: `Can only be specified if VPC flow logging for this subnetwork is enabled. Configures whether metadata fields should be added to the reported VPC -flow logs. Default value: "INCLUDE_ALL_METADATA" Possible values: ["EXCLUDE_ALL_METADATA", "INCLUDE_ALL_METADATA"]`, +flow logs. Default value: "INCLUDE_ALL_METADATA" Possible values: ["EXCLUDE_ALL_METADATA", "INCLUDE_ALL_METADATA", "CUSTOM_METADATA"]`, Default: "INCLUDE_ALL_METADATA", - AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata"}, + AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata", "log_config.0.filter_expr"}, + }, + "metadata_fields": { + Type: schema.TypeSet, + Optional: true, + Description: `List of metadata fields that should be added to reported logs. +Can only be specified if VPC flow logs for this subnetwork is enabled and "metadata" is set to CUSTOM_METADATA.`, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + Set: schema.HashString, }, }, }, @@ -817,6 +835,18 @@ func flattenComputeSubnetworkLogConfig(v interface{}, d *schema.ResourceData, co transformed["flow_sampling"] = original["flowSampling"] transformed["aggregation_interval"] = original["aggregationInterval"] transformed["metadata"] = original["metadata"] + if original["metadata"].(string) == "CUSTOM_METADATA" { + transformed["metadata_fields"] = original["metadataFields"] + } else { + // MetadataFields can only be set when metadata is CUSTOM_METADATA. However, when updating + // from custom to include/exclude, the API will return the previous values of the metadata fields, + // despite not actually having any custom fields at the moment. The API team has confirmed + // this as WAI (b/162771344), so we work around it by clearing the response if metadata is + // not custom. + transformed["metadata_fields"] = nil + } + transformed["filter_expr"] = original["filterExpr"] + return []interface{}{transformed} } @@ -920,6 +950,10 @@ func expandComputeSubnetworkLogConfig(v interface{}, d TerraformResourceData, co transformed["aggregationInterval"] = original["aggregation_interval"] transformed["flowSampling"] = original["flow_sampling"] transformed["metadata"] = original["metadata"] + transformed["filterExpr"] = original["filter_expr"] + + // make it JSON marshallable + transformed["metadataFields"] = original["metadata_fields"].(*schema.Set).List() return transformed, nil } diff --git a/google-beta/resource_compute_subnetwork_test.go b/google-beta/resource_compute_subnetwork_test.go index dad396194c..0c040234b6 100644 --- a/google-beta/resource_compute_subnetwork_test.go +++ b/google-beta/resource_compute_subnetwork_test.go @@ -225,7 +225,31 @@ func TestAccComputeSubnetwork_flowLogs(t *testing.T) { ImportStateVerify: true, }, { - Config: testAccComputeSubnetwork_flowLogsUpdate(cnName, subnetworkName), + Config: testAccComputeSubnetwork_flowLogsUpdate1(cnName, subnetworkName), + Check: resource.ComposeTestCheckFunc( + testAccCheckComputeSubnetworkExists( + t, "google_compute_subnetwork.network-with-flow-logs", &subnetwork), + ), + }, + { + ResourceName: "google_compute_subnetwork.network-with-flow-logs", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccComputeSubnetwork_flowLogsUpdate2(cnName, subnetworkName), + Check: resource.ComposeTestCheckFunc( + testAccCheckComputeSubnetworkExists( + t, "google_compute_subnetwork.network-with-flow-logs", &subnetwork), + ), + }, + { + ResourceName: "google_compute_subnetwork.network-with-flow-logs", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccComputeSubnetwork_flowLogsUpdate3(cnName, subnetworkName), Check: resource.ComposeTestCheckFunc( testAccCheckComputeSubnetworkExists( t, "google_compute_subnetwork.network-with-flow-logs", &subnetwork), @@ -556,7 +580,7 @@ resource "google_compute_subnetwork" "network-with-flow-logs" { `, cnName, subnetworkName) } -func testAccComputeSubnetwork_flowLogsUpdate(cnName, subnetworkName string) string { +func testAccComputeSubnetwork_flowLogsUpdate1(cnName, subnetworkName string) string { return fmt.Sprintf(` resource "google_compute_network" "custom-test" { name = "%s" @@ -577,6 +601,53 @@ resource "google_compute_subnetwork" "network-with-flow-logs" { `, cnName, subnetworkName) } +func testAccComputeSubnetwork_flowLogsUpdate2(cnName, subnetworkName string) string { + return fmt.Sprintf(` +resource "google_compute_network" "custom-test" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "network-with-flow-logs" { + name = "%s" + ip_cidr_range = "10.0.0.0/16" + region = "us-central1" + network = google_compute_network.custom-test.self_link + log_config { + aggregation_interval = "INTERVAL_30_SEC" + flow_sampling = 0.8 + metadata = "CUSTOM_METADATA" + metadata_fields = [ + "src_gke_details", + "dest_gke_details", + ] + filter_expr = "inIpRange(connection.src_ip, '10.0.0.0/8')" + } +} +`, cnName, subnetworkName) +} + +func testAccComputeSubnetwork_flowLogsUpdate3(cnName, subnetworkName string) string { + return fmt.Sprintf(` +resource "google_compute_network" "custom-test" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "network-with-flow-logs" { + name = "%s" + ip_cidr_range = "10.0.0.0/16" + region = "us-central1" + network = google_compute_network.custom-test.self_link + log_config { + aggregation_interval = "INTERVAL_30_SEC" + flow_sampling = 0.8 + metadata = "INCLUDE_ALL_METADATA" + } +} +`, cnName, subnetworkName) +} + func testAccComputeSubnetwork_flowLogsDelete(cnName, subnetworkName string) string { return fmt.Sprintf(` resource "google_compute_network" "custom-test" { diff --git a/website/docs/r/compute_subnetwork.html.markdown b/website/docs/r/compute_subnetwork.html.markdown index 85fbd7f244..d712841005 100644 --- a/website/docs/r/compute_subnetwork.html.markdown +++ b/website/docs/r/compute_subnetwork.html.markdown @@ -49,7 +49,7 @@ of the network, even entire subnets, using firewall rules. To get more information about Subnetwork, see: -* [API documentation](https://cloud.google.com/compute/docs/reference/rest/beta/subnetworks) +* [API documentation](https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks) * How-to Guides * [Private Google Access](https://cloud.google.com/vpc/docs/configure-private-google-access) * [Cloud Networking](https://cloud.google.com/vpc/docs/using-vpc) @@ -264,7 +264,17 @@ The `log_config` block supports: Configures whether metadata fields should be added to the reported VPC flow logs. Default value is `INCLUDE_ALL_METADATA`. - Possible values are `EXCLUDE_ALL_METADATA` and `INCLUDE_ALL_METADATA`. + Possible values are `EXCLUDE_ALL_METADATA`, `INCLUDE_ALL_METADATA`, and `CUSTOM_METADATA`. + +* `metadata_fields` - + (Optional) + List of metadata fields that should be added to reported logs. + Can only be specified if VPC flow logs for this subnetwork is enabled and "metadata" is set to CUSTOM_METADATA. + +* `filter_expr` - + (Optional) + Export filter used to define which VPC flow logs should be logged, as as CEL expression. See + https://cloud.google.com/vpc/docs/flow-logs#filtering for details on how to format this field. ## Attributes Reference