Advanced logging config options in google_compute_subnetwork

.changelog/3603.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+compute: Added custom metadata fields and filter expressions to `google_compute_subnetwork` flow log configuration
+```

google-beta/resource_compute_subnetwork.go

@@ -126,7 +126,15 @@ Toggles the aggregation interval for collecting flow logs. Increasing the
 interval time will reduce the amount of generated flow logs for long
 lasting connections. Default is an interval of 5 seconds per connection. Default value: "INTERVAL_5_SEC" Possible values: ["INTERVAL_5_SEC", "INTERVAL_30_SEC", "INTERVAL_1_MIN", "INTERVAL_5_MIN", "INTERVAL_10_MIN", "INTERVAL_15_MIN"]`,
 							Default:      "INTERVAL_5_SEC",
-							AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata"},
+							AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata", "log_config.0.filter_expr"},
+						},
+						"filter_expr": {
+							Type:     schema.TypeString,
+							Optional: true,
+							Description: `Export filter used to define which VPC flow logs should be logged, as as CEL expression. See
+https://cloud.google.com/vpc/docs/flow-logs#filtering for details on how to format this field.`,
+							Default:      "true",
+							AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata", "log_config.0.filter_expr"},
 						},
 						"flow_sampling": {
 							Type:     schema.TypeFloat,
@@ -137,17 +145,27 @@ flow logs within the subnetwork where 1.0 means all collected logs are
 reported and 0.0 means no logs are reported. Default is 0.5 which means
 half of all collected logs are reported.`,
 							Default:      0.5,
-							AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata"},
+							AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata", "log_config.0.filter_expr"},
 						},
 						"metadata": {
 							Type:         schema.TypeString,
 							Optional:     true,
-							ValidateFunc: validation.StringInSlice([]string{"EXCLUDE_ALL_METADATA", "INCLUDE_ALL_METADATA", ""}, false),
+							ValidateFunc: validation.StringInSlice([]string{"EXCLUDE_ALL_METADATA", "INCLUDE_ALL_METADATA", "CUSTOM_METADATA", ""}, false),
 							Description: `Can only be specified if VPC flow logging for this subnetwork is enabled.
 Configures whether metadata fields should be added to the reported VPC
-flow logs. Default value: "INCLUDE_ALL_METADATA" Possible values: ["EXCLUDE_ALL_METADATA", "INCLUDE_ALL_METADATA"]`,
+flow logs. Default value: "INCLUDE_ALL_METADATA" Possible values: ["EXCLUDE_ALL_METADATA", "INCLUDE_ALL_METADATA", "CUSTOM_METADATA"]`,
 							Default:      "INCLUDE_ALL_METADATA",
-							AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata"},
+							AtLeastOneOf: []string{"log_config.0.aggregation_interval", "log_config.0.flow_sampling", "log_config.0.metadata", "log_config.0.filter_expr"},
+						},
+						"metadata_fields": {
+							Type:     schema.TypeSet,
+							Optional: true,
+							Description: `List of metadata fields that should be added to reported logs.
+Can only be specified if VPC flow logs for this subnetwork is enabled and "metadata" is set to CUSTOM_METADATA.`,
+							Elem: &schema.Schema{
+								Type: schema.TypeString,
+							},
+							Set: schema.HashString,
 						},
 					},
 				},
@@ -817,6 +835,18 @@ func flattenComputeSubnetworkLogConfig(v interface{}, d *schema.ResourceData, co
 	transformed["flow_sampling"] = original["flowSampling"]
 	transformed["aggregation_interval"] = original["aggregationInterval"]
 	transformed["metadata"] = original["metadata"]
+	if original["metadata"].(string) == "CUSTOM_METADATA" {
+		transformed["metadata_fields"] = original["metadataFields"]
+	} else {
+		// MetadataFields can only be set when metadata is CUSTOM_METADATA. However, when updating
+		// from custom to include/exclude, the API will return the previous values of the metadata fields,
+		// despite not actually having any custom fields at the moment. The API team has confirmed
+		// this as WAI (b/162771344), so we work around it by clearing the response if metadata is
+		// not custom.
+		transformed["metadata_fields"] = nil
+	}
+	transformed["filter_expr"] = original["filterExpr"]
+
 	return []interface{}{transformed}
 }
 
@@ -920,6 +950,10 @@ func expandComputeSubnetworkLogConfig(v interface{}, d TerraformResourceData, co
 	transformed["aggregationInterval"] = original["aggregation_interval"]
 	transformed["flowSampling"] = original["flow_sampling"]
 	transformed["metadata"] = original["metadata"]
+	transformed["filterExpr"] = original["filter_expr"]
+
+	// make it JSON marshallable
+	transformed["metadataFields"] = original["metadata_fields"].(*schema.Set).List()
 
 	return transformed, nil
 }

google-beta/resource_compute_subnetwork_test.go

@@ -225,7 +225,31 @@ func TestAccComputeSubnetwork_flowLogs(t *testing.T) {
 				ImportStateVerify: true,
 			},
 			{
-				Config: testAccComputeSubnetwork_flowLogsUpdate(cnName, subnetworkName),
+				Config: testAccComputeSubnetwork_flowLogsUpdate1(cnName, subnetworkName),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckComputeSubnetworkExists(
+						t, "google_compute_subnetwork.network-with-flow-logs", &subnetwork),
+				),
+			},
+			{
+				ResourceName:      "google_compute_subnetwork.network-with-flow-logs",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config: testAccComputeSubnetwork_flowLogsUpdate2(cnName, subnetworkName),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckComputeSubnetworkExists(
+						t, "google_compute_subnetwork.network-with-flow-logs", &subnetwork),
+				),
+			},
+			{
+				ResourceName:      "google_compute_subnetwork.network-with-flow-logs",
+				ImportState:       true,
+				ImportStateVerify: true,
+			},
+			{
+				Config: testAccComputeSubnetwork_flowLogsUpdate3(cnName, subnetworkName),
 				Check: resource.ComposeTestCheckFunc(
 					testAccCheckComputeSubnetworkExists(
 						t, "google_compute_subnetwork.network-with-flow-logs", &subnetwork),
@@ -556,7 +580,7 @@ resource "google_compute_subnetwork" "network-with-flow-logs" {
 `, cnName, subnetworkName)
 }
 
-func testAccComputeSubnetwork_flowLogsUpdate(cnName, subnetworkName string) string {
+func testAccComputeSubnetwork_flowLogsUpdate1(cnName, subnetworkName string) string {
 	return fmt.Sprintf(`
 resource "google_compute_network" "custom-test" {
   name                    = "%s"
@@ -577,6 +601,53 @@ resource "google_compute_subnetwork" "network-with-flow-logs" {
 `, cnName, subnetworkName)
 }
 
+func testAccComputeSubnetwork_flowLogsUpdate2(cnName, subnetworkName string) string {
+	return fmt.Sprintf(`
+resource "google_compute_network" "custom-test" {
+  name                    = "%s"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "network-with-flow-logs" {
+  name          = "%s"
+  ip_cidr_range = "10.0.0.0/16"
+  region        = "us-central1"
+  network       = google_compute_network.custom-test.self_link
+  log_config {
+    aggregation_interval = "INTERVAL_30_SEC"
+    flow_sampling        = 0.8
+    metadata             = "CUSTOM_METADATA"
+    metadata_fields      = [
+        "src_gke_details",
+        "dest_gke_details",
+    ]
+    filter_expr          = "inIpRange(connection.src_ip, '10.0.0.0/8')"
+  }
+}
+`, cnName, subnetworkName)
+}
+
+func testAccComputeSubnetwork_flowLogsUpdate3(cnName, subnetworkName string) string {
+	return fmt.Sprintf(`
+resource "google_compute_network" "custom-test" {
+  name                    = "%s"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "network-with-flow-logs" {
+  name          = "%s"
+  ip_cidr_range = "10.0.0.0/16"
+  region        = "us-central1"
+  network       = google_compute_network.custom-test.self_link
+  log_config {
+    aggregation_interval = "INTERVAL_30_SEC"
+    flow_sampling        = 0.8
+    metadata             = "INCLUDE_ALL_METADATA"
+  }
+}
+`, cnName, subnetworkName)
+}
+
 func testAccComputeSubnetwork_flowLogsDelete(cnName, subnetworkName string) string {
 	return fmt.Sprintf(`
 resource "google_compute_network" "custom-test" {

website/docs/r/compute_subnetwork.html.markdown

@@ -49,7 +49,7 @@ of the network, even entire subnets, using firewall rules.
 
 To get more information about Subnetwork, see:
 
-* [API documentation](https://cloud.google.com/compute/docs/reference/rest/beta/subnetworks)
+* [API documentation](https://cloud.google.com/compute/docs/reference/rest/v1/subnetworks)
 * How-to Guides
     * [Private Google Access](https://cloud.google.com/vpc/docs/configure-private-google-access)
     * [Cloud Networking](https://cloud.google.com/vpc/docs/using-vpc)
@@ -264,7 +264,17 @@ The `log_config` block supports:
   Configures whether metadata fields should be added to the reported VPC
   flow logs.
   Default value is `INCLUDE_ALL_METADATA`.
-  Possible values are `EXCLUDE_ALL_METADATA` and `INCLUDE_ALL_METADATA`.
+  Possible values are `EXCLUDE_ALL_METADATA`, `INCLUDE_ALL_METADATA`, and `CUSTOM_METADATA`.
+
+* `metadata_fields` -
+  (Optional)
+  List of metadata fields that should be added to reported logs.
+  Can only be specified if VPC flow logs for this subnetwork is enabled and "metadata" is set to CUSTOM_METADATA.
+
+* `filter_expr` -
+  (Optional)
+  Export filter used to define which VPC flow logs should be logged, as as CEL expression. See
+  https://cloud.google.com/vpc/docs/flow-logs#filtering for details on how to format this field.
 
 ## Attributes Reference