Add data source for retrieving multiple objects from a GCS bucket

google/provider/provider_mmv1_resources.go

@@ -275,6 +275,7 @@ var handwrittenDatasources = map[string]*schema.Resource{
 	"google_service_networking_peered_dns_domain":         servicenetworking.DataSourceGoogleServiceNetworkingPeeredDNSDomain(),
 	"google_storage_bucket":                               storage.DataSourceGoogleStorageBucket(),
 	"google_storage_bucket_object":                        storage.DataSourceGoogleStorageBucketObject(),
+	"google_storage_bucket_objects":                       storage.DataSourceGoogleStorageBucketObjects(),
 	"google_storage_bucket_object_content":                storage.DataSourceGoogleStorageBucketObjectContent(),
 	"google_storage_object_signed_url":                    storage.DataSourceGoogleSignedUrl(),
 	"google_storage_project_service_account":              storage.DataSourceGoogleStorageProjectServiceAccount(),

google/services/storage/data_source_google_storage_bucket_objects.go

@@ -0,0 +1,156 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+package storage
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-provider-google/google/tpgresource"
+	transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport"
+)
+
+func DataSourceGoogleStorageBucketObjects() *schema.Resource {
+	return &schema.Resource{
+		Read: datasourceGoogleStorageBucketObjectsRead,
+		Schema: map[string]*schema.Schema{
+			"bucket": {
+				Type:     schema.TypeString,
+				Required: true,
+			},
+			"match_glob": {
+				Type:     schema.TypeString,
+				Optional: true,
+			},
+			"prefix": {
+				Type:     schema.TypeString,
+				Optional: true,
+			},
+			"bucket_objects": {
+				Type:     schema.TypeList,
+				Computed: true,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"content_type": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"media_link": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"name": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"self_link": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+						"storage_class": {
+							Type:     schema.TypeString,
+							Computed: true,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func datasourceGoogleStorageBucketObjectsRead(d *schema.ResourceData, meta interface{}) error {
+	config := meta.(*transport_tpg.Config)
+	userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent)
+	if err != nil {
+		return err
+	}
+
+	params := make(map[string]string)
+	bucketObjects := make([]map[string]interface{}, 0)
+
+	for {
+		bucket := d.Get("bucket").(string)
+		url := fmt.Sprintf("https://storage.googleapis.com/storage/v1/b/%s/o", bucket)
+
+		if v, ok := d.GetOk("match_glob"); ok {
+			params["matchGlob"] = v.(string)
+		}
+
+		if v, ok := d.GetOk("prefix"); ok {
+			params["prefix"] = v.(string)
+		}
+
+		url, err := transport_tpg.AddQueryParams(url, params)
+		if err != nil {
+			return err
+		}
+
+		res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{
+			Config:    config,
+			Method:    "GET",
+			RawURL:    url,
+			UserAgent: userAgent,
+		})
+		if err != nil {
+			return fmt.Errorf("Error retrieving bucket objects: %s", err)
+		}
+
+		pageBucketObjects := flattenDatasourceGoogleBucketObjectsList(res["items"])
+		bucketObjects = append(bucketObjects, pageBucketObjects...)
+
+		pToken, ok := res["nextPageToken"]
+		if ok && pToken != nil && pToken.(string) != "" {
+			params["pageToken"] = pToken.(string)
+		} else {
+			break
+		}
+	}
+
+	if err := d.Set("bucket_objects", bucketObjects); err != nil {
+		return fmt.Errorf("Error retrieving bucket_objects: %s", err)
+	}
+
+	d.SetId(d.Get("bucket").(string))
+
+	return nil
+}
+
+func flattenDatasourceGoogleBucketObjectsList(v interface{}) []map[string]interface{} {
+	if v == nil {
+		return make([]map[string]interface{}, 0)
+	}
+
+	ls := v.([]interface{})
+	bucketObjects := make([]map[string]interface{}, 0, len(ls))
+	for _, raw := range ls {
+		o := raw.(map[string]interface{})
+
+		var mContentType, mMediaLink, mName, mSelfLink, mStorageClass interface{}
+		if oContentType, ok := o["contentType"]; ok {
+			mContentType = oContentType
+		}
+		if oMediaLink, ok := o["mediaLink"]; ok {
+			mMediaLink = oMediaLink
+		}
+		if oName, ok := o["name"]; ok {
+			mName = oName
+		}
+		if oSelfLink, ok := o["selfLink"]; ok {
+			mSelfLink = oSelfLink
+		}
+		if oStorageClass, ok := o["storageClass"]; ok {
+			mStorageClass = oStorageClass
+		}
+		bucketObjects = append(bucketObjects, map[string]interface{}{
+			"content_type":  mContentType,
+			"media_link":    mMediaLink,
+			"name":          mName,
+			"self_link":     mSelfLink,
+			"storage_class": mStorageClass,
+		})
+	}
+
+	return bucketObjects
+}

google/services/storage/data_source_google_storage_bucket_objects_test.go

@@ -0,0 +1,114 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+package storage_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+	"github.com/hashicorp/terraform-provider-google/google/acctest"
+	"github.com/hashicorp/terraform-provider-google/google/envvar"
+)
+
+func TestAccDataSourceGoogleStorageBucketObjects_basic(t *testing.T) {
+	t.Parallel()
+
+	project := envvar.GetTestProjectFromEnv()
+	bucket := "tf-bucket-object-test-" + acctest.RandString(t, 10)
+
+	context := map[string]interface{}{
+		"bucket":        bucket,
+		"project":       project,
+		"object_0_name": "bee",
+		"object_1_name": "fly",
+	}
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccCheckGoogleStorageBucketObjectsConfig(context),
+				Check: resource.ComposeTestCheckFunc(
+					// Test schema
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.content_type"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.media_link"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.name"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.self_link"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.storage_class"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.content_type"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.media_link"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.name"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.self_link"),
+					resource.TestCheckResourceAttrSet("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.storage_class"),
+					// Test content
+					resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_insects", "bucket", context["bucket"].(string)),
+					resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_insects", "bucket_objects.0.name", context["object_0_name"].(string)),
+					resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_insects", "bucket_objects.1.name", context["object_1_name"].(string)),
+					// Test match_glob
+					resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_bee_glob", "bucket_objects.0.name", context["object_0_name"].(string)),
+					// Test prefix
+					resource.TestCheckResourceAttr("data.google_storage_bucket_objects.my_fly_prefix", "bucket_objects.0.name", context["object_1_name"].(string)),
+				),
+			},
+		},
+	})
+}
+
+func testAccCheckGoogleStorageBucketObjectsConfig(context map[string]interface{}) string {
+	return fmt.Sprintf(`
+resource "google_storage_bucket" "my_insect_cage" {
+  force_destroy               = true
+  location                    = "EU"
+  name                        = "%s"
+  project                     = "%s"
+  uniform_bucket_level_access = true
+}
+
+resource "google_storage_bucket_object" "bee" {
+  bucket  = google_storage_bucket.my_insect_cage.name
+  content = "bzzzzzt"
+  name    = "%s"
+}
+
+resource "google_storage_bucket_object" "fly" {
+  bucket  = google_storage_bucket.my_insect_cage.name
+  content = "zzzzzt"
+  name    = "%s"
+}
+
+data "google_storage_bucket_objects" "my_insects" {
+  bucket = google_storage_bucket.my_insect_cage.name
+
+  depends_on = [
+    google_storage_bucket_object.bee,
+	google_storage_bucket_object.fly,
+  ]
+}
+
+data "google_storage_bucket_objects" "my_bee_glob" {
+  bucket     = google_storage_bucket.my_insect_cage.name
+  match_glob = "b*"
+
+  depends_on = [
+    google_storage_bucket_object.bee,
+  ]
+}
+
+data "google_storage_bucket_objects" "my_fly_prefix" {
+  bucket = google_storage_bucket.my_insect_cage.name
+  prefix = "f"
+
+  depends_on = [
+    google_storage_bucket_object.fly,
+  ]
+}`,
+		context["bucket"].(string),
+		context["project"].(string),
+		context["object_0_name"].(string),
+		context["object_1_name"].(string),
+	)
+}

website/docs/d/storage_bucket_objects.html.markdown

@@ -0,0 +1,45 @@
+---
+subcategory: "Cloud Storage"
+description: |-
+  Retrieve information about a set of GCS bucket objects in a GCS bucket.
+---
+
+
+# google\_storage\_bucket\_objects
+
+Gets existing objects inside an existing bucket in Google Cloud Storage service (GCS).
+See [the official documentation](https://cloud.google.com/storage/docs/key-terms#objects)
+and [API](https://cloud.google.com/storage/docs/json_api/v1/objects/list).
+
+## Example Usage
+
+Example files stored within a bucket.
+
+```hcl
+data "google_storage_bucket_objects" "files" {
+  bucket = "file-store"
+}
+```
+
+## Argument Reference
+
+The following arguments are supported:
+
+* `bucket` - (Required) The name of the containing bucket.
+* `match_glob` - (Optional) A glob pattern used to filter results (for example, `foo*bar`).
+* `prefix` - (Optional) Filter results to include only objects whose names begin with this prefix.
+
+
+## Attributes Reference
+
+The following attributes are exported:
+
+* `bucket_objects` - A list of retrieved objects contained in the provided GCS bucket. Structure is [defined below](#nested_bucket_objects).
+
+<a name="nested_bucket_objects"></a>The `bucket_objects` block supports:
+
+* `content_type` - [Content-Type](https://tools.ietf.org/html/rfc7231#section-3.1.1.5) of the object data.
+* `media_link` - A url reference to download this object.
+* `name` - The name of the object.
+* `self_link` - A url reference to this object.
+* `storage_class` - The [StorageClass](https://cloud.google.com/storage/docs/storage-classes) of the bucket object.