From e593bae12702472b0359f6e42094fa7b23f1ae55 Mon Sep 17 00:00:00 2001 From: Stephen Lewis Date: Tue, 22 Nov 2022 09:58:30 -0800 Subject: [PATCH] Split handwritten / mmv1 howto guides into separate pages --- docs/config.toml | 5 +- docs/content/_index.md | 4 +- docs/content/docs/how-to/_index.md | 5 + .../docs/how-to/add-handwritten-datasource.md | 27 ++ .../docs/how-to/add-handwritten-iam.md | 49 +++ .../docs/how-to/add-handwritten-test.md | 216 ++++++++++ docs/content/docs/how-to/add-mmv1-iam.md | 103 +++++ .../{mmv1.md => how-to/add-mmv1-resource.md} | 377 +----------------- docs/content/docs/how-to/add-mmv1-test.md | 195 +++++++++ .../how-to/identify-mmv1-vs-handwritten.md | 16 + .../update-handwritten-resource.md} | 363 ++--------------- docs/content/docs/reference/_index.md | 5 + .../docs/reference/api-yaml-resource.md | 6 + .../docs/reference/terraform-yaml-resource.md | 6 + 14 files changed, 678 insertions(+), 699 deletions(-) create mode 100644 docs/content/docs/how-to/_index.md create mode 100644 docs/content/docs/how-to/add-handwritten-datasource.md create mode 100644 docs/content/docs/how-to/add-handwritten-iam.md create mode 100644 docs/content/docs/how-to/add-handwritten-test.md create mode 100644 docs/content/docs/how-to/add-mmv1-iam.md rename docs/content/docs/{mmv1.md => how-to/add-mmv1-resource.md} (52%) create mode 100644 docs/content/docs/how-to/add-mmv1-test.md create mode 100644 docs/content/docs/how-to/identify-mmv1-vs-handwritten.md rename docs/content/docs/{handwritten.md => how-to/update-handwritten-resource.md} (51%) create mode 100644 docs/content/docs/reference/_index.md create mode 100644 docs/content/docs/reference/api-yaml-resource.md create mode 100644 docs/content/docs/reference/terraform-yaml-resource.md diff --git a/docs/config.toml b/docs/config.toml index 1094ea34f39b..3765d0004d06 100644 --- a/docs/config.toml +++ b/docs/config.toml @@ -1,11 +1,14 @@ baseURL = 'https://googlecloudplatform.github.io/magic-modules/' languageCode = 'en-us' title = 'Magic Modules' -theme = ["github.com/alex-shpak/hugo-book"] enableGitInfo = true disableKinds = ['taxonomy', 'taxonomyTerm'] +[module] +[[module.imports]] +path = 'github.com/alex-shpak/hugo-book' + [params] BookTheme = 'light' BookLogo = 'magic-modules.svg' diff --git a/docs/content/_index.md b/docs/content/_index.md index 7db67a3f3177..fbd8dd8d1a09 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -1,5 +1,7 @@ --- -title: "" +title: "Home" +weight: 0 +type: "docs" date: 2022-11-14T09:50:49-08:00 --- diff --git a/docs/content/docs/how-to/_index.md b/docs/content/docs/how-to/_index.md new file mode 100644 index 000000000000..cc98100b41aa --- /dev/null +++ b/docs/content/docs/how-to/_index.md @@ -0,0 +1,5 @@ +--- +title: "How To" +weight: 10 +isSection: true +--- \ No newline at end of file diff --git a/docs/content/docs/how-to/add-handwritten-datasource.md b/docs/content/docs/how-to/add-handwritten-datasource.md new file mode 100644 index 000000000000..253cfd194b02 --- /dev/null +++ b/docs/content/docs/how-to/add-handwritten-datasource.md @@ -0,0 +1,27 @@ +--- +title: "Add a handwritten datasource" +weight: 22 +--- + +# Add a handwritten datasource + +**Note** : only handwritten datasources are currently supported + +Datasources are like terraform resources except they don't *create* anything. +They are simply read-only operations that will expose some sort of values needed +for subsequent resource operations. If you're adding a field to an existing +datasource, check the [Resource](#resource) section. Everything there will +be mostly consistent with the type of change you'll need to make. For adding +a new datasource there are 5 steps to doing so. + +1. Create a new datasource declaration file and a corresponding test file +1. Add Schema and Read operation implementation +1. Add the datasource to the `provider.go.erb` index +1. Implement a test which will create and resources and read the corresponding + datasource. +1. Add documentation. + +For creating a datasource based off an existing resource you can [make use of the +schema directly](https://github.com/GoogleCloudPlatform/magic-modules/blob/1d293f7bfadacaa20580874c8e8634827fb99a14/mmv1/third_party/terraform/data_sources/data_source_cloud_run_service.go). +Otherwise [implementing the schema directly](https://github.com/GoogleCloudPlatform/magic-modules/blob/1d293f7bfadacaa20580874c8e8634827fb99a14/mmv1/third_party/terraform/data_sources/data_source_google_compute_address.go), +similar to normal resource creation, is the desired path. diff --git a/docs/content/docs/how-to/add-handwritten-iam.md b/docs/content/docs/how-to/add-handwritten-iam.md new file mode 100644 index 000000000000..248be707d462 --- /dev/null +++ b/docs/content/docs/how-to/add-handwritten-iam.md @@ -0,0 +1,49 @@ +--- +title: "Add handwritten IAM resources" +weight: 23 +--- + +# Add handwritten IAM resources + +Handwritten IAM support is only recommended for resources that cannot be managed +using [MMv1](/magic-modules/docs/how-to/add-mmv1-iam), +including for handwritten resources, due to the need to manage tests and +documentation by hand. This guidance goes through the motions of adding support +for new handwritten IAM resources, but does not go into the details of the +implementation as any new handwritten IAM resources are expected to be +exceptional. + +IAM resources are implemented using an IAM framework, where you implement an +interface for each parent resource supporting `getIamPolicy`/`setIamPolicy` and +the associated IAM resources that target that parent resource- `_member`, +`_binding`, and `_policy`- are created by the framework. + +To add support for a new target, create a new file in +`mmv1/third_party/terraform/utils` called `iam_{{resource}}.go`, and implement +the `ResourceIamUpdater`, `newResourceIamUpdaterFunc`, `iamPolicyModifyFunc`, +`resourceIdParserFunc` interfaces from +https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/third_party/terraform/utils/iam.go.erb +in public types, alongside a public `map[string]*schema.Schema` containing all +fields referenced in the resource. + +Once your implementation is complete, add the IAM resources to `provider.go` +inside the `START non-generated IAM resources` block, creating the concrete +resource types using the `ResourceIamMember`, `ResourceIamBinding`, and +`ResourceIamPolicy` functions. For example: + +```go + "google_bigtable_instance_iam_binding": ResourceIamBinding(IamBigtableInstanceSchema, NewBigtableInstanceUpdater, BigtableInstanceIdParseFunc), + "google_bigtable_instance_iam_member": ResourceIamMember(IamBigtableInstanceSchema, NewBigtableInstanceUpdater, BigtableInstanceIdParseFunc), + "google_bigtable_instance_iam_policy": ResourceIamPolicy(IamBigtableInstanceSchema, NewBigtableInstanceUpdater, BigtableInstanceIdParseFunc), +``` + +Following that, write a test for each resource exercising create and update for +both `_policy` and `_binding`, and create for `_member`. No special +accommodations are needed for the IAM test compared to a normal Terraform +resource test. + +Documentation for IAM resources is done using single page per target resource, +rather than a distinct page for each IAM resource level. As most of the page is +standard, you can generally copy and edit an existing handwritten page such as +https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/third_party/terraform/website/docs/r/bigtable_instance_iam.html.markdown +to write the documentation. diff --git a/docs/content/docs/how-to/add-handwritten-test.md b/docs/content/docs/how-to/add-handwritten-test.md new file mode 100644 index 000000000000..e7a713a95852 --- /dev/null +++ b/docs/content/docs/how-to/add-handwritten-test.md @@ -0,0 +1,216 @@ +--- +title: "Add a handwritten test" +weight: 21 +--- + + +# Add a handwritten test + +For handwritten resources and generated resources that need to test update, +handwritten tests must be added. + +Tests are made up of a templated Terraform configuration where unique values +like GCE names are passed in as arguments, and boilerplate to exercise that +configuration. + +The test boilerplate effectively does the following: + +1. Run `terraform apply` on the configuration, waiting for it to succeed and + recording the results in Terraform state +2. Run `terraform plan`, and fail if Terraform detects any drift +3. Clear the resource from state and run `terraform import` on it +4. Deeply compare the original state from `terraform apply` and the `terraform + import` results, returning an error if any values are not identical +5. Destroy all resources in the configuration using `terraform destroy`, + waiting for the destroy command to succeed +6. Call `GET` on the resource, and fail the test if it is still present + +## Simple Tests + +Terraform configurations are stored as string constants wrapped in Go functions +like the following: + +```go +func testAccComputeFirewall_basic(network, firewall string) string { + return fmt.Sprintf(` +resource "google_compute_network" "foobar" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_firewall" "foobar" { + name = "%s" + description = "Resource created for Terraform acceptance testing" + network = google_compute_network.foobar.name + source_tags = ["foo"] + allow { + protocol = "icmp" + } +} +`, network, firewall) +} +``` + +For the most part, you can copy and paste a preexisting test case and modify it. +For example, the following test case is a good reference: + +```go +func TestAccComputeFirewall_noSource(t *testing.T) { + t.Parallel() + + networkName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10)) + firewallName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10)) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckComputeFirewallDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComputeFirewall_noSource(networkName, firewallName), + }, + { + ResourceName: "google_compute_firewall.foobar", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccComputeFirewall_noSource(network, firewall string) string { + return fmt.Sprintf(` +resource "google_compute_network" "foobar" { + name = "%s" + auto_create_subnetworks = false +} +resource "google_compute_firewall" "foobar" { + name = "%s" + description = "Resource created for Terraform acceptance testing" + network = google_compute_network.foobar.name + allow { + protocol = "tcp" + ports = [22] + } +} +`, network, firewall) +} +``` + +## Update tests + +Inside of a test, additional steps can be added in order to transition between +Terraform configurations, updating the stored state as it progresses. This +allows you to exercise update behaviour. This modifies the flow from before: + +1. Start with an empty Terraform state +1. For each `Config` and `ImportState` pair: + 1. Run `terraform apply` on the configuration, waiting for it to succeed + and recording the results in Terraform state + 1. Run `terraform plan`, and fail if Terraform detects any drift + 1. Clear the resource from state and run `terraform import` on it + 1. Deeply compare the original state from `terraform apply` and the + `terraform import` results, returning an error if any values are not + identical +1. Destroy all resources in the configuration using `terraform destroy`, + waiting for the destroy command to succeed +1. Call `GET` on the resource, and fail the test if it is still present + +For example: + +```go +func TestAccComputeFirewall_disabled(t *testing.T) { + t.Parallel() + + networkName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10)) + firewallName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10)) + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckComputeFirewallDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccComputeFirewall_disabled(networkName, firewallName), + }, + { + ResourceName: "google_compute_firewall.foobar", + ImportState: true, + ImportStateVerify: true, + }, + { + Config: testAccComputeFirewall_basic(networkName, firewallName), + }, + { + ResourceName: "google_compute_firewall.foobar", + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + +func testAccComputeFirewall_basic(network, firewall string) string { + return fmt.Sprintf(` +resource "google_compute_network" "foobar" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_firewall" "foobar" { + name = "%s" + description = "Resource created for Terraform acceptance testing" + network = google_compute_network.foobar.name + source_tags = ["foo"] + allow { + protocol = "icmp" + } +} +`, network, firewall) +} + +func testAccComputeFirewall_disabled(network, firewall string) string { + return fmt.Sprintf(` +resource "google_compute_network" "foobar" { + name = "%s" + auto_create_subnetworks = false +} + +resource "google_compute_firewall" "foobar" { + name = "%s" + description = "Resource created for Terraform acceptance testing" + network = google_compute_network.foobar.name + source_tags = ["foo"] + allow { + protocol = "icmp" + } + disabled = true +} +`, network, firewall) +} +``` + +## Testing Beta Features + +If you worked with a beta feature and had to use beta version guards in a +handwritten resource or set `min_version: beta` in a generated resource, you'll +want to version guard both the test case and configuration by enclosing them in +ERB tags like below. Additionally, if the filename ends in `.go`, rename it to +end in `.go.erb`. + +``` +<% unless version == 'ga' -%> +// test case + config here +<% end -%> +``` + +Otherwise, tests using a beta feature are written exactly the same as tests +using a GA one. Normally to use the beta provider, it's necessary to specify +`provider = google-beta`, as Terraform maps any resources prefixed with +`google_` to the `google` provider by default. However, inside the test +framework, the `google-beta` provider has been aliased as the `google` provider +and that is not necessary. + +Note: You _may_ use version guards to test different configurations between the +GA and beta provider tests, but it's strongly recommended that you write +different test cases instead, even if they're slightly duplicative. \ No newline at end of file diff --git a/docs/content/docs/how-to/add-mmv1-iam.md b/docs/content/docs/how-to/add-mmv1-iam.md new file mode 100644 index 000000000000..4d0390a7c88e --- /dev/null +++ b/docs/content/docs/how-to/add-mmv1-iam.md @@ -0,0 +1,103 @@ +--- +title: "Add MMv1 IAM resources" +weight: 11 +--- + +# Add MMv1 IAM resources + +For resources implemented through the MMv1 engine, the majority of configuration +for IAM support can be inferred based on the preexisting YAML specification file. + +To add support for IAM resources based on an existing resource, add an +`iam_policy` block to the resource's definition in `api.yaml`, such as the +following: + +```yaml + iam_policy: !ruby/object:Api::Resource::IamPolicy + method_name_separator: ':' + fetch_iam_policy_verb: :POST + parent_resource_attribute: 'registry' + import_format: ["projects/{{project}}/locations/{{location}}/registries/{{name}}", "{{name}}"] +``` + +The specification values can be determined based on a mixture of the resource +specification and the cloud.google.com `setIamPolicy`/`getIamPolicy` REST +documentation, such as +[this page](https://cloud.google.com/iot/docs/reference/cloudiot/rest/v1/projects.locations.registries/setIamPolicy) +for Cloud IOT Registries. + +`parent_resource_attribute` - (Required) determines the field name of the parent +resource reference in the IAM resources. Generally, this should be the singular +form of the parent resource kind in snake case, i.e. `registries` -> `registry` +or `backendServices` -> `backend_service`. + +`method_name_separator` - (Required) should be set to the character preceding +`setIamPolicy` in the "HTTP Request" section on the resource's `setIamPolicy` +page. This is almost always `:` for APIs other than Google Compute Engine (GCE), +MMv1's `compute` product. + +`fetch_iam_policy_verb` - (Required) should be set to the HTTP verb listed in +the "HTTP Request" section on the resource's `getIamPolicy` page. This is +generally `POST` but is occasionally `GET`. Note: This is specified as a Ruby +symbol, prefixed with a `:`. For example, for `GET`, you would specify `:GET`. + +`import_format` - (Optional) A list of templated strings used to determine the +Terraform import format. If the resource has a custom `import_format` or +`id_format` defined in `terraform.yaml`, this must be supplied. + + * If an `import_format` is set on the parent resource use that set of values exactly, substituting `parent_resource_attribute` for the field name of the **final** templated value. + * If an `id_format` is set on the parent resource use that as the first entry (substituting the final templated value, as with `import_format`) and define a second format with **only** the templated values, `/`-separated. For example, `projects/{{project}}/locations/{{region}}/myResources/{{name}}` -> `["projects/{{project}}/locations/{{region}}/myResources/{{myResource}}", "{{project}}/{{region}}/{{myResource}}"]`. + * Optionally, you may provide a version of the shortened format that excludes entries called `{{project}}`, `{{region}}`, and `{{zone}}`. For example, given `{{project}}/{{region}}/{{myResource}}/{{entry}}`, `{{myResource}}/{{entry}}` is a valid format. When a user specifies this format, the provider's default values for `project`/`region`/`zone` will be used. + +`allowed_iam_role` - (Optional) If the resource does not allow the +`roles/viewer` IAM role to be set, an alternate, valid role must be provided. + +`iam_conditions_request_type` - (Optional) The method the IAM policy version is +set in `getIamPolicy`. If unset, IAM conditions are assumed to not be supported for the resource. One of `QUERY_PARAM`, `QUERY_PARAM_NESTED` or `REQUEST_BODY`. For resources where a query parameter is expected, `QUERY_PARAM` should be used if the key is `optionsRequestedPolicyVersion`, while `QUERY_PARAM_NESTED` should be used if it is `options.requestedPolicyVersion`. + +`min_version` - (Optional) If the resource or IAM method is not generally +available, this should be set to `beta` or `alpha` as appropriate. + +`set_iam_policy_verb` - (Optional, rare) Similar to `fetch_iam_policy_verb`, the +HTTP verb expected by `setIamPolicy`. Defaults to `:POST`, and should only be +specified if it differs (typically if `:PUT` is expected). + +Several single-user settings are not documented on this page as they are not +expected to recur often. If you are unable to configure your API successfully, +you may want to consult https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/api/resource/iam_policy.rb +for additional configuration options. + +Additionally, in order to generate IAM tests based on a preexisting resource +configuration, the first `examples` entry in `terraform.yaml` must be modified +to include a `primary_resource_name` entry: + +```diff + - !ruby/object:Provider::Terraform::Examples + name: "disk_basic" + primary_resource_id: "default" ++ primary_resource_name: "fmt.Sprintf(\"tf-test-test-disk%s\", context[\"random_suffix\"])" + vars: + disk_name: "test-disk" +``` + +`primary_resource_name` - Typically +`"fmt.Sprintf(\"tf-test-{{shortname}}%s\", context[\"random_suffix\"])"`, +substituting the parent resource's shortname from the example configuration for +`{{shortname}}`, such as `test-disk` above. This value is variable, as both the +key and value are user-defined parts of the example configuration. In some cases +the value must be customized further, albeit rarely. + +Once an `iam_policy` block is added and filled out, and `primary_resource_name` +is set on the first example, you're finished, and you can run MMv1 to generate +the IAM resources you've added, alongside documentation, and tests. + +## Adding IAM support to nonexistent resources + +Some IAM targets don't exist as distinct resources, such as IAP, or their target +is supported through an engine other than MMv1 (i.e. through tpgtools/DCL or a +handwritten resource). For these resources, the `exclude_resource: true` +annotation can be used. To use it, partially define the resource in the +product's `api.yaml` file and apply the annotation. MMv1 won't attempt to +generate the resource itself and will only generate IAM resources targeting it. + +The IAP product is a good reference for adding these: https://github.com/GoogleCloudPlatform/magic-modules/tree/main/mmv1/products/iap \ No newline at end of file diff --git a/docs/content/docs/mmv1.md b/docs/content/docs/how-to/add-mmv1-resource.md similarity index 52% rename from docs/content/docs/mmv1.md rename to docs/content/docs/how-to/add-mmv1-resource.md index b41facdeb729..a879c067e51e 100644 --- a/docs/content/docs/mmv1.md +++ b/docs/content/docs/how-to/add-mmv1-resource.md @@ -1,56 +1,9 @@ --- -title: "MMv1" -weight: 1 -# bookFlatSection: false -# bookToc: true -# bookHidden: false -# bookCollapseSection: false -# bookComments: false -# bookSearchExclude: false +title: "Add an MMv1 resource" +weight: 10 --- - -# MMv1 - -## Overview - -MMv1 is a Ruby-based code generator that implements Terraform Provider Google (TPG) resources from YAML specification files. - -MMv1-generated resources like [google_compute_address](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) can be identified by looking in their [Go source](https://github.com/hashicorp/terraform-provider-google/blob/main/google/resource_compute_address.go) for an `AUTO GENERATED CODE` header as well as a Type `MMv1`. MMv1-generated resources should have source code present under their product folders, like [mmv1/products/compute](./products/compute) for the [google_compute_address](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) resource. - -## Table of Contents -- [Contributing](#contributing) - - [Resource](#resource) - - [Field Configuration](#field-configuration) - - [`api.yaml`](#apiyaml) - - [`terraform.yaml`](#terraformyaml) - - [Field Configuration - Complex Types](#field-configuration---complex-types) - - [Enum](#enum) - - [ResourceRef](#resourceref) - - [Array](#array) - - [NestedObject](#nestedobject) - - [KeyValuePairs (Labels / Annotations)](#keyvaluepairs-labels--annotations) - - [Exactly One Of](#exactly-one-of) - - [Advanced customization](#advanced-customization) - - [DiffSuppressFunc](#diffsuppressfunc) - - [IAM Resources](#iam-resource) - - [Testing](#testing) - - [Example Configuration File](#example-configuration-file) - - [`terraform.yaml` metadata](#terraformyaml-metadata) - - [Results](#results) - - [Tests that use beta features](#tests-that-use-beta-features) - - [Documentation](#documentation) - - [Beta Feature](#beta-feature) - - [Adding a beta resource](#adding-a-beta-resource) - - [Adding beta field(s)](#adding-beta-fields) - - [Tests that use a beta feature](#tests-that-use-a-beta-feature) - - [Promote a beta feature](#promote-a-beta-feature) - -## Contributing - -We're glad to accept contributions to MMv1-generated resources. Tutorials and guidance on making changes are available below. - -### Resource +# Add an MMv1 resource Generated resources are created using the `mmv1` code generator, and are configured by editing definition files under the @@ -72,9 +25,9 @@ contains the fields of the resource based on how it behaves in the API, and behaviour. Not all fields will need to be added to `terraform.yaml`- only add an entry for your field if you need to configure one of the available option(s). -#### Field Configuration +## Field Configuration -##### `api.yaml` +### `api.yaml` To add a field, you'll append an entry to `properties` within `api.yaml`, such as the following adding support for a `fooBar` field in the API: @@ -117,7 +70,7 @@ setting values to `false`, and omit them instead. as :POST for `POST`) and the URL to a templated URL such as`projects/{{project}}/global/backendServices/{{name}}/setSecurityPolicy`. -##### `terraform.yaml` +### `terraform.yaml` You can add additional values within `terraform.yaml`: @@ -145,9 +98,9 @@ Commonly configured values include the following: [`mmv1/template/terraform/custom_flatten`](https://github.com/GoogleCloudPlatform/magic-modules/tree/8728bc89c37d5033b530c7d7157bb43865d9df58/mmv1/templates/terraform/custom_flatten) respectively. -#### Field Configuration - Complex Types +## Field Configuration - Complex Types -##### Enum +### Enum ```yaml - !ruby/object:Api::Type::Enum @@ -172,7 +125,7 @@ as well. Most API enums should be typed as `String` instead- if the value will not be fixed for >1 year, use a `String`. -##### ResourceRef +### ResourceRef ```yaml - !ruby/object:Api::Type::ResourceRef @@ -193,7 +146,7 @@ In a `ResourceRef`, `resource` and `imports` must be defined but Terraform ignores those values. `resource` should be set to the resource kind, and `imports` to `selfLink` within GCE and `name` elsewhere. -##### Array +### Array ```yaml - !ruby/object:Api::Type::Array @@ -222,7 +175,7 @@ through an `item_type` field. `item_type` accepts any type, although primitives (String / Integer / Boolean) must be specified differently than other types as shown above. -##### NestedObject +### NestedObject ```yaml - !ruby/object:Api::Type::NestedObject @@ -249,7 +202,7 @@ shown above. NestedObject is an object in the JSON API, and contains a `properties` subfield where a sub-properties array can be defined (including additional NestedObjects) -##### KeyValuePairs (Labels / Annotations) +### KeyValuePairs (Labels / Annotations) ```yaml - !ruby/object:Api::Type::KeyValuePairs @@ -260,7 +213,7 @@ where a sub-properties array can be defined (including additional NestedObjects) KeyValuePairs is a special type to handle string -> string maps, such as GCE `labels` fields. No extra configuration is required. -##### Exactly One Of +### Exactly One Of To restrain a parent object to contain exactly one of its nested objects, use `exactly_one_of` in the affected child objects. @@ -289,9 +242,9 @@ objects: ... ``` -#### Advanced customization +## Advanced customization -##### DiffSuppressFunc +### DiffSuppressFunc Terraform allows fields to specify a [DiffSuppressFunc](https://www.terraform.io/plugin/sdkv2/schemas/schema-behaviors#diffsuppressfunc), @@ -353,253 +306,8 @@ Example: DomainMapping (domainMappingLabelDiffSuppress) - [constants file](https://github.com/GoogleCloudPlatform/magic-modules/blob/15fd46f60ed49ec1a6488d1b34394dcbd7cd3a41/mmv1/templates/terraform/constants/cloud_run_domain_mapping.go.erb) - [unit tests](https://github.com/GoogleCloudPlatform/magic-modules/blob/15fd46f60ed49ec1a6488d1b34394dcbd7cd3a41/mmv1/third_party/terraform/tests/resource_cloud_run_domain_mapping_test.go#L9) -### IAM Resource - -For resources implemented through the MMv1 engine, the majority of configuration -for IAM support can be inferred based on the preexisting YAML specification file. - -To add support for IAM resources based on an existing resource, add an -`iam_policy` block to the resource's definition in `api.yaml`, such as the -following: - -```yaml - iam_policy: !ruby/object:Api::Resource::IamPolicy - method_name_separator: ':' - fetch_iam_policy_verb: :POST - parent_resource_attribute: 'registry' - import_format: ["projects/{{project}}/locations/{{location}}/registries/{{name}}", "{{name}}"] -``` - -The specification values can be determined based on a mixture of the resource -specification and the cloud.google.com `setIamPolicy`/`getIamPolicy` REST -documentation, such as -[this page](https://cloud.google.com/iot/docs/reference/cloudiot/rest/v1/projects.locations.registries/setIamPolicy) -for Cloud IOT Registries. - -`parent_resource_attribute` - (Required) determines the field name of the parent -resource reference in the IAM resources. Generally, this should be the singular -form of the parent resource kind in snake case, i.e. `registries` -> `registry` -or `backendServices` -> `backend_service`. - -`method_name_separator` - (Required) should be set to the character preceding -`setIamPolicy` in the "HTTP Request" section on the resource's `setIamPolicy` -page. This is almost always `:` for APIs other than Google Compute Engine (GCE), -MMv1's `compute` product. - -`fetch_iam_policy_verb` - (Required) should be set to the HTTP verb listed in -the "HTTP Request" section on the resource's `getIamPolicy` page. This is -generally `POST` but is occasionally `GET`. Note: This is specified as a Ruby -symbol, prefixed with a `:`. For example, for `GET`, you would specify `:GET`. - -`import_format` - (Optional) A list of templated strings used to determine the -Terraform import format. If the resource has a custom `import_format` or -`id_format` defined in `terraform.yaml`, this must be supplied. - - * If an `import_format` is set on the parent resource use that set of values exactly, substituting `parent_resource_attribute` for the field name of the **final** templated value. - * If an `id_format` is set on the parent resource use that as the first entry (substituting the final templated value, as with `import_format`) and define a second format with **only** the templated values, `/`-separated. For example, `projects/{{project}}/locations/{{region}}/myResources/{{name}}` -> `["projects/{{project}}/locations/{{region}}/myResources/{{myResource}}", "{{project}}/{{region}}/{{myResource}}"]`. - * Optionally, you may provide a version of the shortened format that excludes entries called `{{project}}`, `{{region}}`, and `{{zone}}`. For example, given `{{project}}/{{region}}/{{myResource}}/{{entry}}`, `{{myResource}}/{{entry}}` is a valid format. When a user specifies this format, the provider's default values for `project`/`region`/`zone` will be used. - -`allowed_iam_role` - (Optional) If the resource does not allow the -`roles/viewer` IAM role to be set, an alternate, valid role must be provided. - -`iam_conditions_request_type` - (Optional) The method the IAM policy version is -set in `getIamPolicy`. If unset, IAM conditions are assumed to not be supported for the resource. One of `QUERY_PARAM`, `QUERY_PARAM_NESTED` or `REQUEST_BODY`. For resources where a query parameter is expected, `QUERY_PARAM` should be used if the key is `optionsRequestedPolicyVersion`, while `QUERY_PARAM_NESTED` should be used if it is `options.requestedPolicyVersion`. - -`min_version` - (Optional) If the resource or IAM method is not generally -available, this should be set to `beta` or `alpha` as appropriate. - -`set_iam_policy_verb` - (Optional, rare) Similar to `fetch_iam_policy_verb`, the -HTTP verb expected by `setIamPolicy`. Defaults to `:POST`, and should only be -specified if it differs (typically if `:PUT` is expected). - -Several single-user settings are not documented on this page as they are not -expected to recur often. If you are unable to configure your API successfully, -you may want to consult https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/api/resource/iam_policy.rb -for additional configuration options. - -Additionally, in order to generate IAM tests based on a preexisting resource -configuration, the first `examples` entry in `terraform.yaml` must be modified -to include a `primary_resource_name` entry: - -```diff - - !ruby/object:Provider::Terraform::Examples - name: "disk_basic" - primary_resource_id: "default" -+ primary_resource_name: "fmt.Sprintf(\"tf-test-test-disk%s\", context[\"random_suffix\"])" - vars: - disk_name: "test-disk" -``` - -`primary_resource_name` - Typically -`"fmt.Sprintf(\"tf-test-{{shortname}}%s\", context[\"random_suffix\"])"`, -substituting the parent resource's shortname from the example configuration for -`{{shortname}}`, such as `test-disk` above. This value is variable, as both the -key and value are user-defined parts of the example configuration. In some cases -the value must be customized further, albeit rarely. - -Once an `iam_policy` block is added and filled out, and `primary_resource_name` -is set on the first example, you're finished, and you can run MMv1 to generate -the IAM resources you've added, alongside documentation, and tests. - -#### Adding IAM support to nonexistent resources - -Some IAM targets don't exist as distinct resources, such as IAP, or their target -is supported through an engine other than MMv1 (i.e. through tpgtools/DCL or a -handwritten resource). For these resources, the `exclude_resource: true` -annotation can be used. To use it, partially define the resource in the -product's `api.yaml` file and apply the annotation. MMv1 won't attempt to -generate the resource itself and will only generate IAM resources targeting it. - -The IAP product is a good reference for adding these: https://github.com/GoogleCloudPlatform/magic-modules/tree/main/mmv1/products/iap - -### Testing - -For generated resources, you can add an example to the -[`mmv1/templates/terraform/examples`](https://github.com/GoogleCloudPlatform/magic-modules/tree/master/mmv1/templates/terraform/examples) -directory, which contains a set of templated Terraform configurations. - -After writing out the example and filling out some metadata, Magic Modules will -insert it into the resource documentation page, and generate a test case -stepping through the following stages: - -1. Run `terraform apply` on the configuration, waiting for it to succeed and - recording the results in Terraform state -1. Run `terraform plan`, and fail if Terraform detects any drift -1. Clear the resource from state and run `terraform import` on it -1. Deeply compare the original state from `terraform apply` and the `terraform - import` results, returning an error if any values are not identical -1. Destroy all resources in the configuration using `terraform destroy`, - waiting for the destroy command to succeed -1. Call `GET` on the resource, and fail the test if it is still present - -#### Example Configuration File - -First, you'll want to add the example file. It needs to end in the filename -`.tf.erb`, and is typically named `service_resource_descriptive_name`. For -example, `pubsub_topic_geo_restricted.tf.erb`. Inside, you'll write a complete -Terraform configuration that provisions the resource and all of the required -dependencies. For example, in -[`mmv1/templates/terraform/examples/pubsub_subscription_dead_letter.tf.erb`](https://github.com/GoogleCloudPlatform/magic-modules/blob/e7ef590f6007796f446b2d41875b3d26f4469ff4/mmv1/templates/terraform/examples/pubsub_subscription_dead_letter.tf.erb): - -```tf -resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>" { - name = "<%= ctx[:vars]['topic_name'] %>" -} - -resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>_dead_letter" { - name = "<%= ctx[:vars]['topic_name'] %>-dead-letter" -} - -resource "google_pubsub_subscription" "<%= ctx[:primary_resource_id] %>" { - name = "<%= ctx[:vars]['subscription_name'] %>" - topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>.name - - dead_letter_policy { - dead_letter_topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>_dead_letter.id - max_delivery_attempts = 10 - } -} -``` - -The `ctx` variable provides metadata at generation time, and should be used in -two ways: - -* The Terraform ID of a single instance of the primary resource should be - supplied through `<%= ctx[:primary_resource_id] %>` (in this example - multiple resources use the value, although only the first - `google_pubsub_topic` requires it). The resource kind you are testing with - an id equal to `<%= ctx[:primary_resource_id] %>` is the one that will be - imported. -* Unique values can be supplied through `<%= ctx[:vars]['{{var}}'] %>`, where - `{{var}}` is an arbitrary key you define. These values are created by - appending suffixes to them, and are typically only used for names- most - values should be constant within the configuration. -#### `terraform.yaml` metadata - -Once your configuration is written, go in `terraform.yaml` and find the -`examples` block for the resource. Generally it'll be above the `properties` -block. In there, append an entry such as the -[following](https://github.com/GoogleCloudPlatform/magic-modules/blob/e7ef590f6007796f446b2d41875b3d26f4469ff4/mmv1/products/pubsub/terraform.yaml#L108-L113): - -```yaml - - !ruby/object:Provider::Terraform::Examples - name: "pubsub_subscription_dead_letter" - primary_resource_id: "example" - vars: - topic_name: "example-topic" - subscription_name: "example-subscription" -``` - -The `name` should match the base name of your example file, -`primary_resource_id` is an arbitrary snake_cased string that describes the -resource, and the `vars` map should contain each key you defined previously. - -**Important**: Any vars that are part of the resource's id should include at -least one hyphen or underscore; this -[triggers addition of a `tf-test` or `tf_test` prefix](https://github.com/GoogleCloudPlatform/magic-modules/blob/6858338f013f5dc57729ec037883a3594441ea62/mmv1/provider/terraform/examples.rb#L244), -which is what we use to detect and delete stray resources that are sometimes -left over during test runs. - -#### Results - -Your configuration will ultimately generate a Go test case similar to the -[following](https://github.com/hashicorp/terraform-provider-google/blob/38e2913cb102225f9f9bda9f04b5498d3386a79c/google/resource_pubsub_subscription_generated_test.go#L135-L180) -based on the snippets above: - -```go -func TestAccPubsubSubscription_pubsubSubscriptionDeadLetterExample(t *testing.T) { - t.Parallel() - - context := map[string]interface{}{ - "random_suffix": randString(t, 10), - } - - vcrTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckPubsubSubscriptionDestroyProducer(t), - Steps: []resource.TestStep{ - { - Config: testAccPubsubSubscription_pubsubSubscriptionDeadLetterExample(context), - }, - { - ResourceName: "google_pubsub_subscription.example", - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"topic"}, - }, - }, - }) -} - -func testAccPubsubSubscription_pubsubSubscriptionDeadLetterExample(context map[string]interface{}) string { - return Nprintf(` -resource "google_pubsub_topic" "example" { - name = "tf-test-example-topic%{random_suffix}" -} -resource "google_pubsub_topic" "example_dead_letter" { - name = "tf-test-example-topic%{random_suffix}-dead-letter" -} -resource "google_pubsub_subscription" "example" { - name = "tf-test-example-subscription%{random_suffix}" - topic = google_pubsub_topic.example.name - dead_letter_policy { - dead_letter_topic = google_pubsub_topic.example_dead_letter.id - max_delivery_attempts = 10 - } -} -`, context) -} -``` - -#### Tests that use beta features - -See [Tests that use a beta feature](#tests-that-use-a-beta-feature) - -### Documentation - -### Beta Feature +# Beta features When the underlying API of a feature is not final (i.e. a `vN` version like `v1` or `v2`), is in preview, or the API has no SLO we add it to the @@ -613,7 +321,7 @@ available at, written as `min_version: {{version}}`. This is only specified when a feature is available at `beta`, and omitting a tag indicates the target is generally available, or available at `ga`. -#### Adding a beta resource +## Adding a beta resource To add support for a beta resource in a preexisting product, ensure that a `beta` level exists in the `versions` map in the `api.yaml` file for the product. @@ -670,7 +378,7 @@ to [create tests](#tests-that-use-a-beta-feature)). IAM-level tagging is only necessary in the (rare) case that a resource is available at a higher stability level than its `getIamPolicy`/`setIamPolicy` methods. -#### Adding beta field(s) +## Adding beta field(s) NOTE: If a resource is already tagged as `min_version: beta`, follow the general instructions for adding a field instead. @@ -699,53 +407,7 @@ demonstrated below: ... ``` -#### Tests that use a beta feature - -For tests that use beta features, you'll need to perform two additional steps: - -1. Add `provider = google-beta` to every resource in the test (even resources - that aren't being tested and/or are also in the GA provider) -1. Add `min_version: beta` to the `Provider::Terraform::Examples` block - -For example, modifying the snippets above: - -```tf -resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>" { - provider = google-beta - - name = "<%= ctx[:vars]['topic_name'] %>" -} - -resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>_dead_letter" { - provider = google-beta - - name = "<%= ctx[:vars]['topic_name'] %>-dead-letter" -} - -resource "google_pubsub_subscription" "<%= ctx[:primary_resource_id] %>" { - provider = google-beta - - name = "<%= ctx[:vars]['subscription_name'] %>" - topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>.name - - dead_letter_policy { - dead_letter_topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>_dead_letter.id - max_delivery_attempts = 10 - } -} -``` - -```yaml - - !ruby/object:Provider::Terraform::Examples - name: "pubsub_subscription_dead_letter" - min_version: beta - primary_resource_id: "example" - vars: - topic_name: "example-topic" - subscription_name: "example-subscription" -``` - -#### Promote a beta feature +## Promote a beta feature In order to promote a beta feature to GA, remove the version tags previously set on the feature or its tests. This will automatically make it available in the @@ -790,4 +452,3 @@ Alternatively, for field promotions, you may use "{{service}}: promoted container: promoted `node_locations` field in google_container_cluster` to GA \`\`\` ``` - diff --git a/docs/content/docs/how-to/add-mmv1-test.md b/docs/content/docs/how-to/add-mmv1-test.md new file mode 100644 index 000000000000..66ce4211adb3 --- /dev/null +++ b/docs/content/docs/how-to/add-mmv1-test.md @@ -0,0 +1,195 @@ +--- +title: "Add an MMv1 test" +weight: 12 +--- + +# Add an MMv1 test + +For generated resources, you can add an example to the +[`mmv1/templates/terraform/examples`](https://github.com/GoogleCloudPlatform/magic-modules/tree/master/mmv1/templates/terraform/examples) +directory, which contains a set of templated Terraform configurations. + +After writing out the example and filling out some metadata, Magic Modules will +insert it into the resource documentation page, and generate a test case +stepping through the following stages: + +1. Run `terraform apply` on the configuration, waiting for it to succeed and + recording the results in Terraform state +1. Run `terraform plan`, and fail if Terraform detects any drift +1. Clear the resource from state and run `terraform import` on it +1. Deeply compare the original state from `terraform apply` and the `terraform + import` results, returning an error if any values are not identical +1. Destroy all resources in the configuration using `terraform destroy`, + waiting for the destroy command to succeed +1. Call `GET` on the resource, and fail the test if it is still present + +## Example Configuration File + +First, you'll want to add the example file. It needs to end in the filename +`.tf.erb`, and is typically named `service_resource_descriptive_name`. For +example, `pubsub_topic_geo_restricted.tf.erb`. Inside, you'll write a complete +Terraform configuration that provisions the resource and all of the required +dependencies. For example, in +[`mmv1/templates/terraform/examples/pubsub_subscription_dead_letter.tf.erb`](https://github.com/GoogleCloudPlatform/magic-modules/blob/e7ef590f6007796f446b2d41875b3d26f4469ff4/mmv1/templates/terraform/examples/pubsub_subscription_dead_letter.tf.erb): + +```tf +resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['topic_name'] %>" +} + +resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>_dead_letter" { + name = "<%= ctx[:vars]['topic_name'] %>-dead-letter" +} + +resource "google_pubsub_subscription" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['subscription_name'] %>" + topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>.name + + dead_letter_policy { + dead_letter_topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>_dead_letter.id + max_delivery_attempts = 10 + } +} +``` + +The `ctx` variable provides metadata at generation time, and should be used in +two ways: + +* The Terraform ID of a single instance of the primary resource should be + supplied through `<%= ctx[:primary_resource_id] %>` (in this example + multiple resources use the value, although only the first + `google_pubsub_topic` requires it). The resource kind you are testing with + an id equal to `<%= ctx[:primary_resource_id] %>` is the one that will be + imported. +* Unique values can be supplied through `<%= ctx[:vars]['{{var}}'] %>`, where + `{{var}}` is an arbitrary key you define. These values are created by + appending suffixes to them, and are typically only used for names- most + values should be constant within the configuration. + +## `terraform.yaml` metadata + +Once your configuration is written, go in `terraform.yaml` and find the +`examples` block for the resource. Generally it'll be above the `properties` +block. In there, append an entry such as the +[following](https://github.com/GoogleCloudPlatform/magic-modules/blob/e7ef590f6007796f446b2d41875b3d26f4469ff4/mmv1/products/pubsub/terraform.yaml#L108-L113): + +```yaml + - !ruby/object:Provider::Terraform::Examples + name: "pubsub_subscription_dead_letter" + primary_resource_id: "example" + vars: + topic_name: "example-topic" + subscription_name: "example-subscription" +``` + +The `name` should match the base name of your example file, +`primary_resource_id` is an arbitrary snake_cased string that describes the +resource, and the `vars` map should contain each key you defined previously. + +**Important**: Any vars that are part of the resource's id should include at +least one hyphen or underscore; this +[triggers addition of a `tf-test` or `tf_test` prefix](https://github.com/GoogleCloudPlatform/magic-modules/blob/6858338f013f5dc57729ec037883a3594441ea62/mmv1/provider/terraform/examples.rb#L244), +which is what we use to detect and delete stray resources that are sometimes +left over during test runs. + +## Results + +Your configuration will ultimately generate a Go test case similar to the +[following](https://github.com/hashicorp/terraform-provider-google/blob/38e2913cb102225f9f9bda9f04b5498d3386a79c/google/resource_pubsub_subscription_generated_test.go#L135-L180) +based on the snippets above: + +```go +func TestAccPubsubSubscription_pubsubSubscriptionDeadLetterExample(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": randString(t, 10), + } + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckPubsubSubscriptionDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccPubsubSubscription_pubsubSubscriptionDeadLetterExample(context), + }, + { + ResourceName: "google_pubsub_subscription.example", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"topic"}, + }, + }, + }) +} + +func testAccPubsubSubscription_pubsubSubscriptionDeadLetterExample(context map[string]interface{}) string { + return Nprintf(` +resource "google_pubsub_topic" "example" { + name = "tf-test-example-topic%{random_suffix}" +} +resource "google_pubsub_topic" "example_dead_letter" { + name = "tf-test-example-topic%{random_suffix}-dead-letter" +} +resource "google_pubsub_subscription" "example" { + name = "tf-test-example-subscription%{random_suffix}" + topic = google_pubsub_topic.example.name + dead_letter_policy { + dead_letter_topic = google_pubsub_topic.example_dead_letter.id + max_delivery_attempts = 10 + } +} +`, context) +} +``` + +## Update tests + +Update tests can only be [added as handwritten tests](/magic-modules/docs/how-to/add-handwritten-test/#update-tests). + +## Tests that use beta features + +For tests that use beta features, you'll need to perform two additional steps: + +1. Add `provider = google-beta` to every resource in the test (even resources + that aren't being tested and/or are also in the GA provider) +1. Add `min_version: beta` to the `Provider::Terraform::Examples` block + +For example, modifying the snippets above: + +```tf +resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>" { + provider = google-beta + + name = "<%= ctx[:vars]['topic_name'] %>" +} + +resource "google_pubsub_topic" "<%= ctx[:primary_resource_id] %>_dead_letter" { + provider = google-beta + + name = "<%= ctx[:vars]['topic_name'] %>-dead-letter" +} + +resource "google_pubsub_subscription" "<%= ctx[:primary_resource_id] %>" { + provider = google-beta + + name = "<%= ctx[:vars]['subscription_name'] %>" + topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>.name + + dead_letter_policy { + dead_letter_topic = google_pubsub_topic.<%= ctx[:primary_resource_id] %>_dead_letter.id + max_delivery_attempts = 10 + } +} +``` + +```yaml + - !ruby/object:Provider::Terraform::Examples + name: "pubsub_subscription_dead_letter" + min_version: beta + primary_resource_id: "example" + vars: + topic_name: "example-topic" + subscription_name: "example-subscription" +``` \ No newline at end of file diff --git a/docs/content/docs/how-to/identify-mmv1-vs-handwritten.md b/docs/content/docs/how-to/identify-mmv1-vs-handwritten.md new file mode 100644 index 000000000000..03cb10a4b27e --- /dev/null +++ b/docs/content/docs/how-to/identify-mmv1-vs-handwritten.md @@ -0,0 +1,16 @@ +--- +title: "Identify MMv1 vs handwritten content" +weight: 1 +--- + +# Identify MMv1 vs handwritten content + +## MMv1 + +MMv1 is a Ruby-based code generator that implements Terraform Provider Google (TPG) resources from YAML specification files. + +MMv1-generated resources like [google_compute_address](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) can be identified by looking in their [Go source](https://github.com/hashicorp/terraform-provider-google/blob/main/google/resource_compute_address.go) for an `AUTO GENERATED CODE` header as well as a Type `MMv1`. MMv1-generated resources should have source code present under their product folders, like [mmv1/products/compute](./products/compute) for the [google_compute_address](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) resource. + +## Handwritten + +Handwritten resources like [google_container_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster) can be identified if they have source code present under the [mmv1/third_party/terraform/resources](./resources) folder or by the absence of the `AUTO GENERATED CODE` header in their [Go source](https://github.com/hashicorp/terraform-provider-google/blob/main/google/resource_container_cluster.go) in the downstream repositories. Handwritten datasources should be under the [mmv1/third_party/terraform/data_sources](./data_sources) folder, tests under the [mmv1/third_party/terraform/tests](./tests) folder and web documentation under the [mmv1/third_party/terraform/website](./website) folder. \ No newline at end of file diff --git a/docs/content/docs/handwritten.md b/docs/content/docs/how-to/update-handwritten-resource.md similarity index 51% rename from docs/content/docs/handwritten.md rename to docs/content/docs/how-to/update-handwritten-resource.md index fdf39f51b788..ff22d9b4aebe 100644 --- a/docs/content/docs/handwritten.md +++ b/docs/content/docs/how-to/update-handwritten-resource.md @@ -1,57 +1,37 @@ --- -title: "Handwritten" -weight: 1 -# bookFlatSection: false -# bookToc: true -# bookHidden: false -# bookCollapseSection: false -# bookComments: false -# bookSearchExclude: false +title: "Update a handwritten resource" +weight: 20 --- - -# Handwritten - -## Overview +# Update a handwritten resource The Google providers for Terraform have a large number of handwritten go files, primarily for resources written before Magic Modules was used with them. Most handwritten files are expected to stay handwritten indefinitely, although conversion to a generator may be possible for a limited subset of them. -Handwritten resources like [google_container_cluster](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster) can be identified if they have source code present under the [mmv1/third_party/terraform/resources](./resources) folder or by the absence of the `AUTO GENERATED CODE` header in their [Go source](https://github.com/hashicorp/terraform-provider-google/blob/main/google/resource_container_cluster.go) in the downstream repositories. Handwritten datasources should be under the [mmv1/third_party/terraform/data_sources](./data_sources) folder, tests under the [mmv1/third_party/terraform/tests](./tests) folder and web documentaion under the [mmv1/third_party/terraform/website](./website) folder. -## Table of Contents -- [Contributing](#contributing) - - [Shared Concepts](#shared-concepts) - - [Resource](#resource) - - [Datasource](#datasource) - - [IAM Resources](#iam-resource) - - [Testing](#testing) - - [Simple Tests](#simple-tests) - - [Update tests](#update-tests) - - [Testing Beta Features](#testing-beta-features) - - [Documentation](#documentation) - - [Beta Feature](#beta-feature) - - [Adding a beta resource](#adding-a-beta-resource) - - [Adding beta field(s)](#adding-beta-fields) - - [Tests that use a beta feature](#tests-that-use-a-beta-feature) - - [Promote a beta feature](#promote-a-beta-feature) +We no longer accept new handwritten resources except in rare cases. However, understanding +how to edit and add to existing resources may be important for implementing new fields +or changing existing behavior. +To edit an existing resource to add a field there are four steps you'll go through. -## Contributing +1. Add the new field to the schema +1. Implement the respective flattener and/or expander for the new field +1. Add a testcase for the field or extend an existing one +1. Add documentation for the field to the respective markdown file -We're glad to accept contributions to handwritten resources. Tutorials and guidance on making changes are available below. -### Shared Concepts +## Shared concepts This section will serve as a point of reference for some shared concepts that all handwritten files share. It's meant to be an introduction to our serialization strategy and overview. -#### Serialization strategy +### Serialization strategy The go files within the directory files are copied literally to their respective providers. Our serialization methodology may seem complicated but for the case of handwritten resources its quite simple. Editing the file will change its counterpart downstream. -#### go and go.erb +### go and go.erb Within the third party library you'll notice `go` and `go.erb` files. Go files are native golang code while go.erb pass through ruby before being serialized. The reason `go.erb` files exist are to protect certain @@ -61,14 +41,14 @@ will omit the enclosure from being output to the GA provider. In the rare case where you are promoting all fields to `ga` and these blocks are no longer needed you can remove the `.erb` extension. -#### Create, Read, Update, Delete +### Create, Read, Update, Delete As far as terraform schema is concerned these are the functions we need to provide for terraform to be able to provision and delete resources. In editing any fields you'll likely be adding functionality to these functions or implementing them wholesale. -#### Expanders and Flatteners +### Expanders and Flatteners Expanders and flatteners are concepts created to simplify common patterns and add conformity/code consistency. Essentially expanders are functions used to segregate some translation from terraform representation to api representation. @@ -81,18 +61,7 @@ Thus * expanders - helper functions used for translating tf -> api representation * flatteners - helper functions used for translating api -> tf representation -### Resource -While we no longer accept new handwritten resources except in rare cases. Understanding -how to edit and add to existing resources may be important for implementing new fields -or changing existing behavior. - -To edit an existing resource to add a field there are four steps you'll go through. -1. Add the new field to the schema -1. Implement the respective flattener and/or expander for the new field -1. Add a testcase for the field or extend an existing one -1. Add documentation for the field to the respective markdown file - -##### Adding a new field to the schema +## Adding a new field to the schema To add a new field you will have to compare an existing resource to it's respective rest api documentation. Dependant on how the api implements the field we will in almost all cases mirror the structure. For example if there is @@ -126,7 +95,7 @@ added to it by the api and this future proofs our provider to support new values haven't to make new additions. There will be rare exceptions, but generally its a good practice. -#### Implement the respective flattener and/or expander for the new field +## Implement the respective flattener and/or expander for the new field Once you've added the field to the schema you will implement the corresponding expander/flattener. See [expanders and flatters](#expanders-and-flatteners) for more context on what these fields are used for. Essentially we will be editing the @@ -212,7 +181,7 @@ func flattenGoogleSheetsOptions(opts *bigquery.GoogleSheetsOptions) []map[string ``` -#### Add a testcase for the field or extend an existing one +## Add a testcase for the field or extend an existing one Once your field has been implemented, go to the corresponding test file for your resource and extend it. If your field is updatable it's good practice to have a two step apply to ensure that the field *can* be updated. You'll notice @@ -222,271 +191,12 @@ you just provisioned and *verify* that the field values are consistent with the applied state. Please test all fields you've added to the provider. It's important for us to ensure all fields are usable and workable. -#### Add documentation for the field to the respective markdown file +## Add documentation for the field to the respective markdown file See [Documentation](#documentation) for more information. Essentially you will just be opening the corresponding markdown file and adding documentation, likely copied from the rest api to the markdown file. Follow the existing patterns there-in. -### Datasource -**Note** : only handwritten datasources are currently supported - -Datasources are like terraform resources except they don't *create* anything. -They are simply read-only operations that will expose some sort of values needed -for subsequent resource operations. If you're adding a field to an existing -datasource, check the [Resource](#resource) section. Everything there will -be mostly consistent with the type of change you'll need to make. For adding -a new datasource there are 5 steps to doing so. - -1. Create a new datasource declaration file and a corresponding test file -1. Add Schema and Read operation implementation -1. Add the datasource to the `provider.go.erb` index -1. Implement a test which will create and resources and read the corresponding - datasource. -1. Add documentation. - -For creating a datasource based off an existing resource you can [make use of the -schema directly](https://github.com/GoogleCloudPlatform/magic-modules/blob/1d293f7bfadacaa20580874c8e8634827fb99a14/mmv1/third_party/terraform/data_sources/data_source_cloud_run_service.go). -Otherwise [implementing the schema directly](https://github.com/GoogleCloudPlatform/magic-modules/blob/1d293f7bfadacaa20580874c8e8634827fb99a14/mmv1/third_party/terraform/data_sources/data_source_google_compute_address.go), -similar to normal resource creation, is the desired path. - -### IAM Resource - -Handwritten IAM support is only recommended for resources that cannot be managed -using [MMv1](https://github.com/GoogleCloudPlatform/magic-modules/tree/main/mmv1#iam-resource), -including for handwritten resources, due to the need to manage tests and -documentation by hand. This guidance goes through the motions of adding support -for new handwritten IAM resources, but does not go into the details of the -implementation as any new handwritten IAM resources are expected to be -exceptional. - -IAM resources are implemented using an IAM framework, where you implement an -interface for each parent resource supporting `getIamPolicy`/`setIamPolicy` and -the associated IAM resources that target that parent resource- `_member`, -`_binding`, and `_policy`- are created by the framework. - -To add support for a new target, create a new file in -`mmv1/third_party/terraform/utils` called `iam_{{resource}}.go`, and implement -the `ResourceIamUpdater`, `newResourceIamUpdaterFunc`, `iamPolicyModifyFunc`, -`resourceIdParserFunc` interfaces from -https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/third_party/terraform/utils/iam.go.erb -in public types, alongside a public `map[string]*schema.Schema` containing all -fields referenced in the resource. - -Once your implementation is complete, add the IAM resources to `provider.go` -inside the `START non-generated IAM resources` block, creating the concrete -resource types using the `ResourceIamMember`, `ResourceIamBinding`, and -`ResourceIamPolicy` functions. For example: - -```go - "google_bigtable_instance_iam_binding": ResourceIamBinding(IamBigtableInstanceSchema, NewBigtableInstanceUpdater, BigtableInstanceIdParseFunc), - "google_bigtable_instance_iam_member": ResourceIamMember(IamBigtableInstanceSchema, NewBigtableInstanceUpdater, BigtableInstanceIdParseFunc), - "google_bigtable_instance_iam_policy": ResourceIamPolicy(IamBigtableInstanceSchema, NewBigtableInstanceUpdater, BigtableInstanceIdParseFunc), -``` - -Following that, write a test for each resource exercising create and update for -both `_policy` and `_binding`, and create for `_member`. No special -accommodations are needed for the IAM test compared to a normal Terraform -resource test. - -Documentation for IAM resources is done using single page per target resource, -rather than a distinct page for each IAM resource level. As most of the page is -standard, you can generally copy and edit an existing handwritten page such as -https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/third_party/terraform/website/docs/r/bigtable_instance_iam.html.markdown -to write the documentation. - -### Testing - -For handwritten resources and generated resources that need to test update, -handwritten tests must be added. - -Tests are made up of a templated Terraform configuration where unique values -like GCE names are passed in as arguments, and boilerplate to exercise that -configuration. - -The test boilerplate effectively does the following: - -1. Run `terraform apply` on the configuration, waiting for it to succeed and - recording the results in Terraform state -2. Run `terraform plan`, and fail if Terraform detects any drift -3. Clear the resource from state and run `terraform import` on it -4. Deeply compare the original state from `terraform apply` and the `terraform - import` results, returning an error if any values are not identical -5. Destroy all resources in the configuration using `terraform destroy`, - waiting for the destroy command to succeed -6. Call `GET` on the resource, and fail the test if it is still present - -#### Simple Tests - -Terraform configurations are stored as string constants wrapped in Go functions -like the following: - -```go -func testAccComputeFirewall_basic(network, firewall string) string { - return fmt.Sprintf(` -resource "google_compute_network" "foobar" { - name = "%s" - auto_create_subnetworks = false -} - -resource "google_compute_firewall" "foobar" { - name = "%s" - description = "Resource created for Terraform acceptance testing" - network = google_compute_network.foobar.name - source_tags = ["foo"] - allow { - protocol = "icmp" - } -} -`, network, firewall) -} -``` - -For the most part, you can copy and paste a preexisting test case and modify it. -For example, the following test case is a good reference: - -```go -func TestAccComputeFirewall_noSource(t *testing.T) { - t.Parallel() - - networkName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10)) - firewallName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10)) - - vcrTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckComputeFirewallDestroyProducer(t), - Steps: []resource.TestStep{ - { - Config: testAccComputeFirewall_noSource(networkName, firewallName), - }, - { - ResourceName: "google_compute_firewall.foobar", - ImportState: true, - ImportStateVerify: true, - }, - }, - }) -} - -func testAccComputeFirewall_noSource(network, firewall string) string { - return fmt.Sprintf(` -resource "google_compute_network" "foobar" { - name = "%s" - auto_create_subnetworks = false -} -resource "google_compute_firewall" "foobar" { - name = "%s" - description = "Resource created for Terraform acceptance testing" - network = google_compute_network.foobar.name - allow { - protocol = "tcp" - ports = [22] - } -} -`, network, firewall) -} -``` - -#### Update tests - -Inside of a test, additional steps can be added in order to transition between -Terraform configurations, updating the stored state as it progresses. This -allows you to exercise update behaviour. This modifies the flow from before: - -1. Start with an empty Terraform state -1. For each `Config` and `ImportState` pair: - 1. Run `terraform apply` on the configuration, waiting for it to succeed - and recording the results in Terraform state - 1. Run `terraform plan`, and fail if Terraform detects any drift - 1. Clear the resource from state and run `terraform import` on it - 1. Deeply compare the original state from `terraform apply` and the - `terraform import` results, returning an error if any values are not - identical -1. Destroy all resources in the configuration using `terraform destroy`, - waiting for the destroy command to succeed -1. Call `GET` on the resource, and fail the test if it is still present - -For example: - -```go -func TestAccComputeFirewall_disabled(t *testing.T) { - t.Parallel() - - networkName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10)) - firewallName := fmt.Sprintf("tf-test-firewall-%s", randString(t, 10)) - - vcrTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - Providers: testAccProviders, - CheckDestroy: testAccCheckComputeFirewallDestroyProducer(t), - Steps: []resource.TestStep{ - { - Config: testAccComputeFirewall_disabled(networkName, firewallName), - }, - { - ResourceName: "google_compute_firewall.foobar", - ImportState: true, - ImportStateVerify: true, - }, - { - Config: testAccComputeFirewall_basic(networkName, firewallName), - }, - { - ResourceName: "google_compute_firewall.foobar", - ImportState: true, - ImportStateVerify: true, - }, - }, - }) -} - -func testAccComputeFirewall_basic(network, firewall string) string { - return fmt.Sprintf(` -resource "google_compute_network" "foobar" { - name = "%s" - auto_create_subnetworks = false -} - -resource "google_compute_firewall" "foobar" { - name = "%s" - description = "Resource created for Terraform acceptance testing" - network = google_compute_network.foobar.name - source_tags = ["foo"] - allow { - protocol = "icmp" - } -} -`, network, firewall) -} - -func testAccComputeFirewall_disabled(network, firewall string) string { - return fmt.Sprintf(` -resource "google_compute_network" "foobar" { - name = "%s" - auto_create_subnetworks = false -} - -resource "google_compute_firewall" "foobar" { - name = "%s" - description = "Resource created for Terraform acceptance testing" - network = google_compute_network.foobar.name - source_tags = ["foo"] - allow { - protocol = "icmp" - } - disabled = true -} -`, network, firewall) -} -``` - -#### Testing Beta Features - -See [Tests that use a beta feature](#tests-that-use-a-beta-feature) - -### Documentation - -### Beta Feature +## Beta features When the underlying API of a feature is not final (i.e. a `vN` version like `v1` or `v2`), is in preview, or the API has no SLO we add it to the @@ -538,7 +248,7 @@ have the following guarded import: This is not necessary for beta-only services, or for services that are generally available in their entirety. -#### Adding a beta resource +### Adding a beta resource MMv1 doesn't selectively generate files, and any file that is beta-only must have all of its contents guarded. When writing a resource that's available at @@ -584,7 +294,7 @@ If this is a new service entirely, all service-specific entries like client factory initialization should be guarded as well. However, new services should generally be implemented using an alternate engine- either MMv1 or tpgtools/DCL. -#### Adding beta field(s) +### Adding beta field(s) By contrast to beta resources, adding support for a beta field is much more involved as small snippets of code throughout a resource file must be annotated. @@ -617,32 +327,7 @@ Even if there are other guarded fields, it's recommended that you add distinct guards per feature- that way, promotion (covered below) will be simpler as you'll only need to remove lines rather that move them around. -#### Tests that use a beta feature - -If you worked with a beta feature and had to use beta version guards in a -handwritten resource or set `min_version: beta` in a generated resource, you'll -want to version guard both the test case and configuration by enclosing them in -ERB tags like below. Additionally, if the filename ends in `.go`, rename it to -end in `.go.erb`. - -``` -<% unless version == 'ga' -%> -// test case + config here -<% end -%> -``` - -Otherwise, tests using a beta feature are written exactly the same as tests -using a GA one. Normally to use the beta provider, it's necessary to specify -`provider = google-beta`, as Terraform maps any resources prefixed with -`google_` to the `google` provider by default. However, inside the test -framework, the `google-beta` provider has been aliased as the `google` provider -and that is not necessary. - -Note: You _may_ use version guards to test different configurations between the -GA and beta provider tests, but it's strongly recommended that you write -different test cases instead, even if they're slightly duplicative. - -#### Promote a beta feature +### Promoting a beta feature "Promoting" a beta feature- making it available in the GA `google` provider when the underlying feature or service has gone GA- requires removing the version diff --git a/docs/content/docs/reference/_index.md b/docs/content/docs/reference/_index.md new file mode 100644 index 000000000000..2cc9d2640f4a --- /dev/null +++ b/docs/content/docs/reference/_index.md @@ -0,0 +1,5 @@ +--- +title: "Reference" +weight: 20 +isSection: true +--- \ No newline at end of file diff --git a/docs/content/docs/reference/api-yaml-resource.md b/docs/content/docs/reference/api-yaml-resource.md new file mode 100644 index 000000000000..87e64284565c --- /dev/null +++ b/docs/content/docs/reference/api-yaml-resource.md @@ -0,0 +1,6 @@ +--- +title: "api.yaml resource ↗" +weight: 0 +bookHref: "https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/api/resource.rb" +--- +FORCE MENU RENDER \ No newline at end of file diff --git a/docs/content/docs/reference/terraform-yaml-resource.md b/docs/content/docs/reference/terraform-yaml-resource.md new file mode 100644 index 000000000000..c6104f149cb3 --- /dev/null +++ b/docs/content/docs/reference/terraform-yaml-resource.md @@ -0,0 +1,6 @@ +--- +title: "terraform.yaml resource ↗" +weight: 0 +bookHref: "https://github.com/GoogleCloudPlatform/magic-modules/blob/main/mmv1/overrides/terraform/resource_override.rb" +--- +FORCE MENU RENDER \ No newline at end of file