From 857b23d8ee48c295614cbd1cef693d331628ad13 Mon Sep 17 00:00:00 2001 From: Modular Magician Date: Tue, 12 Apr 2022 20:55:27 +0000 Subject: [PATCH] Add gc_rules to BigTable's Garbage Collection policy TF resource. (#5800) Signed-off-by: Modular Magician --- .changelog/5800.txt | 3 + google-beta/resource_bigtable_gc_policy.go | 139 +++++++++++- .../resource_bigtable_gc_policy_test.go | 208 ++++++++++++++++++ .../docs/r/bigtable_gc_policy.html.markdown | 67 ++++++ 4 files changed, 408 insertions(+), 9 deletions(-) create mode 100644 .changelog/5800.txt diff --git a/.changelog/5800.txt b/.changelog/5800.txt new file mode 100644 index 0000000000..18695c3060 --- /dev/null +++ b/.changelog/5800.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +bigtable: added `gc_rules` to `google_bigtable_gc_policy` resource. +``` diff --git a/google-beta/resource_bigtable_gc_policy.go b/google-beta/resource_bigtable_gc_policy.go index 2fea182444..a814811ddc 100644 --- a/google-beta/resource_bigtable_gc_policy.go +++ b/google-beta/resource_bigtable_gc_policy.go @@ -2,12 +2,15 @@ package google import ( "context" + "encoding/json" "fmt" "log" + "strings" "time" "cloud.google.com/go/bigtable" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/structure" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" ) @@ -56,9 +59,10 @@ func resourceBigtableGCPolicyCustomizeDiff(_ context.Context, d *schema.Resource func resourceBigtableGCPolicy() *schema.Resource { return &schema.Resource{ - Create: resourceBigtableGCPolicyCreate, + Create: resourceBigtableGCPolicyUpsert, Read: resourceBigtableGCPolicyRead, Delete: resourceBigtableGCPolicyDestroy, + Update: resourceBigtableGCPolicyUpsert, CustomizeDiff: resourceBigtableGCPolicyCustomizeDiff, Schema: map[string]*schema.Schema{ @@ -84,12 +88,24 @@ func resourceBigtableGCPolicy() *schema.Resource { Description: `The name of the column family.`, }, + "gc_rules": { + Type: schema.TypeString, + Optional: true, + Description: `Serialized JSON string for garbage collection policy. Conflicts with "mode", "max_age" and "max_version".`, + ValidateFunc: validation.StringIsJSON, + ConflictsWith: []string{"mode", "max_age", "max_version"}, + StateFunc: func(v interface{}) string { + json, _ := structure.NormalizeJsonString(v) + return json + }, + }, "mode": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - Description: `If multiple policies are set, you should choose between UNION OR INTERSECTION.`, - ValidateFunc: validation.StringInSlice([]string{GCPolicyModeIntersection, GCPolicyModeUnion}, false), + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: `If multiple policies are set, you should choose between UNION OR INTERSECTION.`, + ValidateFunc: validation.StringInSlice([]string{GCPolicyModeIntersection, GCPolicyModeUnion}, false), + ConflictsWith: []string{"gc_rules"}, }, "max_age": { @@ -120,6 +136,7 @@ func resourceBigtableGCPolicy() *schema.Resource { }, }, }, + ConflictsWith: []string{"gc_rules"}, }, "max_version": { @@ -137,6 +154,7 @@ func resourceBigtableGCPolicy() *schema.Resource { }, }, }, + ConflictsWith: []string{"gc_rules"}, }, "project": { @@ -151,7 +169,7 @@ func resourceBigtableGCPolicy() *schema.Resource { } } -func resourceBigtableGCPolicyCreate(d *schema.ResourceData, meta interface{}) error { +func resourceBigtableGCPolicyUpsert(d *schema.ResourceData, meta interface{}) error { config := meta.(*Config) userAgent, err := generateUserAgentString(d, config.userAgent) if err != nil { @@ -288,13 +306,22 @@ func generateBigtableGCPolicy(d *schema.ResourceData) (bigtable.GCPolicy, error) mode := d.Get("mode").(string) ma, aok := d.GetOk("max_age") mv, vok := d.GetOk("max_version") + gcRules, gok := d.GetOk("gc_rules") - if !aok && !vok { + if !aok && !vok && !gok { return bigtable.NoGcPolicy(), nil } if mode == "" && aok && vok { - return nil, fmt.Errorf("If multiple policies are set, mode can't be empty") + return nil, fmt.Errorf("if multiple policies are set, mode can't be empty") + } + + if gok { + var j map[string]interface{} + if err := json.Unmarshal([]byte(gcRules.(string)), &j); err != nil { + return nil, err + } + return getGCPolicyFromJSON(j) } if aok { @@ -324,6 +351,100 @@ func generateBigtableGCPolicy(d *schema.ResourceData) (bigtable.GCPolicy, error) return policies[0], nil } +func getGCPolicyFromJSON(topLevelPolicy map[string]interface{}) (bigtable.GCPolicy, error) { + policy := []bigtable.GCPolicy{} + + if err := validateNestedPolicy(topLevelPolicy, true); err != nil { + return nil, err + } + + for _, p := range topLevelPolicy["rules"].([]interface{}) { + childPolicy := p.(map[string]interface{}) + if err := validateNestedPolicy(childPolicy, false); err != nil { + return nil, err + } + + if childPolicy["max_age"] != nil { + maxAge := childPolicy["max_age"].(string) + duration, err := time.ParseDuration(maxAge) + if err != nil { + return nil, fmt.Errorf("invalid duration string: %v", maxAge) + } + policy = append(policy, bigtable.MaxAgePolicy(duration)) + } + + if childPolicy["max_version"] != nil { + version := childPolicy["max_version"].(float64) + policy = append(policy, bigtable.MaxVersionsPolicy(int(version))) + } + + if childPolicy["mode"] != nil { + n, err := getGCPolicyFromJSON(childPolicy) + if err != nil { + return nil, err + } + policy = append(policy, n) + } + } + + switch topLevelPolicy["mode"] { + case strings.ToLower(GCPolicyModeUnion): + return bigtable.UnionPolicy(policy...), nil + case strings.ToLower(GCPolicyModeIntersection): + return bigtable.IntersectionPolicy(policy...), nil + default: + return policy[0], nil + } +} + +func validateNestedPolicy(p map[string]interface{}, topLevel bool) error { + if len(p) > 2 { + return fmt.Errorf("rules has more than 2 fields") + } + maxVersion, maxVersionOk := p["max_version"] + maxAge, maxAgeOk := p["max_age"] + rulesObj, rulesOk := p["rules"] + + _, modeOk := p["mode"] + rules, arrOk := rulesObj.([]interface{}) + _, vCastOk := maxVersion.(float64) + _, aCastOk := maxAge.(string) + + if rulesOk && !arrOk { + return fmt.Errorf("`rules` must be array") + } + + if modeOk && len(rules) < 2 { + return fmt.Errorf("`rules` need at least 2 GC rule when mode is specified") + } + + if topLevel && !rulesOk { + return fmt.Errorf("invalid nested policy, need `rules`") + } + + if topLevel && !modeOk && len(rules) != 1 { + return fmt.Errorf("when `mode` is not specified, `rules` can only have 1 child rule") + } + + if !topLevel && len(p) == 2 && (!modeOk || !rulesOk) { + return fmt.Errorf("need `mode` and `rules` for child nested policies") + } + + if !topLevel && len(p) == 1 && !maxVersionOk && !maxAgeOk { + return fmt.Errorf("need `max_version` or `max_age` for the rule") + } + + if maxVersionOk && !vCastOk { + return fmt.Errorf("`max_version` must be a number") + } + + if maxAgeOk && !aCastOk { + return fmt.Errorf("`max_age must be a string") + } + + return nil +} + func getMaxAgeDuration(values map[string]interface{}) (time.Duration, error) { d := values["duration"].(string) if d != "" { diff --git a/google-beta/resource_bigtable_gc_policy_test.go b/google-beta/resource_bigtable_gc_policy_test.go index 7b4beb58a1..7ed3f61222 100644 --- a/google-beta/resource_bigtable_gc_policy_test.go +++ b/google-beta/resource_bigtable_gc_policy_test.go @@ -2,6 +2,7 @@ package google import ( "context" + "encoding/json" "fmt" "testing" @@ -127,6 +128,42 @@ func TestAccBigtableGCPolicy_multiplePolicies(t *testing.T) { }) } +func TestAccBigtableGCPolicy_gcRulesPolicy(t *testing.T) { + skipIfVcr(t) + t.Parallel() + + instanceName := fmt.Sprintf("tf-test-%s", randString(t, 10)) + tableName := fmt.Sprintf("tf-test-%s", randString(t, 10)) + familyName := fmt.Sprintf("tf-test-%s", randString(t, 10)) + + gcRulesOriginal := "{\"mode\":\"intersection\",\"rules\":[{\"max_age\":\"10h\"},{\"max_version\":2}]}" + gcRulesUpdate := "{\"mode\":\"intersection\",\"rules\":[{\"max_age\":\"16h\"},{\"max_version\":1}]}" + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckBigtableGCPolicyDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccBigtableGCPolicy_gcRulesCreate(instanceName, tableName, familyName), + Check: resource.ComposeTestCheckFunc( + testAccBigtableGCPolicyExists(t, "google_bigtable_gc_policy.policy"), + resource.TestCheckResourceAttr("google_bigtable_gc_policy.policy", "gc_rules", gcRulesOriginal), + ), + }, + // Testing gc_rules update + // TODO: Add test to verify no data loss + { + Config: testAccBigtableGCPolicy_gcRulesUpdate(instanceName, tableName, familyName), + Check: resource.ComposeTestCheckFunc( + testAccBigtableGCPolicyExists(t, "google_bigtable_gc_policy.policy"), + resource.TestCheckResourceAttr("google_bigtable_gc_policy.policy", "gc_rules", gcRulesUpdate), + ), + }, + }, + }) +} + func TestUnitBigtableGCPolicy_customizeDiff(t *testing.T) { for _, tc := range testUnitBigtableGCPolicyCustomizeDiffTestcases { tc.check(t) @@ -157,6 +194,111 @@ func (testcase *testUnitBigtableGCPolicyCustomizeDiffTestcase) check(t *testing. } } +type testUnitBigtableGCPolicyJSONRules struct { + name string + gcJSONString string + want string + errorExpected bool +} + +var testUnitBigtableGCPolicyRulesTestCases = []testUnitBigtableGCPolicyJSONRules{ + { + name: "Simple policy", + gcJSONString: `{"rules":[{"max_age":"10h"}]}`, + want: "age() > 10h", + errorExpected: false, + }, + { + name: "Simple multiple policies", + gcJSONString: `{"mode":"union", "rules":[{"max_age":"10h"},{"max_version":2}]}`, + want: "(age() > 10h || versions() > 2)", + errorExpected: false, + }, + { + name: "Nested policy", + gcJSONString: `{"mode":"union", "rules":[{"max_age":"10h"},{"mode": "intersection", "rules":[{"max_age":"2h"}, {"max_version":2}]}]}`, + want: "(age() > 10h || (age() > 2h && versions() > 2))", + errorExpected: false, + }, + { + name: "JSON with no `rules`", + gcJSONString: `{"mode": "union"}`, + errorExpected: true, + }, + { + name: "Empty JSON", + gcJSONString: "{}", + errorExpected: true, + }, + { + name: "Invalid duration string", + errorExpected: true, + gcJSONString: `{"mode":"union","rules":[{"max_age":"12o"},{"max_version":2}]}`, + }, + { + name: "Empty mode policy with more than 1 rules", + gcJSONString: `{"rules":[{"max_age":"10h"}, {"max_version":2}]}`, + errorExpected: true, + }, + { + name: "Less than 2 rules with mode specified", + gcJSONString: `{"mode":"union", "rules":[{"max_version":2}]}`, + errorExpected: true, + }, + { + name: "Invalid GC rule object", + gcJSONString: `{"mode": "union", "rules": [{"mode": "intersection"}]}`, + errorExpected: true, + }, + { + name: "Invalid GC rule field: not max_version or max_age", + gcJSONString: `{"mode": "union", "rules": [{"max_versions": 2}]}`, + errorExpected: true, + }, + { + name: "Invalid GC rule field: additional fields", + gcJSONString: `{"mode": "union", "rules": [{"max_age": "10h", "something_else": 100}]}`, + errorExpected: true, + }, + { + name: "Invalid GC rule field: more than 2 fields in a gc rule object", + gcJSONString: `{"mode": "union", "rules": [{"max_age": "10h", "max_version": 10, "something": 100}]}`, + errorExpected: true, + }, + { + name: "Invalid GC rule field: max_version or max_age is in the wrong type", + gcJSONString: `{"mode": "union", "rules": [{"max_age": "10d", "max_version": 2}]}`, + errorExpected: true, + }, + { + name: "Invalid GC rule: wrong data type for child gc_rule", + gcJSONString: `{"rules": {"max_version": "456"}}`, + errorExpected: true, + }, +} + +func TestUnitBigtableGCPolicy_getGCPolicyFromJSON(t *testing.T) { + for _, tc := range testUnitBigtableGCPolicyRulesTestCases { + t.Run(tc.name, func(t *testing.T) { + var j map[string]interface{} + err := json.Unmarshal([]byte(tc.gcJSONString), &j) + if err != nil { + t.Fatalf("error unmarshalling JSON string: %v", err) + } + got, err := getGCPolicyFromJSON(j) + if tc.errorExpected && err == nil { + t.Fatal("expect error, got nil") + } else if !tc.errorExpected && err != nil { + t.Fatalf("unexpected error: %v", err) + } else { + if got != nil && got.String() != tc.want { + t.Errorf("error getting policy from JSON, got: %v, want: %v", tc.want, got) + } + } + }) + } +} + type testUnitBigtableGCPolicyCustomizeDiffTestcase struct { testName string arraySize int @@ -520,3 +662,69 @@ resource "google_bigtable_gc_policy" "policyC" { } `, instanceName, instanceName, tableName, family, family, family, family) } + +func testAccBigtableGCPolicy_gcRulesCreate(instanceName, tableName, family string) string { + return fmt.Sprintf(` + resource "google_bigtable_instance" "instance" { + name = "%s" + + cluster { + cluster_id = "%s" + zone = "us-central1-b" + } + + instance_type = "DEVELOPMENT" + deletion_protection = false + } + + resource "google_bigtable_table" "table" { + name = "%s" + instance_name = google_bigtable_instance.instance.id + + column_family { + family = "%s" + } + } + + resource "google_bigtable_gc_policy" "policy" { + instance_name = google_bigtable_instance.instance.id + table = google_bigtable_table.table.name + column_family = "%s" + + gc_rules = "{\"mode\":\"intersection\", \"rules\":[{\"max_age\":\"10h\"},{\"max_version\":2}]}" + } +`, instanceName, instanceName, tableName, family, family) +} + +func testAccBigtableGCPolicy_gcRulesUpdate(instanceName, tableName, family string) string { + return fmt.Sprintf(` + resource "google_bigtable_instance" "instance" { + name = "%s" + + cluster { + cluster_id = "%s" + zone = "us-central1-b" + } + + instance_type = "DEVELOPMENT" + deletion_protection = false + } + + resource "google_bigtable_table" "table" { + name = "%s" + instance_name = google_bigtable_instance.instance.id + + column_family { + family = "%s" + } + } + + resource "google_bigtable_gc_policy" "policy" { + instance_name = google_bigtable_instance.instance.id + table = google_bigtable_table.table.name + column_family = "%s" + + gc_rules = "{\"mode\":\"intersection\", \"rules\":[{\"max_age\":\"16h\"},{\"max_version\":1}]}" + } +`, instanceName, instanceName, tableName, family, family) +} diff --git a/website/docs/r/bigtable_gc_policy.html.markdown b/website/docs/r/bigtable_gc_policy.html.markdown index e589047908..eeb4ce4134 100644 --- a/website/docs/r/bigtable_gc_policy.html.markdown +++ b/website/docs/r/bigtable_gc_policy.html.markdown @@ -66,6 +66,65 @@ resource "google_bigtable_gc_policy" "policy" { } ``` +For complex, nested policies, an optional `gc_rules` field are supported. This field +conflicts with `mode`, `max_age` and `max_version`. This field is a serialized JSON +string. Example: +```hcl +resource "google_bigtable_instance" "instance" { + name = "instance_name" + + cluster { + cluster_id = "cid" + zone = "us-central1-b" + } + + instance_type = "DEVELOPMENT" + deletion_protection = false +} + +resource "google_bigtable_table" "table" { + name = "your-table" + instance_name = google_bigtable_instance.instance.id + + column_family { + family = "cf1" + } +} + +resource "google_bigtable_gc_policy" "policy" { + instance_name = google_bigtable_instance.instance.id + table = google_bigtable_table.table.name + column_family = "cf1" + + gc_rules = <