test: allow using global variables in suite-level variable definitions

Closes #34534

internal/backend/local/test.go

@@ -244,7 +244,10 @@ type TestFileRunner struct {
 	// variables within run blocks.
 	PriorOutputs map[addrs.Run]cty.Value
 
+	// globalVariables are globally defined variables, e.g. through tfvars or CLI flags
 	globalVariables map[string]backend.UnparsedVariableValue
+	// fileVariables are defined in the variables section of a test file
+	fileVariables map[string]hcl.Expression
 }
 
 // TestFileState is a helper struct that just maps a run block to the state that
@@ -992,7 +995,21 @@ func (runner *TestFileRunner) GetVariables(config *configs.Config, run *modulete
 		}
 	}
 
-	// Second, we'll check to see which variables the run block variables
+	// Second, we'll check to see which variables the suites global variables
+	// themselves reference.
+	for _, expr := range runner.fileVariables {
+		for _, variable := range expr.Variables() {
+			reference, referenceDiags := addrs.ParseRefFromTestingScope(variable)
+			diags = diags.Append(referenceDiags)
+			if reference != nil {
+				if addr, ok := reference.Subject.(addrs.InputVariable); ok {
+					relevantVariables[addr.Name] = true
+				}
+			}
+		}
+	}
+
+	// Third, we'll check to see which variables the run block variables
 	// themselves reference. We might be processing variables just for the file
 	// so the run block itself could be nil.
 	for _, expr := range run.Config.Variables {
@@ -1022,6 +1039,8 @@ func (runner *TestFileRunner) GetVariables(config *configs.Config, run *modulete
 	//   - ConfigVariables variables, defined directly within the config.
 	values := make(terraform.InputValues)
 
+	fmt.Printf("\nrunner.globalVariables: \n\t%#v \n", runner.globalVariables)
+
 	// First, let's look at the global variables.
 	for name, variable := range runner.globalVariables {
 		if !relevantVariables[name] {
@@ -1033,6 +1052,7 @@ func (runner *TestFileRunner) GetVariables(config *configs.Config, run *modulete
 		parsingMode := configs.VariableParseHCL
 
 		cfg, exists := config.Module.Variables[name]
+
 		if exists {
 			// Unless we have some configuration that can actually tell us
 			// what parsing mode to use.
@@ -1057,29 +1077,82 @@ func (runner *TestFileRunner) GetVariables(config *configs.Config, run *modulete
 		values[name] = value
 	}
 
-	// Second, we'll check the run level variables.
+	// Second, we'll check the file level variables
+	var exprs []hcl.Expression
+	for _, expr := range runner.fileVariables {
+		exprs = append(exprs, expr)
+	}
+
+	// Preformat the variables we've processed already - these will be made
+	// available to the eval context.
+	variables := make(map[string]cty.Value)
+	for name, value := range values {
+		variables[name] = value.Value
+	}
+
+	fmt.Printf("\nvariables: \n\t%#v \n", variables)
+
+	ctx, ctxDiags := hcltest.EvalContext(hcltest.TargetRunBlock, exprs, variables, runner.PriorOutputs)
+	diags = diags.Append(ctxDiags)
+
+	var failedContext bool
+	if ctxDiags.HasErrors() {
+		// If we couldn't build the context, we won't actually process these
+		// variables. Instead, we'll fill them with an empty value but still
+		// make a note that the user did provide them.
+		failedContext = true
+	}
+
+	for name, expr := range runner.fileVariables {
+		if !relevantVariables[name] {
+			// We'll add a warning for this. Since we're right in the run block
+			// users shouldn't be defining variables that are not relevant.
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagWarning,
+				Summary:  "Value for undeclared variable",
+				Detail:   fmt.Sprintf("The module under test does not declare a variable named %q, but it is declared in run block %q.", name, run.Name),
+				Subject:  expr.Range().Ptr(),
+			})
+			continue
+		}
+
+		value := cty.NilVal
+		if !failedContext {
+			var valueDiags hcl.Diagnostics
+			value, valueDiags = expr.Value(ctx)
+			diags = diags.Append(valueDiags)
+		}
+
+		values[name] = &terraform.InputValue{
+			Value:       value,
+			SourceType:  terraform.ValueFromConfig,
+			SourceRange: tfdiags.SourceRangeFromHCL(expr.Range()),
+		}
+	}
+
+	// Third, we'll check the run level variables.
 
 	// This is a bit more complicated, as the run level variables can reference
 	// previously defined variables.
 
 	// Preload the available expressions, we're going to validate them when we
 	// build the context.
-	var exprs []hcl.Expression
+	exprs = []hcl.Expression{}
 	for _, expr := range run.Config.Variables {
 		exprs = append(exprs, expr)
 	}
 
 	// Preformat the variables we've processed already - these will be made
 	// available to the eval context.
-	variables := make(map[string]cty.Value)
+	variables = make(map[string]cty.Value)
 	for name, value := range values {
 		variables[name] = value.Value
 	}
 
-	ctx, ctxDiags := hcltest.EvalContext(hcltest.TargetRunBlock, exprs, variables, runner.PriorOutputs)
+	ctx, ctxDiags = hcltest.EvalContext(hcltest.TargetRunBlock, exprs, variables, runner.PriorOutputs)
 	diags = diags.Append(ctxDiags)
 
-	var failedContext bool
+	failedContext = false
 	if ctxDiags.HasErrors() {
 		// If we couldn't build the context, we won't actually process these
 		// variables. Instead, we'll fill them with an empty value but still
@@ -1260,8 +1333,9 @@ func (runner *TestFileRunner) initVariables(file *moduletest.File) {
 			runner.globalVariables[name] = value
 		}
 	}
+	runner.fileVariables = make(map[string]hcl.Expression)
 	for name, expr := range file.Config.Variables {
-		runner.globalVariables[name] = unparsedTestVariableValue{expr}
+		runner.fileVariables[name] = expr
 	}
 }
 

internal/command/test_test.go

@@ -26,7 +26,7 @@ func TestTest_Runs(t *testing.T) {
 		override              string
 		args                  []string
 		expectedOut           string
-		expectedErr           string
+		expectedErr           []string
 		expectedResourceCount int
 		code                  int
 		initCode              int
@@ -59,13 +59,13 @@ func TestTest_Runs(t *testing.T) {
 		"simple_pass_bad_test_directory": {
 			override:    "simple_pass",
 			args:        []string{"-test-directory", "../tests"},
-			expectedErr: "Invalid testing directory",
+			expectedErr: []string{"Invalid testing directory"},
 			code:        1,
 		},
 		"simple_pass_bad_test_directory_abs": {
 			override:    "simple_pass",
 			args:        []string{"-test-directory", "/home/username/config/tests"},
-			expectedErr: "Invalid testing directory",
+			expectedErr: []string{"Invalid testing directory"},
 			code:        1,
 		},
 		"pass_with_locals": {
@@ -118,32 +118,32 @@ func TestTest_Runs(t *testing.T) {
 			override:    "variables",
 			args:        []string{"-var=input=foo"},
 			expectedOut: "1 passed, 1 failed",
-			expectedErr: `invalid value`,
+			expectedErr: []string{`invalid value`},
 			code:        1,
 		},
 		"simple_fail": {
 			expectedOut: "0 passed, 1 failed.",
-			expectedErr: "invalid value",
+			expectedErr: []string{"invalid value"},
 			code:        1,
 		},
 		"custom_condition_checks": {
 			expectedOut: "0 passed, 1 failed.",
-			expectedErr: "this really should fail",
+			expectedErr: []string{"this really should fail"},
 			code:        1,
 		},
 		"custom_condition_inputs": {
 			expectedOut: "0 passed, 1 failed.",
-			expectedErr: "this should definitely fail",
+			expectedErr: []string{"this should definitely fail"},
 			code:        1,
 		},
 		"custom_condition_outputs": {
 			expectedOut: "0 passed, 1 failed.",
-			expectedErr: "this should fail",
+			expectedErr: []string{"this should fail"},
 			code:        1,
 		},
 		"custom_condition_resources": {
 			expectedOut: "0 passed, 1 failed.",
-			expectedErr: "this really should fail",
+			expectedErr: []string{"this really should fail"},
 			code:        1,
 		},
 		"no_providers_in_main": {
@@ -190,7 +190,7 @@ func TestTest_Runs(t *testing.T) {
 		},
 		"destroy_fail": {
 			expectedOut:           "1 passed, 0 failed.",
-			expectedErr:           `Terraform left the following resources in state`,
+			expectedErr:           []string{`Terraform left the following resources in state`},
 			code:                  1,
 			expectedResourceCount: 1,
 		},
@@ -217,7 +217,7 @@ func TestTest_Runs(t *testing.T) {
 			code:        0,
 		},
 		"mocking-invalid": {
-			expectedErr: "Invalid outputs attribute",
+			expectedErr: []string{"Invalid outputs attribute"},
 			initCode:    1,
 		},
 		"dangling_data_block": {
@@ -234,9 +234,13 @@ func TestTest_Runs(t *testing.T) {
 		},
 		"global_var_refs": {
 			expectedOut: "2 failed, 1 skipped.",
-			expectedErr: "Variables may not be used here.",
+			expectedErr: []string{"The input variable \"env_var_input\" is not available to the current context", "The input variable \"setup\" is not available to the current context"},
 			code:        1,
 		},
+		"global_var_ref_in_suite_var": {
+			expectedOut: "1 passed, 0 failed.",
+			code:        0,
+		},
 	}
 	for name, tc := range tcs {
 		t.Run(name, func(t *testing.T) {
@@ -289,9 +293,13 @@ func TestTest_Runs(t *testing.T) {
 					t.Errorf("output didn't contain expected string:\n\n%s", stdout)
 				}
 
-				if !strings.Contains(stderr, tc.expectedErr) {
-					t.Errorf("error didn't contain expected string:\n\n%s", stderr)
-				} else if tc.expectedErr == "" && stderr != "" {
+				if len(tc.expectedErr) > 0 {
+					for _, expectedErr := range tc.expectedErr {
+						if !strings.Contains(stderr, expectedErr) {
+							t.Errorf("error didn't contain expected string:\n\n%s", stderr)
+						}
+					}
+				} else if stderr != "" {
 					t.Errorf("unexpected stderr output\n%s", stderr)
 				}
 
@@ -316,9 +324,13 @@ func TestTest_Runs(t *testing.T) {
 				t.Errorf("output didn't contain expected string:\n\n%s", output.Stdout())
 			}
 
-			if !strings.Contains(output.Stderr(), tc.expectedErr) {
-				t.Errorf("error didn't contain expected string:\n\n%s", output.Stderr())
-			} else if tc.expectedErr == "" && output.Stderr() != "" {
+			if len(tc.expectedErr) > 0 {
+				for _, expectedErr := range tc.expectedErr {
+					if !strings.Contains(output.Stderr(), expectedErr) {
+						t.Errorf("error didn't contain expected string:\n\n%s", output.Stderr())
+					}
+				}
+			} else if output.Stderr() != "" {
 				t.Errorf("unexpected stderr output\n%s", output.Stderr())
 			}
 
@@ -1161,7 +1173,7 @@ requested in the configuration may have been ignored and the output values
 may not be fully updated. Run the following command to verify that no other
 changes are pending:
     terraform plan
-	
+
 Note that the -target option is not suitable for routine use, and is provided
 only for exceptional situations such as recovering from errors or mistakes,
 or when Terraform specifically suggests to use it as part of an error
@@ -1240,9 +1252,10 @@ Error: Reference to unavailable variable
   on main.tftest.hcl line 15, in run "test":
   15:     input_one = var.notreal
 
-The input variable "notreal" is not available to the current run block. You
-can only reference variables defined at the file or global levels when
-populating the variables block within a run block.
+The input variable "notreal" is not available to the current context. Within
+the variables block of a run block you can only reference variables defined
+at the file or global levels; within the variables block of a suite you can
+only reference variables defined at the global levels.
 
 Error: Reference to unavailable run block
 
@@ -1267,9 +1280,10 @@ Error: Reference to unavailable variable
   on providers.tftest.hcl line 3, in provider "test":
    3:   resource_prefix = var.default
 
-The input variable "default" is not available to the current run block. You
-can only reference variables defined at the file or global levels when
-populating the variables block within a run block.
+The input variable "default" is not available to the current context. Within
+the variables block of a run block you can only reference variables defined
+at the file or global levels; within the variables block of a suite you can
+only reference variables defined at the global levels.
 `
 	actualErr := output.Stderr()
 	if diff := cmp.Diff(actualErr, expectedErr); len(diff) > 0 {

internal/command/testdata/test/global_var_ref_in_suite_var/main.tf

@@ -0,0 +1,10 @@
+variable "input" {
+  default = null
+  type    = object({
+              organization_name = string
+            })
+}
+
+output "value" {
+  value = var.input
+}

internal/command/testdata/test/global_var_ref_in_suite_var/run_block_output.tftest.hcl

@@ -0,0 +1,13 @@
+
+variables {
+  input = {
+      organization_name = var.org_name
+  }
+}
+
+run "execute" {
+  assert {
+    condition     = output.value.organization_name == "my-org"
+    error_message = "bad output value"
+  }
+}

internal/command/testdata/test/global_var_ref_in_suite_var/terraform.tfvars

@@ -0,0 +1 @@
+org_name = "my-org"

internal/moduletest/hcl/context.go

@@ -132,7 +132,7 @@ func EvalContext(target EvalContextTarget, expressions []hcl.Expression, availab
 				if _, exists := availableVariables[addr.Name]; !exists {
 					// This variable reference doesn't exist.
 
-					detail := fmt.Sprintf("The input variable %q is not available to the current run block. You can only reference variables defined at the file or global levels when populating the variables block within a run block.", addr.Name)
+					detail := fmt.Sprintf("The input variable %q is not available to the current context. Within the variables block of a run block you can only reference variables defined at the file or global levels; within the variables block of a suite you can only reference variables defined at the global levels.", addr.Name)
 					if availableRunOutputs == nil {
 						detail = fmt.Sprintf("The input variable %q is not available to the current provider configuration. You can only reference variables defined at the file or global levels within provider configurations.", addr.Name)
 					}

internal/moduletest/hcl/provider_test.go

@@ -71,7 +71,7 @@ func TestProviderConfig(t *testing.T) {
 				"input": cty.StringVal("string"),
 			},
 			expectedErrors: []string{
-				"The input variable \"missing\" is not available to the current run block. You can only reference variables defined at the file or global levels when populating the variables block within a run block.",
+				"The input variable \"missing\" is not available to the current context. Within the variables block of a run block you can only reference variables defined at the file or global levels; within the variables block of a suite you can only reference variables defined at the global levels.",
 			},
 			validate: func(t *testing.T, content *hcl.BodyContent) {
 				if len(content.Attributes) > 0 {