Terraform Version Checking (#128)

* Add `tfversion` package and `TerraformVersionCheck` interface

* Add `TerraformVersionChecks` field and implement checks in `Test()`

* Extract `testExpectTFatal()` to internal/plugintest package

* Implement built-in version checks

* Add documentation for version checks

* fix typo

* Add Terraform Version Checks page to website navigation

* Add comment for `All()` and fix test

* Add Changie Entries

* Add copyright headers

* Add comments to tests

* Add tfversion_checks_test.go

* Rename interface request and response to avoid abbreviation

* Add version variables

* Update documentation to include and use pre-defined version variables

Co-authored-by: Brian Flad <bflad417@gmail.com>

* Add copyright header

* Properly handle error

Co-authored-by: Brian Flad <bflad417@gmail.com>

* Set golangci-lint run timeout to 5 minutes

* Fix wording in logging messages

Co-authored-by: Brian Flad <bflad417@gmail.com>

---------

Co-authored-by: Brian Flad <bflad417@gmail.com>

.changes/unreleased/FEATURES-20230602-101545.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Introduced new `tfversion` package with interface and built-in Terraform
+  version check functionality'
+time: 2023-06-02T10:15:45.704158-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124403.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `SkipAbove` built-in version check, which skips the test if
+  the Terraform CLI version is above the given maximum.'
+time: 2023-06-02T12:44:03.123635-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124437.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `SkipBelow` built-in version check, which skips the test if
+  the Terraform CLI version is below the given minimum.'
+time: 2023-06-02T12:44:37.228557-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124453.yaml

@@ -0,0 +1,7 @@
+kind: FEATURES
+body: 'tfversion: Added `SkipBetween` built-in version check, which skips the test
+  if the Terraform CLI version is between the given minimum (inclusive) and maximum
+  (exclusive).'
+time: 2023-06-02T12:44:53.737283-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124514.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `SkipIf` built-in version check, which skips the test if the
+  Terraform CLI version matches the given version.'
+time: 2023-06-02T12:45:14.812485-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124532.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `RequireAbove` built-in version check, which fails the test
+  if the Terraform CLI version is below the given maximum.'
+time: 2023-06-02T12:45:32.983833-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124653.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `RequireBelow` built-in version check, which fails the test
+  if the Terraform CLI version is above the given minimum.'
+time: 2023-06-02T12:46:53.705136-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124711.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `RequireBetween` built-in version check, fails the test if
+  the Terraform CLI version is outside the given minimum (exclusive) and maximum (inclusive).'
+time: 2023-06-02T12:47:11.762061-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124732.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `RequireNot` built-in version check, which fails the test
+  if the Terraform CLI version matches the given version.'
+time: 2023-06-02T12:47:32.000508-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124753.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `Any` built-in version check, which fails the test if none
+  of the given sub-checks return a nil error and empty skip message.'
+time: 2023-06-02T12:47:53.181503-04:00
+custom:
+  Issue: "128"

.changes/unreleased/FEATURES-20230602-124807.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'tfversion: Added `All` built-in version check, which fails or skips the test
+  if any of the given sub-checks return a non-nil error or non-empty skip message.'
+time: 2023-06-02T12:48:07.933978-04:00
+custom:
+  Issue: "128"

.golangci.yml

@@ -22,4 +22,8 @@ linters:
     - unconvert
     - unparam
     - unused
-    - vet
+    - vet
+
+run:
+  # Prevent false positive timeouts in CI
+  timeout: 5m

helper/resource/testcase_providers_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+
+	"github.com/hashicorp/terraform-plugin-testing/internal/plugintest"
 )
 
 func TestTestCaseProviderConfig(t *testing.T) {
@@ -332,7 +334,7 @@ func TestTest_TestCase_ExternalProvidersAndProviderFactories_NonHashiCorpNamespa
 func TestTest_TestCase_ExternalProviders_Error(t *testing.T) {
 	t.Parallel()
 
-	testExpectTFatal(t, func() {
+	plugintest.TestExpectTFatal(t, func() {
 		Test(&mockT{}, TestCase{
 			ExternalProviders: map[string]ExternalProvider{
 				"testnonexistent": {
@@ -368,7 +370,7 @@ func TestTest_TestCase_ProtoV5ProviderFactories(t *testing.T) {
 func TestTest_TestCase_ProtoV5ProviderFactories_Error(t *testing.T) {
 	t.Parallel()
 
-	testExpectTFatal(t, func() {
+	plugintest.TestExpectTFatal(t, func() {
 		Test(&mockT{}, TestCase{
 			ProtoV5ProviderFactories: map[string]func() (tfprotov5.ProviderServer, error){
 				"test": func() (tfprotov5.ProviderServer, error) { //nolint:unparam // required signature
@@ -404,7 +406,7 @@ func TestTest_TestCase_ProtoV6ProviderFactories(t *testing.T) {
 func TestTest_TestCase_ProtoV6ProviderFactories_Error(t *testing.T) {
 	t.Parallel()
 
-	testExpectTFatal(t, func() {
+	plugintest.TestExpectTFatal(t, func() {
 		Test(&mockT{}, TestCase{
 			ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
 				"test": func() (tfprotov6.ProviderServer, error) { //nolint:unparam // required signature
@@ -440,7 +442,7 @@ func TestTest_TestCase_ProviderFactories(t *testing.T) {
 func TestTest_TestCase_ProviderFactories_Error(t *testing.T) {
 	t.Parallel()
 
-	testExpectTFatal(t, func() {
+	plugintest.TestExpectTFatal(t, func() {
 		Test(&mockT{}, TestCase{
 			ProviderFactories: map[string]func() (*schema.Provider, error){
 				"test": func() (*schema.Provider, error) { //nolint:unparam // required signature

helper/resource/testing.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/hashicorp/terraform-plugin-testing/plancheck"
 	"github.com/hashicorp/terraform-plugin-testing/terraform"
+	"github.com/hashicorp/terraform-plugin-testing/tfversion"
 
 	"github.com/hashicorp/terraform-plugin-testing/internal/addrs"
 	"github.com/hashicorp/terraform-plugin-testing/internal/logging"
@@ -323,6 +324,12 @@ type TestCase struct {
 	// acceptance tests, such as verifying that keys are setup.
 	PreCheck func()
 
+	// TerraformVersionChecks is a list of checks to run against
+	// the Terraform CLI version which is running the testing.
+	// Each check is executed in order, respecting the first skip
+	// or fail response, unless the Any() meta check is also used.
+	TerraformVersionChecks []tfversion.TerraformVersionCheck
+
 	// ProviderFactories can be specified for the providers that are valid.
 	//
 	// This can also be specified at the TestStep level to enable per-step
@@ -824,6 +831,17 @@ func Test(t testing.T, c TestCase) {
 		}
 	}(helper)
 
+	// Run the TerraformVersionChecks if we have it.
+	// This is done after creating the helper because a working directory is required
+	// to retrieve the Terraform version.
+	if c.TerraformVersionChecks != nil {
+		logging.HelperResourceDebug(ctx, "Calling TestCase Terraform version checks")
+
+		runTFVersionChecks(ctx, t, helper.TerraformVersion(), c.TerraformVersionChecks)
+
+		logging.HelperResourceDebug(ctx, "Called TestCase Terraform version checks")
+	}
+
 	runNewTest(ctx, t, c, helper)
 
 	logging.HelperResourceDebug(ctx, "Finished TestCase")

helper/resource/testing_test.go

@@ -27,45 +27,6 @@ func init() {
 	}
 }
 
-// testExpectTFatal provides a wrapper for logic which should call
-// (*testing.T).Fatal() or (*testing.T).Fatalf().
-//
-// Since we do not want the wrapping test to fail when an expected test error
-// occurs, it is required that the testLogic passed in uses
-// github.com/mitchellh/go-testing-interface.RuntimeT instead of the real
-// *testing.T.
-//
-// If Fatal() or Fatalf() is not called in the logic, the real (*testing.T).Fatal() will
-// be called to fail the test.
-func testExpectTFatal(t *testing.T, testLogic func()) {
-	t.Helper()
-
-	var recoverIface interface{}
-
-	func() {
-		defer func() {
-			recoverIface = recover()
-		}()
-
-		testLogic()
-	}()
-
-	if recoverIface == nil {
-		t.Fatalf("expected t.Fatal(), got none")
-	}
-
-	recoverStr, ok := recoverIface.(string)
-
-	if !ok {
-		t.Fatalf("expected string from recover(), got: %v (%T)", recoverIface, recoverIface)
-	}
-
-	// this string is hardcoded in github.com/mitchellh/go-testing-interface
-	if !strings.HasPrefix(recoverStr, "testing.T failed, see logs for output") {
-		t.Fatalf("expected t.Fatal(), got: %s", recoverStr)
-	}
-}
-
 func TestParallelTest(t *testing.T) {
 	t.Parallel()
 

helper/resource/teststep_providers_test.go

@@ -1053,7 +1053,7 @@ func TestTest_TestStep_ExternalProviders_DifferentVersions(t *testing.T) {
 func TestTest_TestStep_ExternalProviders_Error(t *testing.T) {
 	t.Parallel()
 
-	testExpectTFatal(t, func() {
+	plugintest.TestExpectTFatal(t, func() {
 		Test(&mockT{}, TestCase{
 			Steps: []TestStep{
 				{
@@ -1354,7 +1354,7 @@ func TestTest_TestStep_ProtoV5ProviderFactories(t *testing.T) {
 func TestTest_TestStep_ProtoV5ProviderFactories_Error(t *testing.T) {
 	t.Parallel()
 
-	testExpectTFatal(t, func() {
+	plugintest.TestExpectTFatal(t, func() {
 		Test(&mockT{}, TestCase{
 			Steps: []TestStep{
 				{
@@ -1390,7 +1390,7 @@ func TestTest_TestStep_ProtoV6ProviderFactories(t *testing.T) {
 func TestTest_TestStep_ProtoV6ProviderFactories_Error(t *testing.T) {
 	t.Parallel()
 
-	testExpectTFatal(t, func() {
+	plugintest.TestExpectTFatal(t, func() {
 		Test(&mockT{}, TestCase{
 			Steps: []TestStep{
 				{
@@ -1426,7 +1426,7 @@ func TestTest_TestStep_ProviderFactories(t *testing.T) {
 func TestTest_TestStep_ProviderFactories_Error(t *testing.T) {
 	t.Parallel()
 
-	testExpectTFatal(t, func() {
+	plugintest.TestExpectTFatal(t, func() {
 		Test(&mockT{}, TestCase{
 			Steps: []TestStep{
 				{

helper/resource/tfversion_checks.go

@@ -0,0 +1,31 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package resource
+
+import (
+	"context"
+
+	"github.com/hashicorp/go-version"
+	"github.com/mitchellh/go-testing-interface"
+
+	"github.com/hashicorp/terraform-plugin-testing/tfversion"
+)
+
+func runTFVersionChecks(ctx context.Context, t testing.T, terraformVersion *version.Version, terraformVersionChecks []tfversion.TerraformVersionCheck) {
+	t.Helper()
+
+	for _, tfVersionCheck := range terraformVersionChecks {
+		resp := tfversion.CheckTerraformVersionResponse{}
+		tfVersionCheck.CheckTerraformVersion(ctx, tfversion.CheckTerraformVersionRequest{TerraformVersion: terraformVersion}, &resp)
+
+		if resp.Error != nil {
+			t.Fatalf(resp.Error.Error())
+		}
+
+		if resp.Skip != "" {
+			t.Skip(resp.Skip)
+		}
+	}
+
+}

helper/resource/tfversion_checks_test.go

@@ -0,0 +1,65 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package resource
+
+import (
+	"context"
+	"testing"
+
+	"github.com/hashicorp/go-version"
+
+	"github.com/hashicorp/terraform-plugin-testing/internal/plugintest"
+	"github.com/hashicorp/terraform-plugin-testing/tfversion"
+
+	testinginterface "github.com/mitchellh/go-testing-interface"
+)
+
+func TestRunTFVersionChecks(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]struct {
+		versionChecks []tfversion.TerraformVersionCheck
+		tfVersion     *version.Version
+		expectError   bool
+	}{
+		"run-test": {
+			versionChecks: []tfversion.TerraformVersionCheck{
+				tfversion.SkipIf(version.Must(version.NewVersion("1.1.0"))),
+				tfversion.RequireAbove(version.Must(version.NewVersion("1.2.0"))),
+			},
+			tfVersion:   version.Must(version.NewVersion("1.3.0")),
+			expectError: false,
+		},
+		"skip-test": {
+			versionChecks: []tfversion.TerraformVersionCheck{
+				tfversion.SkipIf(version.Must(version.NewVersion("1.1.0"))),
+			},
+			tfVersion:   version.Must(version.NewVersion("1.1.0")),
+			expectError: false,
+		},
+		"fail-test": {
+			versionChecks: []tfversion.TerraformVersionCheck{
+				tfversion.RequireNot(version.Must(version.NewVersion("1.1.0"))),
+			},
+			tfVersion:   version.Must(version.NewVersion("1.1.0")),
+			expectError: true,
+		},
+	}
+
+	for name, test := range tests {
+		name, test := name, test
+
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			if test.expectError {
+				plugintest.TestExpectTFatal(t, func() {
+					runTFVersionChecks(context.Background(), &testinginterface.RuntimeT{}, test.tfVersion, test.versionChecks)
+				})
+			} else {
+				runTFVersionChecks(context.Background(), t, test.tfVersion, test.versionChecks)
+			}
+		})
+	}
+}