Failed test cleanup usability improvements (#114)

Make cleaning up stacks easier 

- Print instructions on how to retain resources on failure.
- When retaining resources explain why this is happening and write out a
destroy script for the user to run once they've finished their
investigations.

Related to #109

Make temp directory cleanup easier to control 

- Maintain existing defaults - keeping files locally on failure by
default.
- Add explicit control via PULUMITEST_RETAIN_FILES_ON_FAILURE with
prompting to the user on failure.
- Default to retaining if we've retained the resources.
- Finally, check for the `CI` env var.

pulumitest/cleanup.go

@@ -5,13 +5,23 @@ import (
 	"strings"
 )
 
-var skipDestroyOnFailure = (func() func() bool {
+func skipDestroyOnFailure() bool {
 	value, ok := os.LookupEnv("PULUMITEST_SKIP_DESTROY_ON_FAILURE")
 	skipDestroy := ok && strings.EqualFold(value, "true")
-	return func() bool { return skipDestroy }
-})()
+	return skipDestroy
+}
 
-var runningInCI = (func() func() bool {
+func runningInCI() bool {
 	_, ok := os.LookupEnv("CI")
-	return func() bool { return ok }
-})()
+	return ok
+}
+
+func shouldRetainFilesOnFailure() bool {
+	if value, ok := os.LookupEnv("PULUMITEST_RETAIN_FILES_ON_FAILURE"); ok {
+		return !strings.EqualFold(value, "false")
+	}
+	if skipDestroyOnFailure() {
+		return true
+	}
+	return !runningInCI()
+}

pulumitest/newStack.go

@@ -175,12 +175,12 @@ func (pt *PulumiTest) NewStack(t PT, stackName string, opts ...optnewstack.NewSt
 			t.Helper()
 
 			if ptFailed(t) && skipDestroyOnFailure() {
-				t.Log(fmt.Sprintf("refusing to destroy stack at %q to help debug the failing test",
-					stack.Workspace().WorkDir()))
+				t.Log("Skipping destroy because PULUMITEST_SKIP_DESTROY_ON_FAILURE is set to 'true'.")
+				writeDestroyScript(t, stack.Workspace().WorkDir(), stackName, env)
 				return
 			}
 
-			t.Log("cleaning up stack")
+			t.Log("destroying stack, to skip this set PULUMITEST_SKIP_DESTROY_ON_FAILURE=true")
 			_, err := stack.Destroy(pt.ctx)
 			if err != nil {
 				ptErrorF(t, "failed to destroy stack: %s", err)
@@ -195,6 +195,25 @@ func (pt *PulumiTest) NewStack(t PT, stackName string, opts ...optnewstack.NewSt
 	return &stack
 }
 
+func writeDestroyScript(t PT, dir, stackName string, env map[string]string) {
+	t.Helper()
+	envPrefix := ""
+	if passphrase, ok := env["PULUMI_CONFIG_PASSPHRASE"]; ok {
+		envPrefix += fmt.Sprintf("export PULUMI_CONFIG_PASSPHRASE=%q\n", passphrase)
+	}
+	if backendUrl, ok := env["PULUMI_BACKEND_URL"]; ok {
+		envPrefix += fmt.Sprintf("export PULUMI_BACKEND_URL=%q\n", backendUrl)
+	}
+	scriptContent := fmt.Sprintf(`#!/usr/bin/env bash
+%s
+cd "$(dirname "$0")" || exit
+pulumi stack select %q
+pulumi destroy --yes`, envPrefix, stackName)
+	destroyScriptPath := filepath.Join(dir, "destroy.sh")
+	os.WriteFile(destroyScriptPath, []byte(scriptContent), 0755)
+	ptLogF(t, "Destroy can be run manually by running script at %q", destroyScriptPath)
+}
+
 func randomStackName(dir string) string {
 	// Fetch the host and test dir names, cleaned so to contain just [a-zA-Z0-9-_] chars.
 	hostname, err := os.Hostname()

pulumitest/tempdir.go

@@ -53,9 +53,14 @@ func tempDirWithoutCleanupOnFailedTest(t PT, desc string) string {
 		if c.tempDirErr == nil {
 			t.Cleanup(func() {
 				t.Helper()
-				if ptFailed(t) && !runningInCI() {
-					ptErrorF(t, "TempDir leaving %s to help debugging: %q", desc, c.tempDir)
-				} else if err := os.RemoveAll(c.tempDir); err != nil {
+				if ptFailed(t) && shouldRetainFilesOnFailure() {
+					ptLogF(t, "Skipping removal of %s temp directories on failures: %q", desc, c.tempDir)
+					t.Log("To remove these directories on failures, set PULUMITEST_RETAIN_FILES_ON_FAILURE=false")
+					return
+				}
+				err := os.RemoveAll(c.tempDir)
+				t.Log("Removed temp directories. To retain these, set PULUMITEST_RETAIN_FILES_ON_FAILURE=true")
+				if err != nil {
 					ptErrorF(t, "TempDir RemoveAll cleanup: %v", err)
 				}
 			})