Recursive inspection in parallel (#2021)

terraform-linters · Apr 30, 2024 · 0a9117e · 0a9117e
1 parent 0c0f631
commit 0a9117e
Show file tree

Hide file tree

Showing 45 changed files with 1,313 additions and 179 deletions.
diff --git a/README.md b/README.md
@@ -135,6 +135,7 @@ Application Options:
       --no-color                                                Disable colorized output
       --fix                                                     Fix issues automatically
       --no-parallel-runners                                     Disable per-runner parallelism
+      --max-workers=N                                           Set maximum number of workers in recursive inspection (default: number of CPUs)
 
 Help Options:
   -h, --help                                                    Show this help message

diff --git a/cmd/cli.go b/cmd/cli.go
@@ -6,8 +6,10 @@ import (
 	"io"
 	"log"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/fatih/color"
 	"github.com/hashicorp/logutils"
@@ -63,6 +65,7 @@ func (cli *CLI) Run(args []string) int {
 		Stdout: cli.outStream,
 		Stderr: cli.errStream,
 		Format: opts.Format,
+		Fix:    opts.Fix,
 	}
 	if opts.Color {
 		color.NoColor = false
@@ -92,6 +95,10 @@ func (cli *CLI) Run(args []string) int {
 		cli.formatter.Print(tflint.Issues{}, fmt.Errorf("Command line arguments support was dropped in v0.47. Use --chdir or --filter instead."), map[string][]byte{})
 		return ExitCodeError
 	}
+	if opts.MaxWorkers != nil && *opts.MaxWorkers <= 0 {
+		cli.formatter.Print(tflint.Issues{}, fmt.Errorf("Max workers should be greater than 0"), map[string][]byte{})
+		return ExitCodeError
+	}
 
 	switch {
 	case opts.Version:
@@ -103,7 +110,11 @@ func (cli *CLI) Run(args []string) int {
 	case opts.ActAsBundledPlugin:
 		return cli.actAsBundledPlugin()
 	default:
-		return cli.inspect(opts)
+		if opts.Recursive {
+			return cli.inspectParallel(opts)
+		} else {
+			return cli.inspect(opts)
+		}
 	}
 }
 
@@ -171,7 +182,7 @@ func findWorkingDirs(opts Options) ([]string, error) {
 }
 
 func (cli *CLI) withinChangedDir(dir string, proc func() error) (err error) {
-	if dir != "." {
+	if dir != "." && dir != "" {
 		chErr := os.Chdir(dir)
 		if chErr != nil {
 			return fmt.Errorf("Failed to switch to a different working directory; %w", chErr)
@@ -186,3 +197,16 @@ func (cli *CLI) withinChangedDir(dir string, proc func() error) (err error) {
 
 	return proc()
 }
+
+func registerShutdownCh() <-chan os.Signal {
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
+	return ch
+}
+
+func (cli *CLI) registerShutdownHandler(callback func()) {
+	ch := registerShutdownCh()
+	sig := <-ch
+	fmt.Fprintf(cli.errStream, "Received %s, shutting down...\n", sig)
+	callback()
+}
diff --git a/cmd/inspect.go b/cmd/inspect.go
@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -19,86 +20,62 @@ import (
 )
 
 func (cli *CLI) inspect(opts Options) int {
-	// Respect the "--format" flag until a config is loaded
-	cli.formatter.Format = opts.Format
-
-	workingDirs, err := findWorkingDirs(opts)
-	if err != nil {
-		cli.formatter.Print(tflint.Issues{}, fmt.Errorf("Failed to find workspaces; %w", err), map[string][]byte{})
-		return ExitCodeError
-	}
-
 	issues := tflint.Issues{}
 	changes := map[string][]byte{}
 
-	for _, wd := range workingDirs {
-		err := cli.withinChangedDir(wd, func() error {
-			filterFiles := []string{}
-			for _, pattern := range opts.Filter {
-				files, err := filepath.Glob(pattern)
-				if err != nil {
-					return fmt.Errorf("Failed to parse --filter options; %w", err)
-				}
-				// Add the raw pattern to return an empty result if it doesn't match any files
-				if len(files) == 0 {
-					filterFiles = append(filterFiles, pattern)
-				}
-				filterFiles = append(filterFiles, files...)
-			}
-
-			// Join with the working directory to create the fullpath
-			for i, file := range filterFiles {
-				filterFiles[i] = filepath.Join(wd, file)
-			}
-
-			moduleIssues, moduleChanges, err := cli.inspectModule(opts, ".", filterFiles)
+	err := cli.withinChangedDir(opts.Chdir, func() error {
+		filterFiles := []string{}
+		for _, pattern := range opts.Filter {
+			files, err := filepath.Glob(pattern)
 			if err != nil {
-				if len(workingDirs) > 1 {
-					// Print the current working directory in recursive inspection
-					return fmt.Errorf("%w working_dir=%s", err, wd)
-				}
-				return err
+				return fmt.Errorf("Failed to parse --filter options; %w", err)
 			}
-			issues = append(issues, moduleIssues...)
-			for path, source := range moduleChanges {
-				changes[path] = source
+			// Add the raw pattern to return an empty result if it doesn't match any files
+			if len(files) == 0 {
+				filterFiles = append(filterFiles, pattern)
 			}
+			filterFiles = append(filterFiles, files...)
+		}
 
-			return nil
-		})
-		if err != nil {
-			sources := map[string][]byte{}
-			if cli.loader != nil {
-				sources = cli.loader.Sources()
-			}
-			cli.formatter.Print(tflint.Issues{}, err, sources)
-			return ExitCodeError
+		// Join with the working directory to create the fullpath
+		for i, file := range filterFiles {
+			filterFiles[i] = filepath.Join(opts.Chdir, file)
 		}
+
+		var err error
+		issues, changes, err = cli.inspectModule(opts, ".", filterFiles)
+		return err
+	})
+	if err != nil {
+		sources := map[string][]byte{}
+		if cli.loader != nil {
+			sources = cli.loader.Sources()
+		}
+		cli.formatter.Print(tflint.Issues{}, err, sources)
+		return ExitCodeError
 	}
 
-	var force bool
-	if opts.Recursive {
-		// Respect "--format" and "--force" flags in recursive mode
-		cli.formatter.Format = opts.Format
-		if opts.Force != nil {
-			force = *opts.Force
+	if opts.ActAsWorker {
+		// When acting as a recursive inspection worker, the formatter is ignored
+		// and the serialized issues are output.
+		out, err := json.Marshal(issues)
+		if err != nil {
+			fmt.Fprint(cli.errStream, err)
+			return ExitCodeError
 		}
+		fmt.Fprint(cli.outStream, string(out))
 	} else {
-		cli.formatter.Format = cli.config.Format
-		force = cli.config.Force
+		cli.formatter.Print(issues, nil, cli.sources)
 	}
 
-	cli.formatter.Fix = opts.Fix
-	cli.formatter.Print(issues, nil, cli.sources)
-
 	if opts.Fix {
 		if err := writeChanges(changes); err != nil {
 			cli.formatter.Print(tflint.Issues{}, err, cli.sources)
 			return ExitCodeError
 		}
 	}
 
-	if len(issues) > 0 && !force && exceedsMinimumFailure(issues, opts.MinimumFailureSeverity) {
+	if len(issues) > 0 && !cli.config.Force && exceedsMinimumFailure(issues, opts.MinimumFailureSeverity) {
 		return ExitCodeIssuesFound
 	}
 
@@ -122,8 +99,8 @@ func (cli *CLI) inspectModule(opts Options, dir string, filterFiles []string) (t
 	if err != nil {
 		return issues, changes, fmt.Errorf("Failed to prepare loading; %w", err)
 	}
-	if opts.Recursive && !cli.loader.IsConfigDir(dir) {
-		// Ignore non-module directories in recursive mode
+	if opts.ActAsWorker && !cli.loader.IsConfigDir(dir) {
+		// Ignore non-module directories in worker mode
 		return issues, changes, nil
 	}
 
@@ -137,6 +114,10 @@ func (cli *CLI) inspectModule(opts Options, dir string, filterFiles []string) (t
 	rulesetPlugin, err := launchPlugins(cli.config, opts.Fix)
 	if rulesetPlugin != nil {
 		defer rulesetPlugin.Clean()
+		go cli.registerShutdownHandler(func() {
+			rulesetPlugin.Clean()
+			os.Exit(ExitCodeError)
+		})
 	}
 	if err != nil {
 		return issues, changes, err