Walk hierarchy to add root modules (#176)

* Walk hierarchy to add root modules

Because detection of the relevant root module for a file/dir
is a non trivial task, we also introduce candidate finder
which can report all potential candidates for root modules
in the context of a directory being initialized/opened.

* Add testdata

* rootmodule: Store data in slice for consistent results

This turns lookups from O(1) to O(n), but the n is in most cases so low
that it likely doesn't matter anyway.

* Shorten warning for multiple candidates

* Update internal/terraform/schema/schema_storage.go

Co-authored-by: Paul Tyng <ptyng@hashicorp.com>

* rootmodule: embed interfaces

Co-authored-by: Paul Tyng <ptyng@hashicorp.com>

internal/context/context.go

@@ -29,6 +29,8 @@ var (
 	ctxRootModuleMngr   = &contextKey{"root module manager"}
 	ctxParserFinder     = &contextKey{"parser finder"}
 	ctxTfExecFinder     = &contextKey{"terraform exec finder"}
+	ctxRootModuleCaFi   = &contextKey{"root module candidate finder"}
+	ctxRootDir          = &contextKey{"root directory"}
 )
 
 func missingContextErr(ctxKey *contextKey) *MissingContextErr {
@@ -149,3 +151,37 @@ func TerraformExecPath(ctx context.Context) (string, bool) {
 	path, ok := ctx.Value(ctxTfExecPath).(string)
 	return path, ok
 }
+
+func WithRootModuleCandidateFinder(rmcf rootmodule.RootModuleCandidateFinder, ctx context.Context) context.Context {
+	return context.WithValue(ctx, ctxRootModuleCaFi, rmcf)
+}
+
+func RootModuleCandidateFinder(ctx context.Context) (rootmodule.RootModuleCandidateFinder, error) {
+	cf, ok := ctx.Value(ctxRootModuleCaFi).(rootmodule.RootModuleCandidateFinder)
+	if !ok {
+		return nil, missingContextErr(ctxRootModuleCaFi)
+	}
+	return cf, nil
+}
+
+func WithRootDirectory(dir *string, ctx context.Context) context.Context {
+	return context.WithValue(ctx, ctxRootDir, dir)
+}
+
+func SetRootDirectory(ctx context.Context, dir string) error {
+	rootDir, ok := ctx.Value(ctxRootDir).(*string)
+	if !ok {
+		return missingContextErr(ctxRootDir)
+	}
+
+	*rootDir = dir
+	return nil
+}
+
+func RootDirectory(ctx context.Context) (string, bool) {
+	rootDir, ok := ctx.Value(ctxRootDir).(*string)
+	if !ok {
+		return "", false
+	}
+	return *rootDir, true
+}

internal/terraform/rootmodule/root_module.go

@@ -18,6 +18,7 @@ import (
 
 type rootModule struct {
 	ctx                context.Context
+	path               string
 	logger             *log.Logger
 	pluginLockFile     File
 	moduleManifestFile File
@@ -39,9 +40,10 @@ type rootModule struct {
 	moduleMu     *sync.RWMutex
 }
 
-func newRootModule(ctx context.Context) *rootModule {
+func newRootModule(ctx context.Context, dir string) *rootModule {
 	return &rootModule{
 		ctx:      ctx,
+		path:     dir,
 		logger:   defaultLogger,
 		pluginMu: &sync.RWMutex{},
 		moduleMu: &sync.RWMutex{},
@@ -51,7 +53,7 @@ func newRootModule(ctx context.Context) *rootModule {
 var defaultLogger = log.New(ioutil.Discard, "", 0)
 
 func NewRootModule(ctx context.Context, dir string) (RootModule, error) {
-	rm := newRootModule(ctx)
+	rm := newRootModule(ctx, dir)
 
 	d := &discovery.Discovery{}
 	rm.tfDiscoFunc = d.LookPath
@@ -63,15 +65,16 @@ func NewRootModule(ctx context.Context, dir string) (RootModule, error) {
 		return ss
 	}
 
-	return rm, rm.init(ctx, dir)
+	return rm, rm.init(ctx)
 }
 
 func (rm *rootModule) SetLogger(logger *log.Logger) {
 	rm.logger = logger
 }
 
-func (rm *rootModule) init(ctx context.Context, dir string) error {
-	tf, err := rm.initTfExecutor(dir)
+func (rm *rootModule) init(ctx context.Context) error {
+	rm.logger.Printf("initing new root module: %s", rm.path)
+	tf, err := rm.initTfExecutor(rm.path)
 	if err != nil {
 		return err
 	}
@@ -105,11 +108,11 @@ func (rm *rootModule) init(ctx context.Context, dir string) error {
 	rm.tfExec = tf
 	rm.tfVersion = version
 
-	err = rm.initPluginCache(dir)
+	err = rm.initPluginCache(rm.path)
 	if err != nil {
 		return fmt.Errorf("plugin initialization failed: %w", err)
 	}
-	err = rm.initModuleCache(dir)
+	err = rm.initModuleCache(rm.path)
 	if err != nil {
 		return err
 	}
@@ -216,6 +219,10 @@ func (rm *rootModule) initModuleCache(dir string) error {
 	return rm.UpdateModuleManifest(lf)
 }
 
+func (rm *rootModule) Path() string {
+	return rm.path
+}
+
 func (rm *rootModule) UpdateModuleManifest(lockFile File) error {
 	rm.moduleMu.Lock()
 	rm.logger.Printf("updating module manifest based on %s ...", lockFile.Path())

internal/terraform/rootmodule/root_module_manager.go

@@ -16,7 +16,7 @@ import (
 )
 
 type rootModuleManager struct {
-	rms           map[string]*rootModule
+	rms           []*rootModule
 	tfExecPath    string
 	tfExecTimeout time.Duration
 	tfExecLogPath string
@@ -31,15 +31,15 @@ func NewRootModuleManager(ctx context.Context) RootModuleManager {
 
 func newRootModuleManager(ctx context.Context) *rootModuleManager {
 	rmm := &rootModuleManager{
-		rms:    make(map[string]*rootModule, 0),
+		rms:    make([]*rootModule, 0),
 		logger: defaultLogger,
 	}
 	rmm.newRootModule = rmm.defaultRootModuleFactory
 	return rmm
 }
 
 func (rmm *rootModuleManager) defaultRootModuleFactory(ctx context.Context, dir string) (*rootModule, error) {
-	rm := newRootModule(ctx)
+	rm := newRootModule(ctx, dir)
 
 	rm.SetLogger(rmm.logger)
 
@@ -52,7 +52,7 @@ func (rmm *rootModuleManager) defaultRootModuleFactory(ctx context.Context, dir
 	rm.tfExecTimeout = rmm.tfExecTimeout
 	rm.tfExecLogPath = rmm.tfExecLogPath
 
-	return rm, rm.init(ctx, dir)
+	return rm, rm.init(ctx)
 }
 
 func (rmm *rootModuleManager) SetTerraformExecPath(path string) {
@@ -76,8 +76,7 @@ func (rmm *rootModuleManager) AddRootModule(dir string) error {
 
 	// TODO: Follow symlinks (requires proper test data)
 
-	_, exists := rmm.rms[dir]
-	if exists {
+	if rmm.exists(dir) {
 		return fmt.Errorf("root module %s was already added", dir)
 	}
 
@@ -86,33 +85,66 @@ func (rmm *rootModuleManager) AddRootModule(dir string) error {
 		return err
 	}
 
-	rmm.rms[dir] = rm
+	rmm.rms = append(rmm.rms, rm)
 
 	return nil
 }
 
-func (rmm *rootModuleManager) RootModuleByPath(path string) (RootModule, error) {
+func (rmm *rootModuleManager) exists(dir string) bool {
+	for _, rm := range rmm.rms {
+		if rm.Path() == dir {
+			return true
+		}
+	}
+	return false
+}
+
+func (rmm *rootModuleManager) rootModuleByPath(dir string) *rootModule {
+	for _, rm := range rmm.rms {
+		if rm.Path() == dir {
+			return rm
+		}
+	}
+	return nil
+}
+
+func (rmm *rootModuleManager) RootModuleCandidatesByPath(path string) []string {
 	path = filepath.Clean(path)
 
 	// TODO: Follow symlinks (requires proper test data)
 
-	if rm, ok := rmm.rms[path]; ok {
+	if rmm.exists(path) {
 		rmm.logger.Printf("direct root module lookup succeeded: %s", path)
-		return rm, nil
+		return []string{path}
 	}
 
 	dir := rootModuleDirFromFilePath(path)
-	if rm, ok := rmm.rms[dir]; ok {
+	if rmm.exists(dir) {
 		rmm.logger.Printf("dir-based root module lookup succeeded: %s", dir)
-		return rm, nil
+		return []string{dir}
 	}
 
+	candidates := make([]string, 0)
 	for _, rm := range rmm.rms {
-		rmm.logger.Printf("looking up %s in module references", dir)
+		rmm.logger.Printf("looking up %s in module references of %s", dir, rm.Path())
 		if rm.ReferencesModulePath(dir) {
 			rmm.logger.Printf("module-ref-based root module lookup succeeded: %s", dir)
-			return rm, nil
+			candidates = append(candidates, rm.Path())
+		}
+	}
+
+	return candidates
+}
+
+func (rmm *rootModuleManager) RootModuleByPath(path string) (RootModule, error) {
+	candidates := rmm.RootModuleCandidatesByPath(path)
+	if len(candidates) > 0 {
+		firstMatch := candidates[0]
+		if !rmm.exists(firstMatch) {
+			return nil, fmt.Errorf("Discovered root module %s not available,"+
+				" this is most likely a bug, please report it", firstMatch)
 		}
+		return rmm.rootModuleByPath(firstMatch), nil
 	}
 
 	return nil, &RootModuleNotFoundErr{path}

internal/terraform/rootmodule/root_module_manager_mock.go

@@ -22,29 +22,38 @@ type RootModuleMockFactory struct {
 }
 
 func (rmf *RootModuleMockFactory) New(ctx context.Context, dir string) (*rootModule, error) {
-	rm, ok := rmf.rmm[dir]
+	rmm, ok := rmf.rmm[dir]
 	if !ok {
 		return nil, fmt.Errorf("unexpected root module requested: %s (%d available: %#v)", dir, len(rmf.rmm), rmf.rmm)
 	}
 
-	w := newRootModule(ctx)
-	w.SetLogger(rmf.logger)
+	mock := NewRootModuleMock(ctx, rmm, dir)
+	mock.SetLogger(rmf.logger)
+	return mock, mock.init(ctx)
+}
+
+func NewRootModuleMock(ctx context.Context, rmm *RootModuleMock, dir string) *rootModule {
+	rm := newRootModule(ctx, dir)
 
 	md := &discovery.MockDiscovery{Path: "tf-mock"}
-	w.tfDiscoFunc = md.LookPath
+	rm.tfDiscoFunc = md.LookPath
 
 	// For now, until we have better testing strategy to mimic real lock files
-	w.ignorePluginCache = true
+	rm.ignorePluginCache = true
 
-	w.tfNewExecutor = exec.MockExecutor(rm.TerraformExecQueue)
+	rm.tfNewExecutor = exec.MockExecutor(rmm.TerraformExecQueue)
 
-	if rm.ProviderSchemas == nil {
-		w.newSchemaStorage = schema.MockStorage(rm.ProviderSchemas)
+	if rmm.ProviderSchemas == nil {
+		rm.newSchemaStorage = func() *schema.Storage {
+			ss := schema.NewStorage()
+			ss.SetSynchronous()
+			return ss
+		}
 	} else {
-		w.newSchemaStorage = schema.NewStorage
+		rm.newSchemaStorage = schema.MockStorage(rmm.ProviderSchemas)
 	}
 
-	return w, w.init(ctx, dir)
+	return rm
 }
 
 func NewRootModuleManagerMock(m map[string]*RootModuleMock) RootModuleManagerFactory {