Introduce global provider schema cache (#454)

* Introduce state package (memdb)

* Decouple path comparer

* datadir: Add ContainsLocalModule method

* handler/initialize: Remove unused dependency

* Use memdb-driven state for modules and schema

* Add/update dependencies

* bump terraform-schema to 62efc0485fa8

go.mod

@@ -7,16 +7,19 @@ require (
 	github.com/creachadair/jrpc2 v0.13.0
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/google/go-cmp v0.5.5
-	github.com/google/uuid v1.2.0
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/hashicorp/go-memdb v1.3.2
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.3.0
-	github.com/hashicorp/hcl-lang v0.0.0-20210317074414-9bb6847208c3
+	github.com/hashicorp/hcl-lang v0.0.0-20210419185146-8556dd730bc7
 	github.com/hashicorp/hcl/v2 v2.9.1
 	github.com/hashicorp/terraform-exec v0.13.2
 	github.com/hashicorp/terraform-json v0.10.0
-	github.com/hashicorp/terraform-schema v0.0.0-20210401080230-18d878a488bf
+	github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896
+	github.com/hashicorp/terraform-schema v0.0.0-20210419192955-62efc0485fa8
 	github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
 	github.com/mitchellh/cli v1.1.2
+	github.com/mitchellh/copystructure v1.1.2
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/pmezard/go-difflib v1.0.0
@@ -26,4 +29,6 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/vektra/mockery/v2 v2.7.4
 	github.com/zclconf/go-cty v1.8.1
+	github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b
+	golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e
 )

go.sum

@@ -160,6 +160,10 @@ github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/S
 github.com/hashicorp/go-getter v1.5.3 h1:NF5+zOlQegim+w/EUhSLh6QhXHmZMEeHLQzllkQ3ROU=
 github.com/hashicorp/go-getter v1.5.3/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE=
+github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-memdb v1.3.2 h1:RBKHOsnSszpU6vxq80LzC2BaQjuuvoyaQbkLTf7V7g8=
+github.com/hashicorp/go-memdb v1.3.2/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
@@ -178,13 +182,14 @@ github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/hcl-lang v0.0.0-20210317074414-9bb6847208c3 h1:s33G8woMGjgqckJnOXbDqRi42kNfPs0+I5TmqRU5Xts=
-github.com/hashicorp/hcl-lang v0.0.0-20210317074414-9bb6847208c3/go.mod h1:ZGuDQ7IWG2eFZZJC4pBZFnbH8afJ8gyOPmlHt5lJZK0=
+github.com/hashicorp/hcl-lang v0.0.0-20210419185146-8556dd730bc7 h1:6ajS9kFs/7UnRcy3dHThUnmicqU7VjGUn0RLOT6Ewq0=
+github.com/hashicorp/hcl-lang v0.0.0-20210419185146-8556dd730bc7/go.mod h1:VRVfqufUmJSpWsoWDtYV/BejqCV+NNyS9V9vR0dcivs=
 github.com/hashicorp/hcl/v2 v2.0.0/go.mod h1:oVVDG71tEinNGYCxinCYadcmKU9bglqW9pV3txagJ90=
 github.com/hashicorp/hcl/v2 v2.9.1 h1:eOy4gREY0/ZQHNItlfuEZqtcQbXIxzojlP301hDpnac=
 github.com/hashicorp/hcl/v2 v2.9.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg=
@@ -198,8 +203,10 @@ github.com/hashicorp/terraform-exec v0.13.2 h1:r6NDbud2ylSL3/KwX5WFlCIGW6jonTNtF
 github.com/hashicorp/terraform-exec v0.13.2/go.mod h1:kTN5U5yAvqjJiwYyQQXYRevNjVWLzU9TYbW/VxRobkc=
 github.com/hashicorp/terraform-json v0.10.0 h1:9syPD/Y5t+3uFjG8AiWVPu1bklJD8QB8iTCaJASc8oQ=
 github.com/hashicorp/terraform-json v0.10.0/go.mod h1:3defM4kkMfttwiE7VakJDwCd4R+umhSQnvJwORXbprE=
-github.com/hashicorp/terraform-schema v0.0.0-20210401080230-18d878a488bf h1:C4OCWg5S/z6hFPlOr8SUz8vuEQ96GI9WX7y/8XFfcGQ=
-github.com/hashicorp/terraform-schema v0.0.0-20210401080230-18d878a488bf/go.mod h1:fOIXiNW/fHmKEt4EuEyRW6EWo0iz1yvDjKvA3sE9phY=
+github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896 h1:1FGtlkJw87UsTMg5s8jrekrHmUPUJaMcu6ELiVhQrNw=
+github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896/go.mod h1:bzBPnUIkI0RxauU8Dqo+2KrZZ28Cf48s8V6IHt3p4co=
+github.com/hashicorp/terraform-schema v0.0.0-20210419192955-62efc0485fa8 h1:vsnfMTj2BzEELMkkwzIdGRfXeYdzRrNn60wI+fQEwPc=
+github.com/hashicorp/terraform-schema v0.0.0-20210419192955-62efc0485fa8/go.mod h1:FOaoRtKTpQJ5WnCqRqQ9iJbsG8QFD94nlwUqisC295Q=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg=
 github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
@@ -257,8 +264,8 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT
 github.com/mitchellh/cli v1.1.2 h1:PvH+lL2B7IQ101xQL63Of8yFS2y+aDlsFcsqNc+u/Kw=
 github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
-github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4=
-github.com/mitchellh/copystructure v1.1.1/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4=
+github.com/mitchellh/copystructure v1.1.2 h1:Th2TIvG1+6ma3e/0/bopBKohOTY7s4dA8V2q4EUcBJ0=
+github.com/mitchellh/copystructure v1.1.2/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -436,6 +443,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897 h1:KrsHThm5nFk34YtATK1LsThyGhGbGe1olrte/HInHvs=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=

internal/cmd/completion_command.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/terraform-ls/internal/logging"
 	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
 	lsp "github.com/hashicorp/terraform-ls/internal/protocol"
+	"github.com/hashicorp/terraform-ls/internal/state"
 	"github.com/hashicorp/terraform-ls/internal/terraform/module"
 	"github.com/mitchellh/cli"
 )
@@ -104,7 +105,12 @@ func (c *CompletionCommand) Run(args []string) int {
 	}
 
 	ctx := context.Background()
-	modMgr := module.NewSyncModuleManager(ctx, fs)
+	ss, err := state.NewStateStore()
+	if err != nil {
+		c.Ui.Error(err.Error())
+		return 1
+	}
+	modMgr := module.NewSyncModuleManager(ctx, fs, ss.Modules, ss.ProviderSchemas)
 
 	mod, err := modMgr.AddModule(fh.Dir())
 	if err != nil {

internal/cmd/inspect_module_command.go

@@ -16,6 +16,7 @@ import (
 	ictx "github.com/hashicorp/terraform-ls/internal/context"
 	"github.com/hashicorp/terraform-ls/internal/filesystem"
 	"github.com/hashicorp/terraform-ls/internal/logging"
+	"github.com/hashicorp/terraform-ls/internal/state"
 	"github.com/hashicorp/terraform-ls/internal/terraform/datadir"
 	"github.com/hashicorp/terraform-ls/internal/terraform/module"
 	"github.com/mitchellh/cli"
@@ -86,7 +87,11 @@ func (c *InspectModuleCommand) inspect(rootPath string) error {
 	fs := filesystem.NewFilesystem()
 
 	ctx := context.Background()
-	modMgr := module.NewSyncModuleManager(ctx, fs)
+	ss, err := state.NewStateStore()
+	if err != nil {
+		return err
+	}
+	modMgr := module.NewSyncModuleManager(ctx, fs, ss.Modules, ss.ProviderSchemas)
 	modMgr.SetLogger(c.logger)
 
 	walker := module.SyncWalker(fs, modMgr)
@@ -101,34 +106,33 @@ func (c *InspectModuleCommand) inspect(rootPath string) error {
 		return err
 	}
 
-	modules := modMgr.ListModules()
+	modules, err := modMgr.ListModules()
+	if err != nil {
+		return err
+	}
 	c.Ui.Output(fmt.Sprintf("%d modules found in total at %s", len(modules), rootPath))
 	for _, mod := range modules {
 		errs := &multierror.Error{}
 
-		_, err = mod.TerraformVersion()
-		if err != nil {
-			multierror.Append(errs, err)
+		if mod.TerraformVersionErr != nil {
+			multierror.Append(errs, mod.TerraformVersionErr)
 		}
 
-		_, err := mod.ProviderSchema()
-		if err != nil {
-			multierror.Append(errs, err)
+		if mod.ProviderSchemaErr != nil {
+			multierror.Append(errs, mod.ProviderSchemaErr)
 		}
 
-		_, err = mod.ModuleManifest()
-		if err != nil {
-			multierror.Append(errs, err)
+		if mod.ModManifestErr != nil {
+			multierror.Append(errs, mod.ModManifestErr)
 		}
 
-		_, err = mod.ParsedFiles()
-		if err != nil {
-			multierror.Append(errs, err)
+		if mod.ParsingErr != nil {
+			multierror.Append(errs, mod.ParsingErr)
 		}
 
 		errs.ErrorFormat = formatErrors
 
-		modules := formatModuleRecords(mod.ModuleCalls())
+		modules := formatModuleRecords(mod.ModManifest.Records)
 		subModules := fmt.Sprintf("%d modules", len(modules))
 		if len(modules) > 0 {
 			subModules += "\n"
@@ -139,7 +143,7 @@ func (c *InspectModuleCommand) inspect(rootPath string) error {
 
 		c.Ui.Output(fmt.Sprintf(` - %s
    - %s
-   - %s`, mod.Path(), errs, subModules))
+   - %s`, mod.Path, errs, subModules))
 	}
 	c.Ui.Output("")
 

internal/decoder/decoder.go

@@ -19,12 +19,7 @@ func DecoderForModule(ctx context.Context, mod module.Module) (*decoder.Decoder,
 		d.SetUtmMedium(clientName)
 	}
 
-	pf, err := mod.ParsedFiles()
-	if err != nil {
-		return nil, err
-	}
-
-	for name, f := range pf {
+	for name, f := range mod.ParsedFiles {
 		err := d.LoadFile(name, f)
 		if err != nil {
 			return nil, fmt.Errorf("failed to load a file: %w", err)

internal/langserver/handlers/command/modules.go

@@ -3,6 +3,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"sort"
 
 	"github.com/creachadair/jrpc2/code"
@@ -68,8 +69,8 @@ func ModulesHandler(ctx context.Context, args cmd.CommandArgs) (interface{}, err
 	modules := make([]moduleInfo, len(sources))
 	for i, source := range sources {
 		modules[i] = moduleInfo{
-			URI:  uri.FromPath(source.Path()),
-			Name: source.HumanReadablePath(rootDir),
+			URI:  uri.FromPath(source.Path),
+			Name: humanReadablePath(rootDir, source.Path),
 		}
 	}
 	sort.SliceStable(modules, func(i, j int) bool {
@@ -81,3 +82,23 @@ func ModulesHandler(ctx context.Context, args cmd.CommandArgs) (interface{}, err
 		Modules:         modules,
 	}, nil
 }
+
+func humanReadablePath(rootDir, modPath string) string {
+	if rootDir == "" {
+		return modPath
+	}
+
+	// absolute paths can be too long for UI/messages,
+	// so we just display relative to root dir
+	relDir, err := filepath.Rel(rootDir, modPath)
+	if err != nil {
+		return modPath
+	}
+
+	if relDir == "." {
+		// Name of the root dir is more helpful than "."
+		return filepath.Base(rootDir)
+	}
+
+	return relDir
+}

internal/langserver/handlers/command/validate.go

@@ -62,7 +62,7 @@ func TerraformValidateHandler(ctx context.Context, args cmd.CommandArgs) (interf
 
 	diags := diagnostics.HCLDiagsFromJSON(jsonDiags)
 
-	notifier.PublishHCLDiags(ctx, mod.Path(), diags, "terraform validate")
+	notifier.PublishHCLDiags(ctx, mod.Path, diags, "terraform validate")
 
 	return nil, nil
 }

internal/langserver/handlers/did_change.go

@@ -7,7 +7,7 @@ import (
 	lsctx "github.com/hashicorp/terraform-ls/internal/context"
 	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
 	lsp "github.com/hashicorp/terraform-ls/internal/protocol"
-	"github.com/hashicorp/terraform-ls/internal/terraform/module"
+	op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
 )
 
 func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocumentParams) error {
@@ -59,7 +59,11 @@ func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocument
 		return err
 	}
 
-	err = modMgr.EnqueueModuleOpWait(mod.Path(), module.OpTypeParseConfiguration)
+	err = modMgr.EnqueueModuleOpWait(mod.Path, op.OpTypeParseConfiguration)
+	if err != nil {
+		return err
+	}
+	err = modMgr.EnqueueModuleOpWait(mod.Path, op.OpTypeLoadModuleMetadata)
 	if err != nil {
 		return err
 	}
@@ -68,7 +72,7 @@ func TextDocumentDidChange(ctx context.Context, params lsp.DidChangeTextDocument
 	if err != nil {
 		return err
 	}
-	diags.PublishHCLDiags(ctx, mod.Path(), mod.Diagnostics(), "HCL")
+	diags.PublishHCLDiags(ctx, mod.Path, mod.Diagnostics, "HCL")
 
 	return nil
 }