Skip to content

Commit

Permalink
Parse provider versions from lock file before obtaining schema (hashi…
Browse files Browse the repository at this point in the history 
…corp#1014)

* terraform/datadir: Introduce ParsePluginVersions

* Parse plugin versions and update tests

* fix: Avoid empty job IDs & persist errors
  • Loading branch information
@radeksimko
radeksimko authored Aug 4, 2022
1 parent 48ab782 commit 9cfdd2f
Show file tree
Hide file tree
Showing 22 changed files with 941 additions and 108 deletions.
72 changes: 56 additions & 16 deletions internal/indexer/walker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
var errs *multierror.Error

refCollectionDeps := make(job.IDs, 0)
providerVersionDeps := make(job.IDs, 0)

parseId, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Expand All @@ -30,6 +31,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
} else {
ids = append(ids, parseId)
refCollectionDeps = append(refCollectionDeps, parseId)
providerVersionDeps = append(providerVersionDeps, parseId)
}

var metaId job.ID
Expand All @@ -47,6 +49,7 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
} else {
ids = append(ids, metaId)
refCollectionDeps = append(refCollectionDeps, metaId)
providerVersionDeps = append(providerVersionDeps, metaId)
}
}

Expand Down Expand Up @@ -98,41 +101,78 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
dataDir := datadir.WalkDataDirOfModule(idx.fs, modHandle.Path())
idx.logger.Printf("parsed datadir: %#v", dataDir)

if dataDir.PluginLockFilePath != "" {
pSchemaId, err := idx.jobStore.EnqueueJob(job.Job{
var modManifestId job.ID
if dataDir.ModuleManifestPath != "" {
// References are collected *after* manifest parsing
// so that we reflect any references to submodules.
modManifestId, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
return module.ParseModuleManifest(idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleManifest.String(),
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
return idx.decodeInstalledModuleCalls(modHandle)
},
Type: op.OpTypeObtainSchema.String(),
})
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, pSchemaId)
refCollectionDeps = append(refCollectionDeps, pSchemaId)
ids = append(ids, modManifestId)
refCollectionDeps = append(refCollectionDeps, modManifestId)
// provider requirements may be within the (installed) modules
providerVersionDeps = append(providerVersionDeps, modManifestId)
}
}

if dataDir.ModuleManifestPath != "" {
// References are collected *after* manifest parsing
// so that we reflect any references to submodules.
modManifestId, err := idx.jobStore.EnqueueJob(job.Job{
if dataDir.PluginLockFilePath != "" {
pSchemaId, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseModuleManifest(idx.fs, idx.modStore, modHandle.Path())
return module.ParseProviderVersions(idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleManifest.String(),
Type: op.OpTypeParseProviderVersions.String(),
DependsOn: providerVersionDeps,
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
return idx.decodeInstalledModuleCalls(modHandle)
ids := make(job.IDs, 0)

pReqs, err := idx.modStore.ProviderRequirementsForModule(modHandle.Path())
if err != nil {
return ids, err
}

exist, err := idx.schemaStore.AllSchemasExist(pReqs)
if err != nil {
return ids, err
}
if exist {
idx.logger.Printf("Avoiding obtaining schemas as they all exist: %#v", pReqs)
// avoid obtaining schemas if we already have it
return ids, nil
}
idx.logger.Printf("Obtaining schemas for: %#v", pReqs)

id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeObtainSchema.String(),
})
if err != nil {
return ids, err
}
ids = append(ids, id)

return ids, nil
},
})
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, modManifestId)
refCollectionDeps = append(refCollectionDeps, modManifestId)
ids = append(ids, pSchemaId)
refCollectionDeps = append(refCollectionDeps, pSchemaId)
}
}

Expand Down
43 changes: 37 additions & 6 deletions internal/indexer/watcher.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,46 @@ func (idx *Indexer) PluginLockChanged(ctx context.Context, modHandle document.Di
id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
eo, ok := exec.ExecutorOptsFromContext(ctx)
if ok {
ctx = exec.WithExecutorOpts(ctx, eo)
return module.ParseProviderVersions(idx.fs, idx.modStore, modHandle.Path())
},
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
ids := make(job.IDs, 0)

mod, err := idx.modStore.ModuleByPath(modHandle.Path())
if err != nil {
return ids, err
}

exist, err := idx.schemaStore.AllSchemasExist(mod.Meta.ProviderRequirements)
if err != nil {
return ids, err
}
if exist {
// avoid obtaining schemas if we already have it
return ids, nil
}

id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
eo, ok := exec.ExecutorOptsFromContext(ctx)
if ok {
ctx = exec.WithExecutorOpts(ctx, eo)
}

return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeObtainSchema.String(),
})
if err != nil {
return ids, err
}
ids = append(ids, id)

return module.ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
return ids, nil
},
Type: op.OpTypeObtainSchema.String(),
Type: op.OpTypeParseProviderVersions.String(),
})
if err != nil {
return ids, err
Expand Down
14 changes: 12 additions & 2 deletions internal/langserver/handlers/complete_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
Expand Down Expand Up @@ -45,8 +46,13 @@ func TestModuleCompletion_withValidData_basic(t *testing.T) {
tmpDir := TempDir(t)
InitPluginCache(t, tmpDir.Path())

err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
if err != nil {
t.Fatal(err)
}

var testSchema tfjson.ProviderSchemas
err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -259,9 +265,13 @@ func TestModuleCompletion_withValidData_basic(t *testing.T) {
func TestModuleCompletion_withValidDataAndSnippets(t *testing.T) {
tmpDir := TempDir(t)
InitPluginCache(t, tmpDir.Path())
err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
if err != nil {
t.Fatal(err)
}

var testSchema tfjson.ProviderSchemas
err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
if err != nil {
t.Fatal(err)
}
Expand Down
26 changes: 1 addition & 25 deletions internal/langserver/handlers/did_change_watched_files_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,7 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) {
t.Fatal(err)
}

originalTestDir := filepath.Join(testData, "uninitialized-single-submodule")
originalTestDir := filepath.Join(testData, "single-fake-provider")
testDir := t.TempDir()
// Copy test configuration so the test can run in isolation
err = copy.Copy(originalTestDir, testDir)
Expand Down Expand Up @@ -788,30 +788,6 @@ func TestLangServer_DidChangeWatchedFiles_pluginChange(t *testing.T) {
t.Fatal("expected -/foo schema to be missing")
}

// Install Terraform
tfVersion := version.Must(version.NewVersion("1.1.7"))
i := install.NewInstaller()
ctx := context.Background()
execPath, err := i.Install(ctx, []src.Installable{
&releases.ExactVersion{
Product: product.Terraform,
Version: tfVersion,
},
})
if err != nil {
t.Fatal(err)
}

// Install submodule
tf, err := exec.NewExecutor(testHandle.Path(), execPath)
if err != nil {
t.Fatal(err)
}
err = tf.Init(ctx)
if err != nil {
t.Fatal(err)
}

ls.Call(t, &langserver.CallRequest{
Method: "workspace/didChangeWatchedFiles",
ReqParams: fmt.Sprintf(`{
Expand Down
8 changes: 7 additions & 1 deletion internal/langserver/handlers/document_link_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package handlers
import (
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"testing"

"github.com/hashicorp/go-version"
Expand All @@ -17,9 +19,13 @@ import (
func TestDocumentLink_withValidData(t *testing.T) {
tmpDir := TempDir(t)
InitPluginCache(t, tmpDir.Path())
err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
if err != nil {
t.Fatal(err)
}

var testSchema tfjson.ProviderSchemas
err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
if err != nil {
t.Fatal(err)
}
Expand Down
2 changes: 1 addition & 1 deletion internal/langserver/handlers/execute_command_module_providers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func TestLangServer_workspaceExecuteCommand_moduleProviders_basic(t *testing.T)
newDefaultProvider("aws"): version.Must(version.NewVersion("1.2.3")),
newDefaultProvider("google"): version.Must(version.NewVersion("2.5.5")),
}
err = s.Modules.UpdateInstalledProviders(modDir, pVersions)
err = s.Modules.UpdateInstalledProviders(modDir, pVersions, nil)
if err != nil {
t.Fatal(err)
}
Expand Down
22 changes: 19 additions & 3 deletions internal/langserver/handlers/go_to_ref_target_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package handlers
import (
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"testing"

Expand Down Expand Up @@ -118,8 +119,13 @@ func TestDefinition_withLinkToDefLessBlock(t *testing.T) {
tmpDir := TempDir(t)
InitPluginCache(t, tmpDir.Path())

err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
if err != nil {
t.Fatal(err)
}

var testSchema tfjson.ProviderSchemas
err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -266,8 +272,13 @@ func TestDefinition_withLinkToDefBlock(t *testing.T) {
tmpDir := TempDir(t)
InitPluginCache(t, tmpDir.Path())

err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
if err != nil {
t.Fatal(err)
}

var testSchema tfjson.ProviderSchemas
err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -598,8 +609,13 @@ func TestDeclaration_withLinkSupport(t *testing.T) {
tmpDir := TempDir(t)
InitPluginCache(t, tmpDir.Path())

err := ioutil.WriteFile(filepath.Join(tmpDir.Path(), "main.tf"), []byte("provider \"test\" {\n\n}\n"), 0o755)
if err != nil {
t.Fatal(err)
}

var testSchema tfjson.ProviderSchemas
err := json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
err = json.Unmarshal([]byte(testModuleSchemaOutput), &testSchema)
if err != nil {
t.Fatal(err)
}
Expand Down
3 changes: 3 additions & 0 deletions internal/langserver/handlers/testdata/single-fake-provider/main.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
provider "foo" {

}
Loading

0 comments on commit 9cfdd2f

Please sign in to comment.