Skip to content

Commit

Permalink
indexer: Ensure declared module calls get decoded (#1395)
Browse files Browse the repository at this point in the history 
* indexer: decouple decoding of a single module call

* indexer: implement decodeDeclaredModuleCalls

* deps: pin terraform-schema to b39f3de

* indexer: add missing job dependency
  • Loading branch information
@radeksimko
radeksimko authored Sep 11, 2023
1 parent da68b87 commit 5ccfde6
Show file tree
Hide file tree
Showing 4 changed files with 169 additions and 102 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
github.com/hashicorp/terraform-exec v0.19.0
github.com/hashicorp/terraform-json v0.17.1
github.com/hashicorp/terraform-registry-address v0.2.2
github.com/hashicorp/terraform-schema v0.0.0-20230904125443-90a397096838
github.com/hashicorp/terraform-schema v0.0.0-20230908130940-b39f3de08c04
github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5
github.com/mitchellh/cli v1.1.5
github.com/mitchellh/go-homedir v1.1.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,8 @@ github.com/hashicorp/terraform-json v0.17.1 h1:eMfvh/uWggKmY7Pmb3T85u86E2EQg6EQH
github.com/hashicorp/terraform-json v0.17.1/go.mod h1:Huy6zt6euxaY9knPAFKjUITn8QxUFIe9VuSzb4zn/0o=
github.com/hashicorp/terraform-registry-address v0.2.2 h1:lPQBg403El8PPicg/qONZJDC6YlgCVbWDtNmmZKtBno=
github.com/hashicorp/terraform-registry-address v0.2.2/go.mod h1:LtwNbCihUoUZ3RYriyS2wF/lGPB6gF9ICLRtuDk7hSo=
github.com/hashicorp/terraform-schema v0.0.0-20230904125443-90a397096838 h1:sixoZnLpWvkWdPObUKRuDy4dKlStgnkmFjXHqCyUeHQ=
github.com/hashicorp/terraform-schema v0.0.0-20230904125443-90a397096838/go.mod h1:PXhA8crTZkaqg56PrGKM+fHsOgeaORhmjjTj/7N9kyg=
github.com/hashicorp/terraform-schema v0.0.0-20230908130940-b39f3de08c04 h1:3Kemwg4PV/HGrACFEthhiLBF6vmZFT34HFHNIUDx19s=
github.com/hashicorp/terraform-schema v0.0.0-20230908130940-b39f3de08c04/go.mod h1:yxWEW1URl7wekbnH7OEoiwtb7CNYPVeguOv8LwHh0UI=
github.com/hashicorp/terraform-svchost v0.1.1 h1:EZZimZ1GxdqFRinZ1tpJwVxxt49xc/S52uzrw4x0jKQ=
github.com/hashicorp/terraform-svchost v0.1.1/go.mod h1:mNsjQfZyf/Jhz35v6/0LWcv26+X7JPS+buii2c9/ctc=
github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo=
Expand Down
67 changes: 39 additions & 28 deletions internal/indexer/document_change.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,25 +76,50 @@ func (idx *Indexer) decodeModule(ctx context.Context, modHandle document.DirHand
Type: op.OpTypeLoadModuleMetadata.String(),
DependsOn: dependsOn,
IgnoreState: ignoreState,
})
if err != nil {
return ids, err
}
ids = append(ids, metaId)

eSchemaId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.PreloadEmbeddedSchema(ctx, idx.logger, schemas.FS, idx.modStore, idx.schemaStore, modHandle.Path())
Defer: func(ctx context.Context, jobErr error) (jobIds job.IDs, err error) {
if jobErr != nil {
err = jobErr
return
}
modCalls, mcErr := idx.decodeDeclaredModuleCalls(ctx, modHandle, ignoreState)
if mcErr != nil {
idx.logger.Printf("decoding declared module calls for %q failed: %s", modHandle.URI, mcErr)
// We log the error but still continue scheduling other jobs
// which are still valuable for the rest of the configuration
// even if they may not have the data for module calls.
}

eSchemaId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.PreloadEmbeddedSchema(ctx, idx.logger, schemas.FS, idx.modStore, idx.schemaStore, modHandle.Path())
},
DependsOn: modCalls,
Type: op.OpTypePreloadEmbeddedSchema.String(),
IgnoreState: ignoreState,
})
if err != nil {
return
}
jobIds = append(jobIds, eSchemaId)

refOriginsId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceOrigins.String(),
DependsOn: append(modCalls, eSchemaId),
IgnoreState: ignoreState,
})
jobIds = append(jobIds, refOriginsId)
return
},
DependsOn: job.IDs{metaId},
Type: op.OpTypePreloadEmbeddedSchema.String(),
IgnoreState: ignoreState,
})
if err != nil {
return ids, err
}
ids = append(ids, eSchemaId)
ids = append(ids, metaId)

refTargetsId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Expand All @@ -110,20 +135,6 @@ func (idx *Indexer) decodeModule(ctx context.Context, modHandle document.DirHand
}
ids = append(ids, refTargetsId)

refOriginsId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceOrigins.String(),
DependsOn: job.IDs{metaId},
IgnoreState: ignoreState,
})
if err != nil {
return ids, err
}
ids = append(ids, refOriginsId)

// This job may make an HTTP request, and we schedule it in
// the low-priority queue, so we don't want to wait for it.
_, err = idx.jobStore.EnqueueJob(ctx, job.Job{
Expand Down
198 changes: 127 additions & 71 deletions internal/indexer/module_calls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,18 @@ package indexer

import (
"context"
"errors"
"os"
"path/filepath"

"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform-ls/internal/document"
"github.com/hashicorp/terraform-ls/internal/job"
"github.com/hashicorp/terraform-ls/internal/schemas"
"github.com/hashicorp/terraform-ls/internal/state"
"github.com/hashicorp/terraform-ls/internal/terraform/module"
op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
tfmodule "github.com/hashicorp/terraform-schema/module"
)

func (idx *Indexer) decodeInstalledModuleCalls(ctx context.Context, modHandle document.DirHandle, ignoreState bool) (job.IDs, error) {
Expand All @@ -39,99 +43,151 @@ func (idx *Indexer) decodeInstalledModuleCalls(ctx context.Context, modHandle do
}

mcHandle := document.DirHandleFromPath(mc.Path)
// copy path for queued jobs below
mcPath := mc.Path
mcJobIds, mcErr := idx.decodeModuleAtPath(ctx, mcHandle, ignoreState)
jobIds = append(jobIds, mcJobIds...)
multierror.Append(errs, mcErr)
}

return jobIds, errs.ErrorOrNil()
}

func (idx *Indexer) decodeDeclaredModuleCalls(ctx context.Context, modHandle document.DirHandle, ignoreState bool) (job.IDs, error) {
jobIds := make(job.IDs, 0)

moduleCalls, err := idx.modStore.ModuleCalls(modHandle.Path())
if err != nil {
return jobIds, err
}

var errs *multierror.Error

idx.logger.Printf("indexing declared module calls for %q: %d", modHandle.URI, len(moduleCalls.Declared))
for _, mc := range moduleCalls.Declared {
localSource, ok := mc.SourceAddr.(tfmodule.LocalSourceAddr)
if !ok {
continue
}
mcPath := filepath.Join(modHandle.Path(), filepath.FromSlash(localSource.String()))

fi, err := os.Stat(mcPath)
if err != nil || !fi.IsDir() {
multierror.Append(errs, err)
continue
}

mcIgnoreState := ignoreState
err = idx.modStore.Add(mcPath)
if err != nil {
alreadyExistsErr := &state.AlreadyExistsError{}
if errors.As(err, &alreadyExistsErr) {
mcIgnoreState = false
} else {
multierror.Append(errs, err)
continue
}
}

mcHandle := document.DirHandleFromPath(mcPath)
mcJobIds, mcErr := idx.decodeModuleAtPath(ctx, mcHandle, mcIgnoreState)
jobIds = append(jobIds, mcJobIds...)
multierror.Append(errs, mcErr)
}

return jobIds, errs.ErrorOrNil()
}

refCollectionDeps := make(job.IDs, 0)
func (idx *Indexer) decodeModuleAtPath(ctx context.Context, modHandle document.DirHandle, ignoreState bool) (job.IDs, error) {
var errs *multierror.Error
jobIds := make(job.IDs, 0)
refCollectionDeps := make(job.IDs, 0)

parseId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: mcHandle,
parseId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleConfiguration.String(),
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, parseId)
refCollectionDeps = append(refCollectionDeps, parseId)
}

var metaId job.ID
if parseId != "" {
metaId, err = idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Type: op.OpTypeLoadModuleMetadata.String(),
Func: func(ctx context.Context) error {
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, mcPath)
return module.LoadModuleMetadata(ctx, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleConfiguration.String(),
DependsOn: job.IDs{parseId},
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, parseId)
refCollectionDeps = append(refCollectionDeps, parseId)
jobIds = append(jobIds, metaId)
refCollectionDeps = append(refCollectionDeps, metaId)
}

var metaId job.ID
if parseId != "" {
metaId, err = idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: mcHandle,
Type: op.OpTypeLoadModuleMetadata.String(),
Func: func(ctx context.Context) error {
return module.LoadModuleMetadata(ctx, idx.modStore, mcPath)
},
DependsOn: job.IDs{parseId},
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, metaId)
refCollectionDeps = append(refCollectionDeps, metaId)
}

eSchemaId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: mcHandle,
Func: func(ctx context.Context) error {
return module.PreloadEmbeddedSchema(ctx, idx.logger, schemas.FS, idx.modStore, idx.schemaStore, mcPath)
},
Type: op.OpTypePreloadEmbeddedSchema.String(),
DependsOn: job.IDs{metaId},
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, eSchemaId)
refCollectionDeps = append(refCollectionDeps, eSchemaId)
}
eSchemaId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.PreloadEmbeddedSchema(ctx, idx.logger, schemas.FS, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypePreloadEmbeddedSchema.String(),
DependsOn: job.IDs{metaId},
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, eSchemaId)
refCollectionDeps = append(refCollectionDeps, eSchemaId)
}
}

if parseId != "" {
ids, err := idx.collectReferences(ctx, mcHandle, refCollectionDeps, ignoreState)
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, ids...)
}
if parseId != "" {
ids, err := idx.collectReferences(ctx, modHandle, refCollectionDeps, ignoreState)
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, ids...)
}
}

varsParseId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: mcHandle,
varsParseId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseVariables.String(),
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, varsParseId)
}

if varsParseId != "" {
varsRefId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseVariables(ctx, idx.fs, idx.modStore, mcPath)
return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeParseVariables.String(),
Type: op.OpTypeDecodeVarsReferences.String(),
DependsOn: job.IDs{varsParseId},
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, varsParseId)
}

if varsParseId != "" {
varsRefId, err := idx.jobStore.EnqueueJob(ctx, job.Job{
Dir: mcHandle,
Func: func(ctx context.Context) error {
return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, mcPath)
},
Type: op.OpTypeDecodeVarsReferences.String(),
DependsOn: job.IDs{varsParseId},
IgnoreState: ignoreState,
})
if err != nil {
multierror.Append(errs, err)
} else {
jobIds = append(jobIds, varsRefId)
}
jobIds = append(jobIds, varsRefId)
}
}

Expand Down

0 comments on commit 5ccfde6

Please sign in to comment.