Skip to content

Commit

Permalink
refactor: Decouple walker & indexing logic (#981)
Browse files Browse the repository at this point in the history
* refactor: Decouple walker & indexing logic

This refactoring solves two problems:
 - reduce dependencies of the walker and make it focused on _walking_ by moving out the indexing logic
 - move all indexing logic to a new dedicated indexer package, so that it’s easier deduplicate and reuse code within the package

* move terraform/module/testdata -> walker/testdata

* avoid error catching in Exists to aid readability
  • Loading branch information
radeksimko authored Jul 1, 2022
1 parent b93fcac commit 8fba5f0
Show file tree
Hide file tree
Showing 831 changed files with 726 additions and 626 deletions.
19 changes: 12 additions & 7 deletions internal/cmd/inspect_module_command.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@ import (
ictx "github.com/hashicorp/terraform-ls/internal/context"
"github.com/hashicorp/terraform-ls/internal/document"
"github.com/hashicorp/terraform-ls/internal/filesystem"
"github.com/hashicorp/terraform-ls/internal/indexer"
"github.com/hashicorp/terraform-ls/internal/logging"
"github.com/hashicorp/terraform-ls/internal/registry"
"github.com/hashicorp/terraform-ls/internal/state"
"github.com/hashicorp/terraform-ls/internal/terraform/datadir"
"github.com/hashicorp/terraform-ls/internal/terraform/exec"
"github.com/hashicorp/terraform-ls/internal/terraform/module"
"github.com/hashicorp/terraform-ls/internal/walker"
"github.com/mitchellh/cli"
)

Expand Down Expand Up @@ -97,19 +99,22 @@ func (c *InspectModuleCommand) inspect(rootPath string) error {

ctx := context.Background()

indexer := indexer.NewIndexer(fs, ss.Modules, ss.ProviderSchemas, ss.RegistryModules, ss.JobStore,
exec.NewExecutor, registry.NewClient())

pa := state.NewPathAwaiter(ss.WalkerPaths, false)
walker := module.NewWalker(fs, pa, ss.Modules, ss.ProviderSchemas, ss.JobStore, exec.NewExecutor)
walker.Collector = module.NewWalkerCollector()
walker.SetLogger(c.logger)
w := walker.NewWalker(pa, ss.Modules, indexer.WalkedModule)
w.Collector = walker.NewWalkerCollector()
w.SetLogger(c.logger)
err = ss.WalkerPaths.WaitForDirs(ctx, []document.DirHandle{dir})
if err != nil {
return err
}
err = ss.JobStore.WaitForJobs(ctx, walker.Collector.JobIds()...)
err = ss.JobStore.WaitForJobs(ctx, w.Collector.JobIds()...)
if err != nil {
return err
}
err = walker.Collector.ErrorOrNil()
err = w.Collector.ErrorOrNil()
if err != nil {
return err
}
Expand All @@ -122,7 +127,7 @@ func (c *InspectModuleCommand) inspect(rootPath string) error {
if err != nil {
return err
}
err = walker.StartWalking(ctx)
err = w.StartWalking(ctx)
if err != nil {
return err
}
Expand Down
120 changes: 120 additions & 0 deletions internal/indexer/document_change.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package indexer

import (
"context"

"github.com/hashicorp/terraform-ls/internal/document"
"github.com/hashicorp/terraform-ls/internal/job"
"github.com/hashicorp/terraform-ls/internal/terraform/module"
op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
)

func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, error) {
ids := make(job.IDs, 0)

id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleConfiguration.String(),
Defer: func(ctx context.Context, jobErr error) job.IDs {
ids, err := idx.decodeModule(ctx, modHandle)
if err != nil {
idx.logger.Printf("error: %s", err)
}
return ids
},
})
if err != nil {
return ids, err
}
ids = append(ids, id)

id, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseVariables.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeVarsReferences.String(),
})
if err != nil {
return
}
ids = append(ids, id)
return
},
})
if err != nil {
return ids, err
}
ids = append(ids, id)

return ids, nil
}

func (idx *Indexer) decodeModule(ctx context.Context, modHandle document.DirHandle) (job.IDs, error) {
ids := make(job.IDs, 0)

id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.LoadModuleMetadata(idx.modStore, modHandle.Path())
},
Type: op.OpTypeLoadModuleMetadata.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.DecodeReferenceTargets(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceTargets.String(),
})
if err != nil {
return
}
ids = append(ids, id)

id, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceOrigins.String(),
})
if err != nil {
return
}
ids = append(ids, id)

id, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.GetModuleDataFromRegistry(ctx, idx.registryClient,
idx.modStore, idx.registryModStore, modHandle.Path())
},
Priority: job.LowPriority,
Type: op.OpTypeGetModuleDataFromRegistry.String(),
})
if err != nil {
return
}
ids = append(ids, id)

return
},
})
if err != nil {
return ids, err
}
ids = append(ids, id)

return ids, nil
}
10 changes: 10 additions & 0 deletions internal/indexer/fs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package indexer

import "io/fs"

type ReadOnlyFS interface {
fs.FS
ReadDir(name string) ([]fs.DirEntry, error)
ReadFile(name string) ([]byte, error)
Stat(name string) (fs.FileInfo, error)
}
111 changes: 36 additions & 75 deletions internal/terraform/module/indexer.go → internal/indexer/indexer.go
Original file line number Diff line number Diff line change
@@ -1,94 +1,55 @@
package module
package indexer

import (
"context"
"io/ioutil"
"log"
"os"

"github.com/hashicorp/terraform-ls/internal/document"
"github.com/hashicorp/terraform-ls/internal/job"
"github.com/hashicorp/terraform-ls/internal/registry"
"github.com/hashicorp/terraform-ls/internal/state"
"github.com/hashicorp/terraform-ls/internal/terraform/exec"
"github.com/hashicorp/terraform-ls/internal/terraform/module"
op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
)

type Indexer struct {
fs ReadOnlyFS
modStore *state.ModuleStore
schemaStore *state.ProviderSchemaStore
jobStore job.JobStore
tfExecFactory exec.ExecutorFactory
logger *log.Logger
fs ReadOnlyFS
modStore *state.ModuleStore
schemaStore *state.ProviderSchemaStore
registryModStore *state.RegistryModuleStore
jobStore job.JobStore
tfExecFactory exec.ExecutorFactory
registryClient registry.Client
}

func NewIndexer(fs ReadOnlyFS, modStore *state.ModuleStore, schemaStore *state.ProviderSchemaStore,
jobStore job.JobStore, tfExec exec.ExecutorFactory) *Indexer {
return &Indexer{
fs: fs,
modStore: modStore,
schemaStore: schemaStore,
jobStore: jobStore,
tfExecFactory: tfExec,
}
}
registryModStore *state.RegistryModuleStore, jobStore job.JobStore,
tfExec exec.ExecutorFactory, registryClient registry.Client) *Indexer {

func (idx *Indexer) ModuleManifestChanged(ctx context.Context, modHandle document.DirHandle) (job.IDs, error) {
ids := make(job.IDs, 0)
discardLogger := log.New(ioutil.Discard, "", 0)

id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return ParseModuleManifest(idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleManifest.String(),
Defer: decodeInstalledModuleCalls(idx.fs, idx.modStore, idx.schemaStore, modHandle.Path()),
})
if err != nil {
return ids, err
return &Indexer{
fs: fs,
modStore: modStore,
schemaStore: schemaStore,
registryModStore: registryModStore,
jobStore: jobStore,
tfExecFactory: tfExec,
registryClient: registryClient,
logger: discardLogger,
}
ids = append(ids, id)

return ids, nil
}

func (idx *Indexer) PluginLockChanged(ctx context.Context, modHandle document.DirHandle) (job.IDs, error) {
ids := make(job.IDs, 0)

id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
eo, ok := exec.ExecutorOptsFromContext(ctx)
if ok {
ctx = exec.WithExecutorOpts(ctx, eo)
}

return ObtainSchema(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeObtainSchema.String(),
})
if err != nil {
return ids, err
}
ids = append(ids, id)

id, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
ctx = exec.WithExecutorFactory(ctx, idx.tfExecFactory)
eo, ok := exec.ExecutorOptsFromContext(ctx)
if ok {
ctx = exec.WithExecutorOpts(ctx, eo)
}

return GetTerraformVersion(ctx, idx.modStore, modHandle.Path())
},
Type: op.OpTypeGetTerraformVersion.String(),
})
if err != nil {
return ids, err
}
ids = append(ids, id)
func (idx *Indexer) SetLogger(logger *log.Logger) {
idx.logger = logger
}

return ids, nil
type Collector interface {
CollectJobId(jobId job.ID)
}

func decodeInstalledModuleCalls(fs ReadOnlyFS, modStore *state.ModuleStore, schemaReader state.SchemaReader, modPath string) job.DeferFunc {
Expand Down Expand Up @@ -121,15 +82,15 @@ func decodeInstalledModuleCalls(fs ReadOnlyFS, modStore *state.ModuleStore, sche
id, err := jobStore.EnqueueJob(job.Job{
Dir: mcHandle,
Func: func(ctx context.Context) error {
return ParseModuleConfiguration(fs, modStore, mcPath)
return module.ParseModuleConfiguration(fs, modStore, mcPath)
},
Type: op.OpTypeParseModuleConfiguration.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
id, err := jobStore.EnqueueJob(job.Job{
Dir: mcHandle,
Type: op.OpTypeLoadModuleMetadata.String(),
Func: func(ctx context.Context) error {
return LoadModuleMetadata(modStore, mcPath)
return module.LoadModuleMetadata(modStore, mcPath)
},
})
if err != nil {
Expand All @@ -151,14 +112,14 @@ func decodeInstalledModuleCalls(fs ReadOnlyFS, modStore *state.ModuleStore, sche
id, err = jobStore.EnqueueJob(job.Job{
Dir: mcHandle,
Func: func(ctx context.Context) error {
return ParseVariables(fs, modStore, mcPath)
return module.ParseVariables(fs, modStore, mcPath)
},
Type: op.OpTypeParseVariables.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
id, err = jobStore.EnqueueJob(job.Job{
Dir: mcHandle,
Func: func(ctx context.Context) error {
return DecodeVarsReferences(ctx, modStore, schemaReader, mcPath)
return module.DecodeVarsReferences(ctx, modStore, schemaReader, mcPath)
},
Type: op.OpTypeDecodeVarsReferences.String(),
})
Expand Down Expand Up @@ -188,7 +149,7 @@ func collectReferences(ctx context.Context, dirHandle document.DirHandle, modSto
id, err := jobStore.EnqueueJob(job.Job{
Dir: dirHandle,
Func: func(ctx context.Context) error {
return DecodeReferenceTargets(ctx, modStore, schemaReader, dirHandle.Path())
return module.DecodeReferenceTargets(ctx, modStore, schemaReader, dirHandle.Path())
},
Type: op.OpTypeDecodeReferenceTargets.String(),
})
Expand All @@ -200,7 +161,7 @@ func collectReferences(ctx context.Context, dirHandle document.DirHandle, modSto
id, err = jobStore.EnqueueJob(job.Job{
Dir: dirHandle,
Func: func(ctx context.Context) error {
return DecodeReferenceOrigins(ctx, modStore, schemaReader, dirHandle.Path())
return module.DecodeReferenceOrigins(ctx, modStore, schemaReader, dirHandle.Path())
},
Type: op.OpTypeDecodeReferenceOrigins.String(),
})
Expand Down
Loading

0 comments on commit 8fba5f0

Please sign in to comment.