Skip to content

Commit

Permalink
indexer: improve/cleanup error handling
Browse files Browse the repository at this point in the history 
 - capture and log any errors from Deferred jobs
 - capture and log any errors from other places which were previously skipped on error (via multierror)
  • Loading branch information
@radeksimko
radeksimko committed Jul 5, 2022
1 parent 19ec15f commit b1d2c50
Show file tree
Hide file tree
Showing 7 changed files with 110 additions and 85 deletions.
28 changes: 13 additions & 15 deletions internal/indexer/document_change.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,8 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseModuleConfiguration.String(),
Defer: func(ctx context.Context, jobErr error) job.IDs {
ids, err := idx.decodeModule(ctx, modHandle)
if err != nil {
idx.logger.Printf("error: %s", err)
}
return ids
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
return idx.decodeModule(ctx, modHandle)
},
})
if err != nil {
Expand All @@ -37,7 +33,8 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseVariables.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
ids := make(job.IDs, 0)
id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
Expand All @@ -46,10 +43,10 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
Type: op.OpTypeDecodeVarsReferences.String(),
})
if err != nil {
return
return ids, err
}
ids = append(ids, id)
return
return ids, nil
},
})
if err != nil {
Expand All @@ -69,7 +66,8 @@ func (idx *Indexer) decodeModule(ctx context.Context, modHandle document.DirHand
return module.LoadModuleMetadata(idx.modStore, modHandle.Path())
},
Type: op.OpTypeLoadModuleMetadata.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
ids := make(job.IDs, 0)
id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
Expand All @@ -78,7 +76,7 @@ func (idx *Indexer) decodeModule(ctx context.Context, modHandle document.DirHand
Type: op.OpTypeDecodeReferenceTargets.String(),
})
if err != nil {
return
return ids, err
}
ids = append(ids, id)

Expand All @@ -90,7 +88,7 @@ func (idx *Indexer) decodeModule(ctx context.Context, modHandle document.DirHand
Type: op.OpTypeDecodeReferenceOrigins.String(),
})
if err != nil {
return
return ids, err
}
ids = append(ids, id)

Expand All @@ -104,11 +102,11 @@ func (idx *Indexer) decodeModule(ctx context.Context, modHandle document.DirHand
Type: op.OpTypeGetModuleDataFromRegistry.String(),
})
if err != nil {
return
return ids, err
}
ids = append(ids, id)

return
ids = append(ids, id)
return ids, nil
},
})
if err != nil {
Expand Down
74 changes: 50 additions & 24 deletions internal/indexer/module_calls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,43 @@ import (
"context"
"os"

"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform-ls/internal/document"
"github.com/hashicorp/terraform-ls/internal/job"
"github.com/hashicorp/terraform-ls/internal/terraform/module"
op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
)

func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job.DeferFunc {
return func(ctx context.Context, opErr error) (jobIds job.IDs) {
return func(ctx context.Context, opErr error) (job.IDs, error) {
jobIds := make(job.IDs, 0)
if opErr != nil {
return
return jobIds, opErr
}

moduleCalls, err := idx.modStore.ModuleCalls(modHandle.Path())
if err != nil {
return
return jobIds, err
}

jobStore, err := job.JobStoreFromContext(ctx)
if err != nil {
return
return jobIds, err
}

var errs *multierror.Error

for _, mc := range moduleCalls.Installed {
fi, err := os.Stat(mc.Path)
if err != nil || !fi.IsDir() {
multierror.Append(errs, err)
continue
}
err = idx.modStore.Add(mc.Path)
if err != nil {
multierror.Append(errs, err)
continue
}
idx.modStore.Add(mc.Path)

mcHandle := document.DirHandleFromPath(mc.Path)
// copy path for queued jobs below
Expand All @@ -43,7 +52,10 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job
return module.ParseModuleConfiguration(idx.fs, idx.modStore, mcPath)
},
Type: op.OpTypeParseModuleConfiguration.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
ids := make(job.IDs, 0)
var errs *multierror.Error

id, err := jobStore.EnqueueJob(job.Job{
Dir: mcHandle,
Type: op.OpTypeLoadModuleMetadata.String(),
Expand All @@ -52,18 +64,24 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job
},
})
if err != nil {
return
errs = multierror.Append(errs, err)
} else {
ids = append(ids, id)
}
ids = append(ids, id)

rIds := idx.collectReferences(ctx, mcHandle)
ids = append(ids, rIds...)
rIds, err := idx.collectReferences(ctx, mcHandle)
if err != nil {
errs = multierror.Append(errs, err)
} else {
ids = append(ids, rIds...)
}

return
return ids, errs.ErrorOrNil()
},
})
if err != nil {
return
multierror.Append(errs, err)
continue
}
jobIds = append(jobIds, id)

Expand All @@ -73,7 +91,8 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job
return module.ParseVariables(idx.fs, idx.modStore, mcPath)
},
Type: op.OpTypeParseVariables.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
ids := make(job.IDs, 0)
id, err = jobStore.EnqueueJob(job.Job{
Dir: mcHandle,
Func: func(ctx context.Context) error {
Expand All @@ -82,28 +101,33 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) job
Type: op.OpTypeDecodeVarsReferences.String(),
})
if err != nil {
return
return ids, err
}
ids = append(ids, id)
return
return ids, err
},
})
if err != nil {
return
multierror.Append(errs, err)
continue
}
jobIds = append(jobIds, id)
}

return
return jobIds, errs.ErrorOrNil()
}
}

func (idx *Indexer) collectReferences(ctx context.Context, modHandle document.DirHandle) (ids job.IDs) {
func (idx *Indexer) collectReferences(ctx context.Context, modHandle document.DirHandle) (job.IDs, error) {
ids := make(job.IDs, 0)

jobStore, err := job.JobStoreFromContext(ctx)
if err != nil {
return
return ids, err
}

var errs *multierror.Error

id, err := jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
Expand All @@ -112,9 +136,10 @@ func (idx *Indexer) collectReferences(ctx context.Context, modHandle document.Di
Type: op.OpTypeDecodeReferenceTargets.String(),
})
if err != nil {
return
errs = multierror.Append(errs, err)
} else {
ids = append(ids, id)
}
ids = append(ids, id)

id, err = jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Expand All @@ -124,9 +149,10 @@ func (idx *Indexer) collectReferences(ctx context.Context, modHandle document.Di
Type: op.OpTypeDecodeReferenceOrigins.String(),
})
if err != nil {
return
errs = multierror.Append(errs, err)
} else {
ids = append(ids, id)
}
ids = append(ids, id)

return
return ids, errs.ErrorOrNil()
}
61 changes: 27 additions & 34 deletions internal/indexer/walker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package indexer
import (
"context"

"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform-ls/internal/document"
"github.com/hashicorp/terraform-ls/internal/job"
"github.com/hashicorp/terraform-ls/internal/terraform/datadir"
Expand All @@ -13,6 +14,7 @@ import (

func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHandle) (job.IDs, error) {
ids := make(job.IDs, 0)
var errs *multierror.Error

id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Expand All @@ -22,17 +24,20 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
Type: op.OpTypeParseModuleConfiguration.String(),
})
if err != nil {
return ids, err
errs = multierror.Append(errs, err)
} else {
ids = append(ids, id)
}
ids = append(ids, id)

id, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path())
},
Type: op.OpTypeParseVariables.String(),
Defer: func(ctx context.Context, jobErr error) (ids job.IDs) {
Defer: func(ctx context.Context, jobErr error) (job.IDs, error) {
ids := make(job.IDs, 0)

id, err := idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
Expand All @@ -41,16 +46,17 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
Type: op.OpTypeDecodeVarsReferences.String(),
})
if err != nil {
return
return ids, err
}
ids = append(ids, id)
return
return ids, err
},
})
if err != nil {
return ids, err
errs = multierror.Append(errs, err)
} else {
ids = append(ids, id)
}
ids = append(ids, id)

id, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Expand All @@ -61,9 +67,10 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
Type: op.OpTypeGetTerraformVersion.String(),
})
if err != nil {
return ids, err
errs = multierror.Append(errs, err)
} else {
ids = append(ids, id)
}
ids = append(ids, id)

dataDir := datadir.WalkDataDirOfModule(idx.fs, modHandle.Path())
idx.logger.Printf("parsed datadir: %#v", dataDir)
Expand All @@ -78,9 +85,10 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
Type: op.OpTypeObtainSchema.String(),
})
if err != nil {
return ids, err
errs = multierror.Append(errs, err)
} else {
ids = append(ids, id)
}
ids = append(ids, id)
}

if dataDir.ModuleManifestPath != "" {
Expand All @@ -95,7 +103,9 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
Defer: idx.decodeInstalledModuleCalls(modHandle),
})
if err != nil {
return ids, err
errs = multierror.Append(errs, err)
} else {
ids = append(ids, id)
}

// Here we wait for all module calls to be processed to
Expand All @@ -105,29 +115,12 @@ func (idx *Indexer) WalkedModule(ctx context.Context, modHandle document.DirHand
idx.jobStore.WaitForJobs(ctx, id)
}

id, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.DecodeReferenceTargets(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceTargets.String(),
})
if err != nil {
return ids, err
}
ids = append(ids, id)

id, err = idx.jobStore.EnqueueJob(job.Job{
Dir: modHandle,
Func: func(ctx context.Context) error {
return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
},
Type: op.OpTypeDecodeReferenceOrigins.String(),
})
rIds, err := idx.collectReferences(ctx, modHandle)
if err != nil {
return ids, err
errs = multierror.Append(errs, err)
} else {
ids = append(ids, rIds...)
}
ids = append(ids, id)

return ids, nil
return ids, errs.ErrorOrNil()
}
2 changes: 1 addition & 1 deletion internal/job/job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ type Job struct {
// DeferFunc represents a deferred function scheduling more jobs
// based on jobErr (any error returned from the main job).
// Newly queued job IDs should be returned to allow for synchronization.
type DeferFunc func(ctx context.Context, jobErr error) IDs
type DeferFunc func(ctx context.Context, jobErr error) (IDs, error)

func (job Job) Copy() Job {
return Job{
Expand Down
Loading

0 comments on commit b1d2c50

Please sign in to comment.