From 130111e68f7a4ef4211103da9810ceeee0b6e19d Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Tue, 26 Oct 2021 20:15:41 +0100 Subject: [PATCH] Implement opt-in telemetry (#681) * Implement opt-in telemetry events * Update README.md Co-authored-by: James Pogran * Update README.md Co-authored-by: James Pogran Co-authored-by: James Pogran --- README.md | 7 +- docs/language-clients.md | 4 + docs/telemetry.md | 88 ++++++++++++++++ go.mod | 3 +- go.sum | 6 +- internal/langserver/handlers/hooks_module.go | 87 ++++++++++++++++ internal/langserver/handlers/initialize.go | 44 +++++++- internal/langserver/handlers/service.go | 17 ++++ internal/langserver/session/types.go | 4 + internal/protocol/experimental.go | 12 +++ internal/protocol/telemetry.go | 10 ++ internal/state/hooks.go | 11 ++ internal/state/module.go | 102 ++++++++++++++++++- internal/state/module_ids.go | 38 +++++++ internal/state/provider_ids.go | 41 ++++++++ internal/state/state.go | 39 +++++-- internal/telemetry/noop.go | 22 ++++ internal/telemetry/sender.go | 7 ++ internal/telemetry/telemetry.go | 47 +++++++++ internal/terraform/module/module_ops.go | 15 ++- 20 files changed, 582 insertions(+), 22 deletions(-) create mode 100644 docs/telemetry.md create mode 100644 internal/langserver/handlers/hooks_module.go create mode 100644 internal/protocol/telemetry.go create mode 100644 internal/state/hooks.go create mode 100644 internal/state/module_ids.go create mode 100644 internal/state/provider_ids.go create mode 100644 internal/telemetry/noop.go create mode 100644 internal/telemetry/sender.go create mode 100644 internal/telemetry/telemetry.go diff --git a/README.md b/README.md index 65274992b..dd6006aa3 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,12 @@ Please follow the [relevant guide for your IDE](./docs/USAGE.md). - [Martin Atkins](https://github.com/apparentlymart) - particularly the virtual filesystem - [Zhe Cheng](https://github.com/njuCZ) - research, design, prototyping assistance - [Julio Sueiras](https://github.com/juliosueiras) - particularly his [language server implementation](https://github.com/juliosueiras/terraform-lsp) - + +## Telemetry + +The server will collect data only if the _client_ requests it during initialization. Telemetry is opt-in by default. + +[Read more about telemetry](./docs/telemetry.md). ## `terraform-ls` VS `terraform-lsp` diff --git a/docs/language-clients.md b/docs/language-clients.md index f2c6f7fdc..0bca4a746 100644 --- a/docs/language-clients.md +++ b/docs/language-clients.md @@ -119,3 +119,7 @@ Clients are encouraged to implement custom commands in a command palette or similar functionality. See [./commands.md](./commands.md) for more. + +## Telemetry + +See [./telemetry.md](./telemetry.md). diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 000000000..c2d18838c --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,88 @@ +# Telemetry + +The language server is capable of sending telemetry using the native LSP `telemetry/event` method. +Telemetry is off by default and can enabled by passing a supported request format version +as an experimental client capability. + +```json +{ + "capabilities": { + "experimental": { + "telemetryVersion": 1 + } + } +} +``` + +Clients then implement opt-in or opt-out in the UI and should reflect the user's choice. + +## Privacy + +Sensitive data, such as filesystem paths or addresses of providers sourced from outside the Terraform Registry +are anonymized. Random UUID is generated in memory and tracked instead of a path or a private provider address. + +Mapping of such UUIDs is not persisted anywhere other than in memory during process lifetime. + +## Request Format + +The only supported version is currently `1`. Version negotiation allows the server +to introduce breaking changes to the format and have clients adopt gradually. + +### `v1` + +[`telemetry/event`](https://microsoft.github.io/language-server-protocol/specifications/specification-3-16/#telemetry_event) structure + +```json +{ + "v": 1, + "name": "eventName", + "properties": {} +} +``` + +`properties` may contain **any (valid) JSON types** +including arrays and arbitrarily nested objects. It is client's +reponsibility to serialize these properties when and if necessary. + +Example events: + +```json +{ + "v": 1, + "name": "initialize", + "properties": { + "experimentalCapabilities.referenceCountCodeLens": true, + "lsVersion": "0.23.0", + "options.commandPrefix": true, + "options.excludeModulePaths": false, + "options.experimentalFeatures.prefillRequiredFields": false, + "options.experimentalFeatures.validateOnSave": false, + "options.rootModulePaths": false, + "options.terraformExecPath": false, + "options.terraformExecTimeout": "", + "options.terraformLogFilePath": false, + "root_uri": "dir" + } +} +``` +```json +{ + "v": 1, + "name": "moduleData", + "properties": { + "backend": "remote", + "backend.remote.hostname": "app.terraform.io", + "installedProviders": { + "registry.terraform.io/hashicorp/aws": "3.57.0", + "registry.terraform.io/hashicorp/null": "3.1.0" + }, + "moduleId": "8aa5a4dc-4780-2d90-b8fb-57de8288fb32", + "providerRequirements": { + "registry.terraform.io/hashicorp/aws": "", + "registry.terraform.io/hashicorp/null": "~> 3.1" + }, + "tfRequirements": "~> 1.0", + "tfVersion": "1.0.7" + } +} +``` diff --git a/go.mod b/go.mod index 726b82d01..c9e4a36f4 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,14 @@ require ( github.com/hashicorp/go-getter v1.5.9 github.com/hashicorp/go-memdb v1.3.2 github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-uuid v1.0.1 github.com/hashicorp/go-version v1.3.0 github.com/hashicorp/hcl-lang v0.0.0-20211014152429-0bfbdcca0902 github.com/hashicorp/hcl/v2 v2.10.1 github.com/hashicorp/terraform-exec v0.15.0 github.com/hashicorp/terraform-json v0.13.0 github.com/hashicorp/terraform-registry-address v0.0.0-20210816115301-cb2034eba045 - github.com/hashicorp/terraform-schema v0.0.0-20210823185306-e7a9c4e84cd1 + github.com/hashicorp/terraform-schema v0.0.0-20211021151419-21dfff199031 github.com/kylelemons/godebug v1.1.0 // indirect github.com/mh-cbon/go-fmt-fail v0.0.0-20160815164508-67765b3fbcb5 github.com/mitchellh/cli v1.1.2 diff --git a/go.sum b/go.sum index 91b759635..5749ff58e 100644 --- a/go.sum +++ b/go.sum @@ -201,14 +201,13 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/terraform-exec v0.15.0 h1:cqjh4d8HYNQrDoEmlSGelHmg2DYDh5yayckvJ5bV18E= github.com/hashicorp/terraform-exec v0.15.0/go.mod h1:H4IG8ZxanU+NW0ZpDRNsvh9f0ul7C0nHP+rUR/CHs7I= -github.com/hashicorp/terraform-json v0.12.0/go.mod h1:pmbq9o4EuL43db5+0ogX10Yofv1nozM+wskr/bGFJpI= github.com/hashicorp/terraform-json v0.13.0 h1:Li9L+lKD1FO5RVFRM1mMMIBDoUHslOniyEi5CM+FWGY= github.com/hashicorp/terraform-json v0.13.0/go.mod h1:y5OdLBCT+rxbwnpxZs9kGL7R9ExU76+cpdY8zHwoazk= github.com/hashicorp/terraform-registry-address v0.0.0-20210412075316-9b2996cce896/go.mod h1:bzBPnUIkI0RxauU8Dqo+2KrZZ28Cf48s8V6IHt3p4co= github.com/hashicorp/terraform-registry-address v0.0.0-20210816115301-cb2034eba045 h1:R/I8ofvXuPcTNoc//N4ruvaHGZcShI/VuU2iXo875Lo= github.com/hashicorp/terraform-registry-address v0.0.0-20210816115301-cb2034eba045/go.mod h1:anRyJbe12BZscpFgaeGu9gH12qfdBP094LYFtuAFzd4= -github.com/hashicorp/terraform-schema v0.0.0-20210823185306-e7a9c4e84cd1 h1:K6wkKTi4+aSYXDFGbGWgd3sP+gWTTM9VYOVeCXjJJm8= -github.com/hashicorp/terraform-schema v0.0.0-20210823185306-e7a9c4e84cd1/go.mod h1:wG+IttAk2LqgHE76fD0wt2kucaKzV7pzm7OTqCJJC3M= +github.com/hashicorp/terraform-schema v0.0.0-20211021151419-21dfff199031 h1:HwQTGktZUBlRENcwb9MKm+cfqNcv0C5vagJnjKAqNKY= +github.com/hashicorp/terraform-schema v0.0.0-20211021151419-21dfff199031/go.mod h1:DlxWg9rEgltUs+FD5ElEgBoP985cjAeA9YHcYliAGVg= github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0= github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734/go.mod h1:kNDNcF7sN4DocDLBkQYz73HGKwN1ANB1blq4lIYLYvg= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= @@ -381,7 +380,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= -github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.9.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.9.1 h1:viqrgQwFl5UpSxc046qblj78wZXVDFnSOufaOTER+cc= diff --git a/internal/langserver/handlers/hooks_module.go b/internal/langserver/handlers/hooks_module.go new file mode 100644 index 000000000..f26c37e80 --- /dev/null +++ b/internal/langserver/handlers/hooks_module.go @@ -0,0 +1,87 @@ +package handlers + +import ( + "context" + "fmt" + + "github.com/hashicorp/terraform-ls/internal/state" + "github.com/hashicorp/terraform-ls/internal/telemetry" + "github.com/hashicorp/terraform-schema/backend" +) + +func sendModuleTelemetry(ctx context.Context, store *state.StateStore, telemetrySender telemetry.Sender) state.ModuleChangeHook { + return func(_, newMod *state.Module) { + modId, err := store.GetModuleID(newMod.Path) + if err != nil { + return + } + + properties := map[string]interface{}{ + "moduleId": modId, + } + + if len(newMod.Meta.CoreRequirements) > 0 { + properties["tfRequirements"] = newMod.Meta.CoreRequirements.String() + } + if newMod.Meta.Backend != nil { + properties["backend"] = newMod.Meta.Backend.Type + if data, ok := newMod.Meta.Backend.Data.(*backend.Remote); ok { + hostname := data.Hostname + + // anonymize any non-default hostnames + if hostname != "" && hostname != "app.terraform.io" { + hostname = "custom-hostname" + } + + properties["backend.remote.hostname"] = hostname + } + } + if len(newMod.Meta.ProviderRequirements) > 0 { + reqs := make(map[string]string, 0) + for pAddr, cons := range newMod.Meta.ProviderRequirements { + if telemetry.IsPublicProvider(pAddr) { + reqs[pAddr.String()] = cons.String() + continue + } + + // anonymize any unknown providers or the ones not publicly listed + id, err := store.GetProviderID(pAddr) + if err != nil { + continue + } + addr := fmt.Sprintf("unlisted/%s", id) + reqs[addr] = cons.String() + } + properties["providerRequirements"] = reqs + } + + if newMod.TerraformVersion != nil { + properties["tfVersion"] = newMod.TerraformVersion.String() + } + + if len(newMod.InstalledProviders) > 0 { + installedProviders := make(map[string]string, 0) + for pAddr, pv := range newMod.InstalledProviders { + if telemetry.IsPublicProvider(pAddr) { + versionString := "" + if pv != nil { + versionString = pv.String() + } + installedProviders[pAddr.String()] = versionString + continue + } + + // anonymize any unknown providers or the ones not publicly listed + id, err := store.GetProviderID(pAddr) + if err != nil { + continue + } + addr := fmt.Sprintf("unlisted/%s", id) + installedProviders[addr] = "" + } + properties["installedProviders"] = installedProviders + } + + telemetrySender.SendEvent(ctx, "moduleData", properties) + } +} diff --git a/internal/langserver/handlers/initialize.go b/internal/langserver/handlers/initialize.go index 8de7fe125..6f4ed908f 100644 --- a/internal/langserver/handlers/initialize.go +++ b/internal/langserver/handlers/initialize.go @@ -52,12 +52,41 @@ func (svc *service) Initialize(ctx context.Context, params lsp.InitializeParams) serverCaps.ServerInfo.Version = version } + clientCaps := params.Capabilities + + properties := map[string]interface{}{ + "experimentalCapabilities.referenceCountCodeLens": false, + "options.rootModulePaths": false, + "options.excludeModulePaths": false, + "options.commandPrefix": false, + "options.experimentalFeatures.validateOnSave": false, + "options.terraformExecPath": false, + "options.terraformExecTimeout": "", + "options.terraformLogFilePath": false, + "root_uri": "dir", + "lsVersion": serverCaps.ServerInfo.Version, + } + + expClientCaps := lsp.ExperimentalClientCapabilities(clientCaps.Experimental) + + if tv, ok := expClientCaps.TelemetryVersion(); ok { + svc.logger.Printf("enabling telemetry (version: %d)", tv) + err := svc.setupTelemetry(tv, jrpc2.ServerFromContext(ctx)) + if err != nil { + svc.logger.Printf("failed to setup telemetry: %s", err) + } + svc.logger.Printf("telemetry enabled (version: %d)", tv) + } + defer svc.telemetry.SendEvent(ctx, "initialize", properties) + fh := ilsp.FileHandlerFromDirURI(params.RootURI) if fh.URI() == "" || !fh.IsDir() { + properties["root_uri"] = "file" return serverCaps, fmt.Errorf("Editing a single file is not yet supported." + " Please open a directory.") } if !fh.Valid() { + properties["root_uri"] = "invalid" return serverCaps, fmt.Errorf("URI %q is not valid", params.RootURI) } @@ -74,12 +103,11 @@ func (svc *service) Initialize(ctx context.Context, params lsp.InitializeParams) } } - clientCaps := params.Capabilities - - if _, ok = lsp.ExperimentalClientCapabilities(clientCaps.Experimental).ShowReferencesCommandId(); ok { + if _, ok = expClientCaps.ShowReferencesCommandId(); ok { serverCaps.Capabilities.Experimental = lsp.ExperimentalServerCapabilities{ ReferenceCountCodeLens: true, } + properties["experimentalCapabilities.referenceCountCodeLens"] = true } err = lsctx.SetClientCapabilities(ctx, &clientCaps) @@ -91,6 +119,16 @@ func (svc *service) Initialize(ctx context.Context, params lsp.InitializeParams) if err != nil { return serverCaps, err } + + properties["options.rootModulePaths"] = len(out.Options.ModulePaths) > 0 + properties["options.excludeModulePaths"] = len(out.Options.ExcludeModulePaths) > 0 + properties["options.commandPrefix"] = len(out.Options.CommandPrefix) > 0 + properties["options.experimentalFeatures.prefillRequiredFields"] = out.Options.ExperimentalFeatures.PrefillRequiredFields + properties["options.experimentalFeatures.validateOnSave"] = out.Options.ExperimentalFeatures.ValidateOnSave + properties["options.terraformExecPath"] = len(out.Options.TerraformExecPath) > 0 + properties["options.terraformExecTimeout"] = out.Options.TerraformExecTimeout + properties["options.terraformLogFilePath"] = len(out.Options.TerraformLogFilePath) > 0 + err = out.Options.Validate() if err != nil { return serverCaps, err diff --git a/internal/langserver/handlers/service.go b/internal/langserver/handlers/service.go index e9ef163b6..3fc15d17d 100644 --- a/internal/langserver/handlers/service.go +++ b/internal/langserver/handlers/service.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/terraform-ls/internal/schemas" "github.com/hashicorp/terraform-ls/internal/settings" "github.com/hashicorp/terraform-ls/internal/state" + "github.com/hashicorp/terraform-ls/internal/telemetry" "github.com/hashicorp/terraform-ls/internal/terraform/discovery" "github.com/hashicorp/terraform-ls/internal/terraform/exec" "github.com/hashicorp/terraform-ls/internal/terraform/module" @@ -48,6 +49,7 @@ type service struct { tfDiscoFunc discovery.DiscoveryFunc tfExecFactory exec.ExecutorFactory tfExecOpts *exec.ExecutorOpts + telemetry telemetry.Sender additionalHandlers map[string]rpch.Func } @@ -70,6 +72,7 @@ func NewSession(srvCtx context.Context) session.Session { newWalker: module.NewWalker, tfDiscoFunc: d.LookPath, tfExecFactory: exec.NewExecutor, + telemetry: &telemetry.NoopSender{}, } } @@ -89,6 +92,7 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) { return nil, fmt.Errorf("Unable to prepare session: %w", err) } + svc.telemetry = &telemetry.NoopSender{Logger: svc.logger} svc.fs.SetLogger(svc.logger) lh := LogHandler(svc.logger) @@ -443,6 +447,9 @@ func (svc *service) configureSessionDependencies(cfgOpts *settings.Options) erro return err } store.SetLogger(svc.logger) + store.Modules.ChangeHooks = state.ModuleChangeHooks{ + sendModuleTelemetry(svc.sessCtx, store, svc.telemetry), + } err = schemas.PreloadSchemasToStore(store.ProviderSchemas) if err != nil { @@ -469,6 +476,16 @@ func (svc *service) configureSessionDependencies(cfgOpts *settings.Options) erro return nil } +func (svc *service) setupTelemetry(version int, notifier session.ClientNotifier) error { + t, err := telemetry.NewSender(version, notifier) + if err != nil { + return err + } + + svc.telemetry = t + return nil +} + func (svc *service) Finish(_ jrpc2.Assigner, status jrpc2.ServerStatus) { if status.Closed || status.Err != nil { svc.logger.Printf("session stopped unexpectedly (err: %v)", status.Err) diff --git a/internal/langserver/session/types.go b/internal/langserver/session/types.go index 7575676dc..194cab8e2 100644 --- a/internal/langserver/session/types.go +++ b/internal/langserver/session/types.go @@ -13,4 +13,8 @@ type Session interface { SetLogger(*log.Logger) } +type ClientNotifier interface { + Notify(ctx context.Context, method string, params interface{}) error +} + type SessionFactory func(context.Context) Session diff --git a/internal/protocol/experimental.go b/internal/protocol/experimental.go index f8a6a6024..b89410301 100644 --- a/internal/protocol/experimental.go +++ b/internal/protocol/experimental.go @@ -21,3 +21,15 @@ func (cc ExpClientCapabilities) ShowReferencesCommandId() (string, bool) { cmdId, ok := cc["showReferencesCommandId"].(string) return cmdId, ok } + +func (cc ExpClientCapabilities) TelemetryVersion() (int, bool) { + if cc == nil { + return 0, false + } + + // numbers are unmarshalled as float64 from JSON + // per https://pkg.go.dev/encoding/json#Unmarshal + v, ok := cc["telemetryVersion"].(float64) + + return int(v), ok +} diff --git a/internal/protocol/telemetry.go b/internal/protocol/telemetry.go new file mode 100644 index 000000000..10a925cb2 --- /dev/null +++ b/internal/protocol/telemetry.go @@ -0,0 +1,10 @@ +package protocol + +const TelemetryFormatVersion = 1 + +type TelemetryEvent struct { + Version int `json:"v"` + + Name string `json:"name"` + Properties map[string]interface{} `json:"properties"` +} diff --git a/internal/state/hooks.go b/internal/state/hooks.go new file mode 100644 index 000000000..2c6c03251 --- /dev/null +++ b/internal/state/hooks.go @@ -0,0 +1,11 @@ +package state + +type ModuleChangeHook func(oldMod, newMod *Module) + +type ModuleChangeHooks []ModuleChangeHook + +func (mh ModuleChangeHooks) notifyModuleChange(oldMod, newMod *Module) { + for _, h := range mh { + h(oldMod, newMod) + } +} diff --git a/internal/state/module.go b/internal/state/module.go index 284bf9853..11ad84bef 100644 --- a/internal/state/module.go +++ b/internal/state/module.go @@ -17,6 +17,7 @@ import ( type ModuleMetadata struct { CoreRequirements version.Constraints + Backend *tfmod.Backend ProviderReferences map[tfmod.ProviderRef]tfaddr.Provider ProviderRequirements map[tfaddr.Provider]version.Constraints Variables map[string]tfmod.Variable @@ -29,6 +30,13 @@ func (mm ModuleMetadata) Copy() ModuleMetadata { CoreRequirements: mm.CoreRequirements, } + if mm.Backend != nil { + newMm.Backend = &tfmod.Backend{ + Type: mm.Backend.Type, + Data: mm.Backend.Data.Copy(), + } + } + if mm.ProviderReferences != nil { newMm.ProviderReferences = make(map[tfmod.ProviderRef]tfaddr.Provider, len(mm.ProviderReferences)) for ref, provider := range mm.ProviderReferences { @@ -72,6 +80,8 @@ type Module struct { TerraformVersionErr error TerraformVersionState op.OpState + InstalledProviders map[tfaddr.Provider]*version.Version + ProviderSchemaErr error ProviderSchemaState op.OpState @@ -135,6 +145,14 @@ func (m *Module) Copy() *Module { MetaState: m.MetaState, } + if m.InstalledProviders != nil { + newMod.InstalledProviders = make(map[tfaddr.Provider]*version.Version, 0) + for addr, pv := range m.InstalledProviders { + // version.Version is practically immutable once parsed + newMod.InstalledProviders[addr] = pv + } + } + if m.ParsedModuleFiles != nil { newMod.ParsedModuleFiles = make(ast.ModFiles, len(m.ParsedModuleFiles)) for name, f := range m.ParsedModuleFiles { @@ -203,11 +221,16 @@ func (s *ModuleStore) Add(modPath string) error { } } - err = txn.Insert(s.tableName, newModule(modPath)) + mod := newModule(modPath) + err = txn.Insert(s.tableName, mod) if err != nil { return err } + txn.Defer(func() { + go s.ChangeHooks.notifyModuleChange(nil, mod) + }) + txn.Commit() return nil } @@ -216,7 +239,22 @@ func (s *ModuleStore) Remove(modPath string) error { txn := s.db.Txn(true) defer txn.Abort() - _, err := txn.DeleteAll(s.tableName, "id", modPath) + oldObj, err := txn.First(s.tableName, "id", modPath) + if err != nil { + return err + } + + if oldObj == nil { + // already removed + return nil + } + + txn.Defer(func() { + oldMod := oldObj.(*Module) + go s.ChangeHooks.notifyModuleChange(oldMod, nil) + }) + + _, err = txn.DeleteAll(s.tableName, "id", modPath) if err != nil { return err } @@ -315,6 +353,44 @@ func moduleCopyByPath(txn *memdb.Txn, path string) (*Module, error) { return mod.Copy(), nil } +func (s *ModuleStore) UpdateInstalledProviders(path string, pvs map[tfaddr.Provider]*version.Version) error { + txn := s.db.Txn(true) + defer txn.Abort() + + oldMod, err := moduleByPath(txn, path) + if err != nil { + return err + } + + mod := oldMod.Copy() + + // Providers may come from different sources (schema or version command) + // and we don't get their versions in both cases, so we make sure the existing + // versions are retained to get the most of both sources. + newProviders := make(map[tfaddr.Provider]*version.Version, 0) + for addr, pv := range pvs { + if pv == nil { + if v, ok := oldMod.InstalledProviders[addr]; ok && v != nil { + pv = v + } + } + newProviders[addr] = pv + } + mod.InstalledProviders = newProviders + + err = txn.Insert(s.tableName, mod) + if err != nil { + return err + } + + txn.Defer(func() { + go s.ChangeHooks.notifyModuleChange(oldMod, mod) + }) + + txn.Commit() + return nil +} + func (s *ModuleStore) List() ([]*Module, error) { txn := s.db.Txn(false) @@ -421,11 +497,12 @@ func (s *ModuleStore) FinishProviderSchemaLoading(path string, psErr error) erro }) defer txn.Abort() - mod, err := moduleCopyByPath(txn, path) + oldMod, err := moduleByPath(txn, path) if err != nil { return err } + mod := oldMod.Copy() mod.ProviderSchemaErr = psErr err = txn.Insert(s.tableName, mod) @@ -433,6 +510,10 @@ func (s *ModuleStore) FinishProviderSchemaLoading(path string, psErr error) erro return err } + txn.Defer(func() { + go s.ChangeHooks.notifyModuleChange(oldMod, mod) + }) + txn.Commit() return nil } @@ -444,11 +525,12 @@ func (s *ModuleStore) UpdateTerraformVersion(modPath string, tfVer *version.Vers }) defer txn.Abort() - mod, err := moduleCopyByPath(txn, modPath) + oldMod, err := moduleByPath(txn, modPath) if err != nil { return err } + mod := oldMod.Copy() mod.TerraformVersion = tfVer mod.TerraformVersionErr = vErr @@ -457,6 +539,10 @@ func (s *ModuleStore) UpdateTerraformVersion(modPath string, tfVer *version.Vers return err } + txn.Defer(func() { + go s.ChangeHooks.notifyModuleChange(oldMod, mod) + }) + err = updateProviderVersions(txn, modPath, pv) if err != nil { return err @@ -580,13 +666,15 @@ func (s *ModuleStore) UpdateMetadata(path string, meta *tfmod.Meta, mErr error) }) defer txn.Abort() - mod, err := moduleCopyByPath(txn, path) + oldMod, err := moduleByPath(txn, path) if err != nil { return err } + mod := oldMod.Copy() mod.Meta = ModuleMetadata{ CoreRequirements: meta.CoreRequirements, + Backend: meta.Backend, ProviderReferences: meta.ProviderReferences, ProviderRequirements: meta.ProviderRequirements, Variables: meta.Variables, @@ -599,6 +687,10 @@ func (s *ModuleStore) UpdateMetadata(path string, meta *tfmod.Meta, mErr error) return err } + txn.Defer(func() { + go s.ChangeHooks.notifyModuleChange(oldMod, mod) + }) + txn.Commit() return nil } diff --git a/internal/state/module_ids.go b/internal/state/module_ids.go new file mode 100644 index 000000000..099c0ed76 --- /dev/null +++ b/internal/state/module_ids.go @@ -0,0 +1,38 @@ +package state + +import "github.com/hashicorp/go-uuid" + +type ModuleIds struct { + Path string + ID string +} + +func (s *StateStore) GetModuleID(path string) (string, error) { + txn := s.db.Txn(true) + defer txn.Abort() + + obj, err := txn.First(moduleIdsTableName, "id", path) + if err != nil { + return "", err + } + + if obj != nil { + return obj.(ModuleIds).ID, nil + } + + newId, err := uuid.GenerateUUID() + if err != nil { + return "", err + } + + err = txn.Insert(moduleIdsTableName, ModuleIds{ + ID: newId, + Path: path, + }) + if err != nil { + return "", err + } + + txn.Commit() + return newId, nil +} diff --git a/internal/state/provider_ids.go b/internal/state/provider_ids.go new file mode 100644 index 000000000..63bb612bc --- /dev/null +++ b/internal/state/provider_ids.go @@ -0,0 +1,41 @@ +package state + +import ( + "github.com/hashicorp/go-uuid" + tfaddr "github.com/hashicorp/terraform-registry-address" +) + +type ProviderIds struct { + Address tfaddr.Provider + ID string +} + +func (s *StateStore) GetProviderID(addr tfaddr.Provider) (string, error) { + txn := s.db.Txn(true) + defer txn.Abort() + + obj, err := txn.First(providerIdsTableName, "id", addr) + if err != nil { + return "", err + } + + if obj != nil { + return obj.(ProviderIds).ID, nil + } + + newId, err := uuid.GenerateUUID() + if err != nil { + return "", err + } + + err = txn.Insert(providerIdsTableName, ProviderIds{ + ID: newId, + Address: addr, + }) + if err != nil { + return "", err + } + + txn.Commit() + return newId, nil +} diff --git a/internal/state/state.go b/internal/state/state.go index 8faff52da..199063eb1 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -13,7 +13,9 @@ import ( const ( moduleTableName = "module" + moduleIdsTableName = "module_ids" providerSchemaTableName = "provider_schema" + providerIdsTableName = "provider_ids" ) var dbSchema = &memdb.DBSchema{ @@ -45,18 +47,41 @@ var dbSchema = &memdb.DBSchema{ }, }, }, + providerIdsTableName: { + Name: providerIdsTableName, + Indexes: map[string]*memdb.IndexSchema{ + "id": { + Name: "id", + Unique: true, + Indexer: &memdb.StringFieldIndex{Field: "Address"}, + }, + }, + }, + moduleIdsTableName: { + Name: moduleIdsTableName, + Indexes: map[string]*memdb.IndexSchema{ + "id": { + Name: "id", + Unique: true, + Indexer: &memdb.StringFieldIndex{Field: "Path"}, + }, + }, + }, }, } type StateStore struct { Modules *ModuleStore ProviderSchemas *ProviderSchemaStore + + db *memdb.MemDB } type ModuleStore struct { - db *memdb.MemDB - tableName string - logger *log.Logger + db *memdb.MemDB + ChangeHooks ModuleChangeHooks + tableName string + logger *log.Logger } type ModuleReader interface { @@ -87,10 +112,12 @@ func NewStateStore() (*StateStore, error) { } return &StateStore{ + db: db, Modules: &ModuleStore{ - db: db, - tableName: moduleTableName, - logger: defaultLogger, + db: db, + ChangeHooks: make(ModuleChangeHooks, 0), + tableName: moduleTableName, + logger: defaultLogger, }, ProviderSchemas: &ProviderSchemaStore{ db: db, diff --git a/internal/telemetry/noop.go b/internal/telemetry/noop.go new file mode 100644 index 000000000..086926723 --- /dev/null +++ b/internal/telemetry/noop.go @@ -0,0 +1,22 @@ +package telemetry + +import ( + "context" + "io/ioutil" + "log" +) + +type NoopSender struct { + Logger *log.Logger +} + +func (t *NoopSender) log() *log.Logger { + if t.Logger != nil { + return t.Logger + } + return log.New(ioutil.Discard, "", 0) +} + +func (t *NoopSender) SendEvent(ctx context.Context, name string, properties map[string]interface{}) { + t.log().Printf("telemetry disabled %q: %#v", name, properties) +} diff --git a/internal/telemetry/sender.go b/internal/telemetry/sender.go new file mode 100644 index 000000000..e762a461b --- /dev/null +++ b/internal/telemetry/sender.go @@ -0,0 +1,7 @@ +package telemetry + +import "context" + +type Sender interface { + SendEvent(ctx context.Context, name string, properties map[string]interface{}) +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 000000000..cdaf4eac7 --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,47 @@ +package telemetry + +import ( + "context" + "fmt" + + lsp "github.com/hashicorp/terraform-ls/internal/protocol" + tfaddr "github.com/hashicorp/terraform-registry-address" +) + +type Telemetry struct { + version int + notifier Notifier +} + +type Notifier interface { + Notify(ctx context.Context, method string, params interface{}) error +} + +func NewSender(version int, notifier Notifier) (*Telemetry, error) { + if version != lsp.TelemetryFormatVersion { + return nil, fmt.Errorf("unsupported telemetry format version: %d", version) + } + + return &Telemetry{ + version: version, + notifier: notifier, + }, nil +} + +func (t *Telemetry) SendEvent(ctx context.Context, name string, properties map[string]interface{}) { + t.notifier.Notify(ctx, "telemetry/event", lsp.TelemetryEvent{ + Version: t.version, + Name: name, + Properties: properties, + }) +} + +func IsPublicProvider(addr tfaddr.Provider) bool { + if addr.Hostname == tfaddr.DefaultRegistryHost { + return true + } + if addr.IsDefault() || addr.IsLegacy() || addr.IsBuiltIn() { + return true + } + return false +} diff --git a/internal/terraform/module/module_ops.go b/internal/terraform/module/module_ops.go index 90a3cc25f..a1ce39d1b 100644 --- a/internal/terraform/module/module_ops.go +++ b/internal/terraform/module/module_ops.go @@ -69,9 +69,15 @@ func GetTerraformVersion(ctx context.Context, modStore *state.ModuleStore, modPa pVersions := providerVersions(pv) sErr := modStore.UpdateTerraformVersion(modPath, v, pVersions, err) - if err != nil { + if sErr != nil { return sErr } + + ipErr := modStore.UpdateInstalledProviders(modPath, pVersions) + if ipErr != nil { + return ipErr + } + return err } @@ -117,12 +123,17 @@ func ObtainSchema(ctx context.Context, modStore *state.ModuleStore, schemaStore return err } + installedProviders := make(map[tfaddr.Provider]*version.Version, 0) + for rawAddr, pJsonSchema := range ps.Schemas { pAddr, err := tfaddr.ParseRawProviderSourceString(rawAddr) if err != nil { // skip unparsable address continue } + + installedProviders[pAddr] = nil + if pAddr.IsLegacy() { // TODO: check for migrations via Registry API? } @@ -135,7 +146,7 @@ func ObtainSchema(ctx context.Context, modStore *state.ModuleStore, schemaStore } } - return nil + return modStore.UpdateInstalledProviders(modPath, installedProviders) } func ParseModuleConfiguration(fs filesystem.Filesystem, modStore *state.ModuleStore, modPath string) error {