Implement textDocument/semanticTokens (semantic highlighting)

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/google/uuid v1.1.2
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/go-version v1.2.1
-	github.com/hashicorp/hcl-lang v0.0.0-20201116081236-948e43712a65
+	github.com/hashicorp/hcl-lang v0.0.0-20201209145723-0c4061e492db
 	github.com/hashicorp/hcl/v2 v2.6.0
 	github.com/hashicorp/terraform-exec v0.11.1-0.20201207223938-9186a7c3bb24
 	github.com/hashicorp/terraform-json v0.7.0

go.sum

@@ -185,10 +185,9 @@ github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f h1:UdxlrJz4JOnY8W+Db
 github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/hcl-lang v0.0.0-20201110071249-4e412924f52b h1:EjnMRaTQlomBMNRQfyWoLEg9IdqxeN1R2mb3ZZetCBs=
 github.com/hashicorp/hcl-lang v0.0.0-20201110071249-4e412924f52b/go.mod h1:vd3BPEDWrYMAgAnB0MRlBdZknrpUXf8Jk2PNaHIbwhg=
-github.com/hashicorp/hcl-lang v0.0.0-20201116081236-948e43712a65 h1:kF6Dxt2kPNj8+Px7LyK7nxPDQjYKwGrKxxYnSu+LOXM=
-github.com/hashicorp/hcl-lang v0.0.0-20201116081236-948e43712a65/go.mod h1:vd3BPEDWrYMAgAnB0MRlBdZknrpUXf8Jk2PNaHIbwhg=
+github.com/hashicorp/hcl-lang v0.0.0-20201209145723-0c4061e492db h1:Euxzz3x8BlYyNKiENK5LdOjA3i9C0UiqALp3TnWbkck=
+github.com/hashicorp/hcl-lang v0.0.0-20201209145723-0c4061e492db/go.mod h1:TZ5tpvmgJSHfmIndN4WP9SpZvyWK8tHPBY8LDRyU+pI=
 github.com/hashicorp/hcl/v2 v2.0.0/go.mod h1:oVVDG71tEinNGYCxinCYadcmKU9bglqW9pV3txagJ90=
 github.com/hashicorp/hcl/v2 v2.6.0 h1:3krZOfGY6SziUXa6H9PJU6TyohHn7I+ARYnhbeNBz+o=
 github.com/hashicorp/hcl/v2 v2.6.0/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY=
@@ -202,8 +201,6 @@ github.com/hashicorp/terraform-exec v0.11.1-0.20201207223938-9186a7c3bb24 h1:kMl
 github.com/hashicorp/terraform-exec v0.11.1-0.20201207223938-9186a7c3bb24/go.mod h1:o1PZgrOpDMiDeKT0Hcr2oo6KoVKyaeTN867jcaXMg4E=
 github.com/hashicorp/terraform-json v0.7.0 h1:DgkfLARKMQ/xmzVtSRX9Vz/fzPCL3vskHIgj6s+SQwQ=
 github.com/hashicorp/terraform-json v0.7.0/go.mod h1:3defM4kkMfttwiE7VakJDwCd4R+umhSQnvJwORXbprE=
-github.com/hashicorp/terraform-schema v0.0.0-20201208004742-b5e321a36f41 h1:FvVpjaQJXT9AH70Vs9i90QDTLz92BsL69N9kaLGK8qE=
-github.com/hashicorp/terraform-schema v0.0.0-20201208004742-b5e321a36f41/go.mod h1:eRHMO4QL4TTka07aC7fH+AXvi/tYlv6udrA8nSFOl6g=
 github.com/hashicorp/terraform-schema v0.0.0-20201208163444-44d0347ab290 h1:kAs5ZG+cgtWy3+81Z7G/Blj2imiDLfFBRaqmNs8mD4o=
 github.com/hashicorp/terraform-schema v0.0.0-20201208163444-44d0347ab290/go.mod h1:eRHMO4QL4TTka07aC7fH+AXvi/tYlv6udrA8nSFOl6g=
 github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 h1:HKLsbzeOsfXmKNpr3GiT18XAblV0BjCbzL8KQAMZGa0=

internal/langserver/handlers/handlers_test.go

@@ -47,6 +47,13 @@ func initializeResponse(t *testing.T, commandPrefix string) string {
 					"commands": %s,
 					"workDoneProgress":true
 				},
+				"semanticTokensProvider": {
+					"legend": {
+						"tokenTypes": [],
+						"tokenModifiers": []
+					},
+					"full": false
+				},
 				"workspace": {
 					"workspaceFolders": {}
 				}

internal/langserver/handlers/initialize.go

@@ -51,7 +51,8 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam
 		return serverCaps, err
 	}
 
-	err = lsctx.SetClientCapabilities(ctx, &params.Capabilities)
+	clientCaps := params.Capabilities
+	err = lsctx.SetClientCapabilities(ctx, &clientCaps)
 	if err != nil {
 		return serverCaps, err
 	}
@@ -80,6 +81,20 @@ func (lh *logHandler) Initialize(ctx context.Context, params lsp.InitializeParam
 		return serverCaps, err
 	}
 
+	stCaps := clientCaps.TextDocument.SemanticTokens
+	caps := ilsp.SemanticTokensClientCapabilities{
+		SemanticTokensClientCapabilities: clientCaps.TextDocument.SemanticTokens,
+	}
+	semanticTokensOpts := lsp.SemanticTokensOptions{
+		Legend: lsp.SemanticTokensLegend{
+			TokenTypes:     ilsp.TokenTypesLegend(stCaps.TokenTypes).AsStrings(),
+			TokenModifiers: ilsp.TokenModifiersLegend(stCaps.TokenModifiers).AsStrings(),
+		},
+		Full: caps.FullRequest(),
+	}
+
+	serverCaps.Capabilities.SemanticTokensProvider = semanticTokensOpts
+
 	// set commandPrefix for session
 	lsctx.SetCommandPrefix(ctx, out.Options.CommandPrefix)
 	// apply prefix to executeCommand handler names

internal/langserver/handlers/semantic_tokens.go

@@ -0,0 +1,75 @@
+package handlers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/creachadair/jrpc2/code"
+	lsctx "github.com/hashicorp/terraform-ls/internal/context"
+	ilsp "github.com/hashicorp/terraform-ls/internal/lsp"
+	lsp "github.com/hashicorp/terraform-ls/internal/protocol"
+)
+
+func (lh *logHandler) TextDocumentSemanticTokensFull(ctx context.Context, params lsp.SemanticTokensParams) (lsp.SemanticTokens, error) {
+	tks := lsp.SemanticTokens{}
+
+	cc, err := lsctx.ClientCapabilities(ctx)
+	if err != nil {
+		return tks, err
+	}
+
+	caps := ilsp.SemanticTokensClientCapabilities{
+		SemanticTokensClientCapabilities: cc.TextDocument.SemanticTokens,
+	}
+	if !caps.FullRequest() {
+		// This would indicate a buggy client which sent a request
+		// it didn't claim to support, so we just strictly follow
+		// the protocol here and avoid serving buggy clients.
+		return tks, code.MethodNotFound.Err()
+	}
+
+	ds, err := lsctx.DocumentStorage(ctx)
+	if err != nil {
+		return tks, err
+	}
+
+	rmf, err := lsctx.RootModuleFinder(ctx)
+	if err != nil {
+		return tks, err
+	}
+
+	fh := ilsp.FileHandlerFromDocumentURI(params.TextDocument.URI)
+	doc, err := ds.GetDocument(fh)
+	if err != nil {
+		return tks, err
+	}
+
+	rm, err := rmf.RootModuleByPath(doc.Dir())
+	if err != nil {
+		return tks, fmt.Errorf("finding compatible decoder failed: %w", err)
+	}
+
+	schema, err := rmf.SchemaForPath(doc.Dir())
+	if err != nil {
+		return tks, err
+	}
+
+	d, err := rm.DecoderWithSchema(schema)
+	if err != nil {
+		return tks, err
+	}
+
+	tokens, err := d.SemanticTokensInFile(doc.Filename())
+	if err != nil {
+		return tks, err
+	}
+
+	te := &ilsp.TokenEncoder{
+		Lines:      doc.Lines(),
+		Tokens:     tokens,
+		ClientCaps: cc.TextDocument.SemanticTokens,
+	}
+	tks.Data = te.Encode()
+
+	return tks, nil
+}

internal/langserver/handlers/semantic_tokens_test.go

@@ -0,0 +1,121 @@
+package handlers
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/go-version"
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/hashicorp/terraform-ls/internal/langserver"
+	"github.com/hashicorp/terraform-ls/internal/terraform/exec"
+	"github.com/hashicorp/terraform-ls/internal/terraform/rootmodule"
+	"github.com/stretchr/testify/mock"
+)
+
+func TestSemanticTokensFull(t *testing.T) {
+	tmpDir := TempDir(t)
+	InitPluginCache(t, tmpDir.Dir())
+
+	var testSchema tfjson.ProviderSchemas
+	err := json.Unmarshal([]byte(testSchemaOutput), &testSchema)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ls := langserver.NewLangServerMock(t, NewMockSession(&MockSessionInput{
+		RootModules: map[string]*rootmodule.RootModuleMock{
+			tmpDir.Dir(): {
+				TfExecFactory: exec.NewMockExecutor([]*mock.Call{
+					{
+						Method:        "Version",
+						Repeatability: 1,
+						Arguments: []interface{}{
+							mock.AnythingOfType(""),
+						},
+						ReturnArguments: []interface{}{
+							version.Must(version.NewVersion("0.12.0")),
+							nil,
+							nil,
+						},
+					},
+					{
+						Method:        "GetExecPath",
+						Repeatability: 1,
+						ReturnArguments: []interface{}{
+							"",
+						},
+					},
+					{
+						Method:        "ProviderSchemas",
+						Repeatability: 1,
+						Arguments: []interface{}{
+							mock.AnythingOfType(""),
+						},
+						ReturnArguments: []interface{}{
+							&testSchema,
+							nil,
+						},
+					},
+				}),
+			},
+		}}))
+	stop := ls.Start(t)
+	defer stop()
+
+	ls.Call(t, &langserver.CallRequest{
+		Method: "initialize",
+		ReqParams: fmt.Sprintf(`{
+		"capabilities": {
+			"textDocument": {
+				"semanticTokens": {
+					"tokenTypes": [
+						"type",
+						"property",
+						"string"
+					],
+					"tokenModifiers": [
+						"deprecated",
+						"modification"
+					],
+					"requests": {
+						"full": true
+					}
+				}
+			}
+		},
+		"rootUri": %q,
+		"processId": 12345
+	}`, TempDir(t).URI())})
+	ls.Notify(t, &langserver.CallRequest{
+		Method:    "initialized",
+		ReqParams: "{}",
+	})
+	ls.Call(t, &langserver.CallRequest{
+		Method: "textDocument/didOpen",
+		ReqParams: fmt.Sprintf(`{
+		"textDocument": {
+			"version": 0,
+			"languageId": "terraform",
+			"text": "provider \"test\" {\n\n}\n",
+			"uri": "%s/main.tf"
+		}
+	}`, TempDir(t).URI())})
+
+	ls.CallAndExpectResponse(t, &langserver.CallRequest{
+		Method: "textDocument/semanticTokens/full",
+		ReqParams: fmt.Sprintf(`{
+			"textDocument": {
+				"uri": "%s/main.tf"
+			}
+		}`, TempDir(t).URI())}, `{
+			"jsonrpc": "2.0",
+			"id": 3,
+			"result": {
+				"data": [
+					0,0,8,0,0,
+					0,9,6,1,2
+				]
+			}
+		}`)
+}

internal/langserver/handlers/service.go

@@ -260,6 +260,18 @@ func (svc *service) Assigner() (jrpc2.Assigner, error) {
 
 			return handle(ctx, req, lh.TextDocumentFormatting)
 		},
+		"textDocument/semanticTokens/full": func(ctx context.Context, req *jrpc2.Request) (interface{}, error) {
+			err := session.CheckInitializationIsConfirmed()
+			if err != nil {
+				return nil, err
+			}
+
+			ctx = lsctx.WithDocumentStorage(ctx, svc.fs)
+			ctx = lsctx.WithClientCapabilities(ctx, cc)
+			ctx = lsctx.WithRootModuleFinder(ctx, svc.modMgr)
+
+			return handle(ctx, req, lh.TextDocumentSemanticTokensFull)
+		},
 		"workspace/executeCommand": func(ctx context.Context, req *jrpc2.Request) (interface{}, error) {
 			err := session.CheckInitializationIsConfirmed()
 			if err != nil {

internal/lsp/semantic_tokens.go

@@ -0,0 +1,23 @@
+package lsp
+
+import (
+	lsp "github.com/hashicorp/terraform-ls/internal/protocol"
+)
+
+type semanticTokensFull struct {
+	Delta bool `json:"delta,omitempty"`
+}
+
+type SemanticTokensClientCapabilities struct {
+	lsp.SemanticTokensClientCapabilities
+}
+
+func (c SemanticTokensClientCapabilities) FullRequest() bool {
+	switch full := c.Requests.Full.(type) {
+	case bool:
+		return full
+	case semanticTokensFull:
+		return true
+	}
+	return false
+}