refactoring(parser): Pass around tokens instead of blocks

internal/hcl/hcl.go

@@ -1,19 +1,27 @@
 package hcl
 
 import (
+	"fmt"
+
+	"github.com/hashicorp/hcl/v2"
 	hcllib "github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/hashicorp/terraform-ls/internal/filesystem"
 )
 
 type File interface {
-	BlockAtPosition(filesystem.FilePosition) (*hcllib.Block, hcllib.Pos, error)
+	BlockTokensAtPosition(filesystem.FilePosition) (hclsyntax.Tokens, hcllib.Pos, error)
 }
 
 type file struct {
 	filename string
 	content  []byte
-	f        *hcllib.File
+	pf       *parsedFile
+}
+
+type parsedFile struct {
+	Body   hcllib.Body
+	Tokens hclsyntax.Tokens
 }
 
 func NewFile(f filesystem.File) File {
@@ -23,51 +31,116 @@ func NewFile(f filesystem.File) File {
 	}
 }
 
-func (f *file) ast() (*hcllib.File, error) {
-	if f.f != nil {
-		return f.f, nil
+func (f *file) parse() (*parsedFile, error) {
+	if f.pf != nil {
+		return f.pf, nil
+	}
+
+	var parseDiags hcllib.Diagnostics
+
+	tokens, diags := hclsyntax.LexConfig(f.content, f.filename, hcllib.InitialPos)
+	if diags.HasErrors() {
+		parseDiags = append(parseDiags, diags...)
 	}
 
-	hf, err := hclsyntax.ParseConfig(f.content, f.filename, hcllib.InitialPos)
-	f.f = hf
+	body, diags := hclsyntax.ParseBodyFromTokens(tokens, hclsyntax.TokenEOF)
+	if diags.HasErrors() {
+		parseDiags = append(parseDiags, diags...)
+	}
 
-	return f.f, err
+	f.pf = &parsedFile{
+		Tokens: tokens,
+		Body:   body,
+	}
+
+	if parseDiags.HasErrors() {
+		return f.pf, parseDiags
+	}
+	return f.pf, nil
 }
 
-func (f *file) BlockAtPosition(filePos filesystem.FilePosition) (*hcllib.Block, hcllib.Pos, error) {
+func (f *file) BlockTokensAtPosition(filePos filesystem.FilePosition) (hclsyntax.Tokens, hcllib.Pos, error) {
 	pos := filePos.Position()
 
 	b, err := f.blockAtPosition(pos)
 	if err != nil {
-		return nil, pos, err
+		return hclsyntax.Tokens{}, pos, err
 	}
 
 	return b, pos, nil
 }
 
-func (f *file) blockAtPosition(pos hcllib.Pos) (*hcllib.Block, error) {
-	ast, _ := f.ast()
+func (f *file) blockAtPosition(pos hcllib.Pos) (hclsyntax.Tokens, error) {
+	pf, _ := f.parse()
 
-	if body, ok := ast.Body.(*hclsyntax.Body); ok {
-		if body.SrcRange.Empty() && pos != hcllib.InitialPos {
-			return nil, &InvalidHclPosErr{pos, body.SrcRange}
+	body, ok := pf.Body.(*hclsyntax.Body)
+	if !ok {
+		return hclsyntax.Tokens{}, fmt.Errorf("unexpected body type (%T)", body)
+	}
+	if body.SrcRange.Empty() && pos != hcllib.InitialPos {
+		return hclsyntax.Tokens{}, &InvalidHclPosErr{pos, body.SrcRange}
+	}
+	if !body.SrcRange.Empty() {
+		if posIsEqual(body.SrcRange.End, pos) {
+			return hclsyntax.Tokens{}, &NoBlockFoundErr{pos}
 		}
-		if !body.SrcRange.Empty() {
-			if posIsEqual(body.SrcRange.End, pos) {
-				return nil, &NoBlockFoundErr{pos}
-			}
-			if !body.SrcRange.ContainsPos(pos) {
-				return nil, &InvalidHclPosErr{pos, body.SrcRange}
-			}
+		if !body.SrcRange.ContainsPos(pos) {
+			return hclsyntax.Tokens{}, &InvalidHclPosErr{pos, body.SrcRange}
 		}
 	}
 
-	block := ast.OutermostBlockAtPos(pos)
-	if block == nil {
-		return nil, &NoBlockFoundErr{pos}
+	for _, block := range body.Blocks {
+		wholeRange := hcllib.RangeBetween(block.TypeRange, block.CloseBraceRange)
+		if wholeRange.ContainsPos(pos) {
+			return tokensInRange(f.pf.Tokens, block.Range()), nil
+		}
+	}
+
+	return nil, &NoBlockFoundErr{pos}
+}
+
+func tokensInRange(tokens hclsyntax.Tokens, rng hcllib.Range) hclsyntax.Tokens {
+	var ts hclsyntax.Tokens
+
+	for _, t := range tokens {
+		// We could check more strictly if the whole token is in given range
+		// but we just assume block always contains whole tokens
+		if rng.Overlaps(t.Range) {
+			ts = append(ts, t)
+		}
+	}
+
+	if len(ts) > 0 {
+		lastToken := ts[len(ts)-1]
+		if lastToken.Type != hclsyntax.TokenEOF &&
+			lastToken.Type != hclsyntax.TokenNewline {
+			tRng := lastToken.Range
+
+			// Parser expects some form of closing token, so we attach a newline
+			trailingNewLine := hclsyntax.Token{
+				Type:  hclsyntax.TokenNewline,
+				Bytes: []byte("\n"),
+				Range: hcl.Range{
+					Filename: tRng.Filename,
+					Start:    tRng.End,
+					End: hcl.Pos{
+						Byte:   tRng.End.Byte + 1,
+						Column: 1,
+						Line:   tRng.End.Line + 1,
+					},
+				},
+			}
+
+			ts = append(ts, trailingNewLine)
+		}
+
+		if ts[0].Type == hclsyntax.TokenNewline {
+			// Remove initial newline
+			return ts[1:]
+		}
 	}
 
-	return block, nil
+	return ts
 }
 
 func posIsEqual(a, b hcllib.Pos) bool {

internal/hcl/hcl_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/hashicorp/terraform-ls/internal/filesystem"
 )
 
@@ -17,8 +18,8 @@ func TestFile_BlockAtPosition(t *testing.T) {
 		content string
 		pos     hcl.Pos
 
-		expectedErr   error
-		expectedBlock *hcl.Block
+		expectedErr    error
+		expectedTokens []hclsyntax.Token
 	}{
 		{
 			"invalid config",
@@ -29,9 +30,31 @@ func TestFile_BlockAtPosition(t *testing.T) {
 				Byte:   0,
 			},
 			nil, // Expect errors to be ignored
-			&hcl.Block{
-				Type:   "provider",
-				Labels: []string{"aws"},
+			[]hclsyntax.Token{
+				{
+					Type:  hclsyntax.TokenIdent,
+					Bytes: []byte("provider"),
+				},
+				{
+					Type:  hclsyntax.TokenOQuote,
+					Bytes: []byte(`"`),
+				},
+				{
+					Type:  hclsyntax.TokenQuotedLit,
+					Bytes: []byte("aws"),
+				},
+				{
+					Type:  hclsyntax.TokenCQuote,
+					Bytes: []byte(`"`),
+				},
+				{
+					Type:  hclsyntax.TokenOBrace,
+					Bytes: []byte("{"),
+				},
+				{
+					Type:  hclsyntax.TokenNewline,
+					Bytes: []byte("\n"),
+				},
 			},
 		},
 		{
@@ -46,9 +69,43 @@ func TestFile_BlockAtPosition(t *testing.T) {
 				Byte:   17,
 			},
 			nil,
-			&hcl.Block{
-				Type:   "provider",
-				Labels: []string{"aws"},
+			[]hclsyntax.Token{
+				{
+					Type:  hclsyntax.TokenIdent,
+					Bytes: []byte("provider"),
+				},
+				{
+					Type:  hclsyntax.TokenOQuote,
+					Bytes: []byte(`"`),
+				},
+				{
+					Type:  hclsyntax.TokenQuotedLit,
+					Bytes: []byte("aws"),
+				},
+				{
+					Type:  hclsyntax.TokenCQuote,
+					Bytes: []byte(`"`),
+				},
+				{
+					Type:  hclsyntax.TokenOBrace,
+					Bytes: []byte("{"),
+				},
+				{
+					Type:  hclsyntax.TokenNewline,
+					Bytes: []byte("\n"),
+				},
+				{
+					Type:  hclsyntax.TokenNewline,
+					Bytes: []byte("\n"),
+				},
+				{
+					Type:  hclsyntax.TokenCBrace,
+					Bytes: []byte("}"),
+				},
+				{
+					Type:  hclsyntax.TokenNewline,
+					Bytes: []byte("\n"),
+				},
 			},
 		},
 		{
@@ -128,8 +185,8 @@ func TestFile_BlockAtPosition(t *testing.T) {
 	}
 
 	opts := cmp.Options{
-		cmpopts.IgnoreFields(hcl.Block{},
-			"Body", "DefRange", "TypeRange", "LabelRanges"),
+		cmpopts.IgnoreFields(hclsyntax.Token{},
+			"Range"),
 		cmpopts.IgnoreFields(hcl.Diagnostic{}, "Subject"),
 	}
 
@@ -142,7 +199,7 @@ func TestFile_BlockAtPosition(t *testing.T) {
 				pos:         tc.pos,
 			}
 
-			block, _, err := f.BlockAtPosition(fp)
+			tokens, _, err := f.BlockTokensAtPosition(fp)
 			if err != nil {
 				if tc.expectedErr == nil {
 					t.Fatal(err)
@@ -156,8 +213,8 @@ func TestFile_BlockAtPosition(t *testing.T) {
 				t.Fatalf("Expected error: %s", tc.expectedErr)
 			}
 
-			if diff := cmp.Diff(tc.expectedBlock, block, opts...); diff != "" {
-				t.Fatalf("Unexpected block difference: %s", diff)
+			if diff := cmp.Diff(hclsyntax.Tokens(tc.expectedTokens), tokens, opts...); diff != "" {
+				t.Fatalf("Unexpected token difference: %s", diff)
 			}
 
 		})

internal/terraform/lang/config_block.go

@@ -11,7 +11,7 @@ import (
 )
 
 type configBlockFactory interface {
-	New(*hclsyntax.Block) (ConfigBlock, error)
+	New(hclsyntax.Tokens) (ConfigBlock, error)
 	LabelSchema() LabelSchema
 	Documentation() MarkupContent
 }

internal/terraform/lang/config_block_test.go

@@ -256,14 +256,15 @@ func TestCompletableBlock_CompletionCandidatesAtPos(t *testing.T) {
 
 	for i, tc := range testCases {
 		t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) {
-			block, err := AsHCLSyntaxBlock(parseHclBlock(t, tc.src))
+			tokens := lexConfig(t, tc.src)
+			block, err := ParseBlock(tokens, []*ParsedLabel{}, tc.sb)
 			if err != nil {
 				t.Fatal(err)
 			}
 
 			cb := &completableBlock{
 				logger: testLogger(),
-				block:  ParseBlock(block, []*ParsedLabel{}, tc.sb),
+				block:  block,
 			}
 
 			list, err := cb.completionCandidatesAtPos(tc.pos)

internal/terraform/lang/datasource_block.go

@@ -16,7 +16,7 @@ type datasourceBlockFactory struct {
 	schemaReader schema.Reader
 }
 
-func (f *datasourceBlockFactory) New(block *hclsyntax.Block) (ConfigBlock, error) {
+func (f *datasourceBlockFactory) New(tokens hclsyntax.Tokens) (ConfigBlock, error) {
 	if f.logger == nil {
 		f.logger = discardLog()
 	}
@@ -25,7 +25,7 @@ func (f *datasourceBlockFactory) New(block *hclsyntax.Block) (ConfigBlock, error
 		logger: f.logger,
 
 		labelSchema: f.LabelSchema(),
-		hclBlock:    block,
+		tokens:      tokens,
 		sr:          f.schemaReader,
 	}, nil
 }
@@ -52,7 +52,7 @@ type datasourceBlock struct {
 
 	labelSchema LabelSchema
 	labels      []*ParsedLabel
-	hclBlock    *hclsyntax.Block
+	tokens      hclsyntax.Tokens
 	sr          schema.Reader
 }
 
@@ -81,7 +81,9 @@ func (r *datasourceBlock) Labels() []*ParsedLabel {
 	if r.labels != nil {
 		return r.labels
 	}
-	r.labels = parseLabels(r.BlockType(), r.labelSchema, r.hclBlock.Labels)
+	labels, _ := ParseLabels(r.tokens, r.labelSchema)
+	r.labels = labels
+
 	return r.labels
 }
 
@@ -102,7 +104,10 @@ func (r *datasourceBlock) CompletionCandidatesAtPos(pos hcl.Pos) (CompletionCand
 		}
 		schemaBlock = rSchema.Block
 	}
-	block := ParseBlock(r.hclBlock, r.Labels(), schemaBlock)
+	block, err := ParseBlock(r.tokens, r.Labels(), schemaBlock)
+	if err != nil {
+		return nil, err
+	}
 
 	if block.PosInLabels(pos) {
 		dataSources, err := r.sr.DataSources()