refactoring(parser): Pass around tokens instead of blocks

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/google/go-cmp v0.4.0
 	github.com/hashicorp/go-version v1.2.0
-	github.com/hashicorp/hcl/v2 v2.3.0
+	github.com/hashicorp/hcl/v2 v2.5.2-0.20200528183353-fa7c453538de
 	github.com/hashicorp/terraform-json v0.5.0
 	github.com/mitchellh/cli v1.0.0
 	github.com/pmezard/go-difflib v1.0.0

go.sum

@@ -7,6 +7,8 @@ github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki
 github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
 github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
+github.com/apparentlymart/go-textseg/v12 v12.0.0 h1:bNEQyAGak9tojivJNkoqWErVCQbjdL7GzRt3F8NvfJ0=
+github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
@@ -32,8 +34,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/hcl/v2 v2.3.0 h1:iRly8YaMwTBAKhn1Ybk7VSdzbnopghktCD031P8ggUE=
-github.com/hashicorp/hcl/v2 v2.3.0/go.mod h1:d+FwDBbOLvpAM3Z6J7gPj/VoAGkNe/gm352ZhjJ/Zv8=
+github.com/hashicorp/hcl/v2 v2.5.2-0.20200528183353-fa7c453538de h1:bCeWhTigOmP9am0cJA+5kaTtA2RFDmnWIRtBIxo+Ydg=
+github.com/hashicorp/hcl/v2 v2.5.2-0.20200528183353-fa7c453538de/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY=
 github.com/hashicorp/terraform-json v0.5.0 h1:7TV3/F3y7QVSuN4r9BEXqnWqrAyeOtON8f0wvREtyzs=
 github.com/hashicorp/terraform-json v0.5.0/go.mod h1:eAbqb4w0pSlRmdvl8fOyHAi/+8jnkVYN28gJkSJrLhU=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=

internal/hcl/hcl.go

@@ -1,19 +1,27 @@
 package hcl
 
 import (
+	"fmt"
+
+	"github.com/hashicorp/hcl/v2"
 	hcllib "github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/hashicorp/terraform-ls/internal/filesystem"
 )
 
 type File interface {
-	BlockAtPosition(filesystem.FilePosition) (*hcllib.Block, hcllib.Pos, error)
+	BlockTokensAtPosition(filesystem.FilePosition) (hclsyntax.Tokens, hcllib.Pos, error)
 }
 
 type file struct {
 	filename string
 	content  []byte
-	f        *hcllib.File
+	pf       *parsedFile
+}
+
+type parsedFile struct {
+	Body   hcllib.Body
+	Tokens hclsyntax.Tokens
 }
 
 func NewFile(f filesystem.File) File {
@@ -23,51 +31,139 @@ func NewFile(f filesystem.File) File {
 	}
 }
 
-func (f *file) ast() (*hcllib.File, error) {
-	if f.f != nil {
-		return f.f, nil
+func (f *file) parse() (*parsedFile, error) {
+	if f.pf != nil {
+		return f.pf, nil
 	}
 
-	hf, err := hclsyntax.ParseConfig(f.content, f.filename, hcllib.InitialPos)
-	f.f = hf
+	var parseDiags hcllib.Diagnostics
+
+	tokens, diags := hclsyntax.LexConfig(f.content, f.filename, hcllib.InitialPos)
+	if diags.HasErrors() {
+		parseDiags = append(parseDiags, diags...)
+	}
+
+	body, diags := hclsyntax.ParseBodyFromTokens(tokens, hclsyntax.TokenEOF)
+	if diags.HasErrors() {
+		parseDiags = append(parseDiags, diags...)
+	}
+
+	f.pf = &parsedFile{
+		Tokens: tokens,
+		Body:   body,
+	}
 
-	return f.f, err
+	if parseDiags.HasErrors() {
+		return f.pf, parseDiags
+	}
+	return f.pf, nil
 }
 
-func (f *file) BlockAtPosition(filePos filesystem.FilePosition) (*hcllib.Block, hcllib.Pos, error) {
+func (f *file) BlockTokensAtPosition(filePos filesystem.FilePosition) (hclsyntax.Tokens, hcllib.Pos, error) {
 	pos := filePos.Position()
 
 	b, err := f.blockAtPosition(pos)
 	if err != nil {
-		return nil, pos, err
+		return hclsyntax.Tokens{}, pos, err
 	}
 
 	return b, pos, nil
 }
 
-func (f *file) blockAtPosition(pos hcllib.Pos) (*hcllib.Block, error) {
-	ast, _ := f.ast()
+func (f *file) blockAtPosition(pos hcllib.Pos) (hclsyntax.Tokens, error) {
+	pf, _ := f.parse()
 
-	if body, ok := ast.Body.(*hclsyntax.Body); ok {
-		if body.SrcRange.Empty() && pos != hcllib.InitialPos {
-			return nil, &InvalidHclPosErr{pos, body.SrcRange}
+	body, ok := pf.Body.(*hclsyntax.Body)
+	if !ok {
+		return hclsyntax.Tokens{}, fmt.Errorf("unexpected body type (%T)", body)
+	}
+	if body.SrcRange.Empty() && pos != hcllib.InitialPos {
+		return hclsyntax.Tokens{}, &InvalidHclPosErr{pos, body.SrcRange}
+	}
+	if !body.SrcRange.Empty() {
+		if posIsEqual(body.SrcRange.End, pos) {
+			return hclsyntax.Tokens{}, &NoBlockFoundErr{pos}
 		}
-		if !body.SrcRange.Empty() {
-			if posIsEqual(body.SrcRange.End, pos) {
-				return nil, &NoBlockFoundErr{pos}
-			}
-			if !body.SrcRange.ContainsPos(pos) {
-				return nil, &InvalidHclPosErr{pos, body.SrcRange}
-			}
+		if !body.SrcRange.ContainsPos(pos) {
+			return hclsyntax.Tokens{}, &InvalidHclPosErr{pos, body.SrcRange}
 		}
 	}
 
-	block := ast.OutermostBlockAtPos(pos)
-	if block == nil {
-		return nil, &NoBlockFoundErr{pos}
+	for _, block := range body.Blocks {
+		wholeRange := hcllib.RangeBetween(block.TypeRange, block.CloseBraceRange)
+		if wholeRange.ContainsPos(pos) {
+			return definitionTokens(tokensInRange(f.pf.Tokens, block.Range())), nil
+		}
 	}
 
-	return block, nil
+	return nil, &NoBlockFoundErr{pos}
+}
+
+func tokensInRange(tokens hclsyntax.Tokens, rng hcllib.Range) hclsyntax.Tokens {
+	var ts hclsyntax.Tokens
+
+	for _, t := range tokens {
+		if rangeContainsRange(rng, t.Range) {
+			ts = append(ts, t)
+		}
+	}
+
+	return ts
+}
+
+func rangeContainsRange(a, b hcl.Range) bool {
+	switch {
+	case a.Filename != b.Filename:
+		// If the ranges are in different files then they can't possibly contain each other
+		return false
+	case a.Empty() || b.Empty():
+		// Empty ranges can will never be contained in each other
+		return false
+	case rangeContainsOffset(a, b.Start.Byte) && rangeContainsOffset(a, b.End.Byte):
+		return true
+	case rangeContainsOffset(b, a.Start.Byte) && rangeContainsOffset(b, a.End.Byte):
+		return true
+	default:
+		return false
+	}
+}
+
+// rangeContainsOffset is a reimplementation of hcl.Range.ContainsOffset
+// which treats offset matching the end of a range as contained
+func rangeContainsOffset(rng hcl.Range, offset int) bool {
+	return offset >= rng.Start.Byte && offset <= rng.End.Byte
+}
+
+// definitionTokens turns any non-empty sequence of tokens into one that
+// satisfies HCL's loose definition of a valid block or attribute
+// as represented by tokens
+func definitionTokens(tokens hclsyntax.Tokens) hclsyntax.Tokens {
+	if len(tokens) > 0 {
+		// Check if seqence has a terminating token
+		lastToken := tokens[len(tokens)-1]
+		if lastToken.Type != hclsyntax.TokenEOF &&
+			lastToken.Type != hclsyntax.TokenNewline {
+			tRng := lastToken.Range
+
+			// if not we attach a newline
+			trailingNewLine := hclsyntax.Token{
+				Type:  hclsyntax.TokenNewline,
+				Bytes: []byte("\n"),
+				Range: hcl.Range{
+					Filename: tRng.Filename,
+					Start:    tRng.End,
+					End: hcl.Pos{
+						Byte:   tRng.End.Byte + 1,
+						Column: 1,
+						Line:   tRng.End.Line + 1,
+					},
+				},
+			}
+
+			tokens = append(tokens, trailingNewLine)
+		}
+	}
+	return tokens
 }
 
 func posIsEqual(a, b hcllib.Pos) bool {