diff --git a/go.mod b/go.mod
index a1794718d..a04814f1c 100644
--- a/go.mod
+++ b/go.mod
@@ -19,7 +19,6 @@ require (
 	github.com/zclconf/go-cty v1.2.1
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
-	golang.org/x/text v0.3.2
 )
 
 replace github.com/sourcegraph/go-lsp => github.com/radeksimko/go-lsp v0.1.0
diff --git a/internal/hcl/hcl.go b/internal/hcl/hcl.go
index 84fd1e40b..0b31337b4 100644
--- a/internal/hcl/hcl.go
+++ b/internal/hcl/hcl.go
@@ -61,26 +61,21 @@ func (f *file) parse() (*parsedFile, error) {
 		return f.pf, nil
 	}
 
-	var parseDiags hcl.Diagnostics
-
 	tokens, diags := hclsyntax.LexConfig(f.content, f.filename, hcl.InitialPos)
 	if diags.HasErrors() {
-		parseDiags = append(parseDiags, diags...)
+		// The hclsyntax parser assumes all tokens are valid
+		// so we return early here
+		// TODO: Avoid ignoring TokenQuotedNewline to provide completion in unclosed string
+		return nil, diags
 	}
 
-	body, diags := hclsyntax.ParseBodyFromTokens(tokens, hclsyntax.TokenEOF)
-	if diags.HasErrors() {
-		parseDiags = append(parseDiags, diags...)
-	}
+	body, _ := hclsyntax.ParseBodyFromTokens(tokens, hclsyntax.TokenEOF)
 
 	f.pf = &parsedFile{
 		Tokens: tokens,
 		Body:   body,
 	}
 
-	if parseDiags.HasErrors() {
-		return f.pf, parseDiags
-	}
 	return f.pf, nil
 }
 
@@ -94,7 +89,10 @@ func (f *file) PosInBlock(pos hcl.Pos) bool {
 }
 
 func (f *file) BlockAtPosition(pos hcl.Pos) (TokenizedBlock, error) {
-	pf, _ := f.parse()
+	pf, err := f.parse()
+	if err != nil {
+		return nil, err
+	}
 
 	body, ok := pf.Body.(*hclsyntax.Body)
 	if !ok {
diff --git a/internal/hcl/hcl_test.go b/internal/hcl/hcl_test.go
index f3f548d1a..ed96cff25 100644
--- a/internal/hcl/hcl_test.go
+++ b/internal/hcl/hcl_test.go
@@ -56,6 +56,28 @@ func TestFile_BlockAtPosition(t *testing.T) {
 				},
 			},
 		},
+		{
+			"invalid tokens",
+			`provider "aws" {
+  arg = "
+}`,
+			hcl.Pos{
+				Line:   2,
+				Column: 10,
+				Byte:   26,
+			},
+			hcl.Diagnostics{
+				&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid multi-line string",
+					Detail: `Quoted strings may not be split over multiple lines. ` +
+						`To produce a multi-line string, either use the \n escape to ` +
+						`represent a newline character or use the "heredoc" ` +
+						`multi-line template syntax.`,
+				},
+			},
+			[]hclsyntax.Token{},
+		},
 		{
 			"valid config and position",
 			`provider "aws" {
diff --git a/internal/terraform/lang/parser_test.go b/internal/terraform/lang/parser_test.go
index 342798071..5a61bf8f3 100644
--- a/internal/terraform/lang/parser_test.go
+++ b/internal/terraform/lang/parser_test.go
@@ -124,6 +124,7 @@ func TestParser_ParseBlockFromTokens(t *testing.T) {
 }
 
 func newTestBlock(t *testing.T, src string) ihcl.TokenizedBlock {
+	t.Helper()
 	b, err := ihcl.NewTestBlock([]byte(src))
 	if err != nil {
 		t.Fatal(err)