Parser: rewrite tuple expr recovery to allow better items recovery (#…

…15227)

src/Compiler/SyntaxTree/SyntaxTreeOps.fs

@@ -939,15 +939,6 @@ let mkDynamicArgExpr expr =
     | SynExpr.Paren (expr = e) -> e
     | e -> e
 
-let rec normalizeTupleExpr exprs commas : SynExpr list * range list =
-    match exprs with
-    | SynExpr.Tuple (false, innerExprs, innerCommas, _) :: rest ->
-        let innerExprs, innerCommas =
-            normalizeTupleExpr (List.rev innerExprs) (List.rev innerCommas)
-
-        innerExprs @ rest, innerCommas @ commas
-    | _ -> exprs, commas
-
 let rec normalizeTuplePat pats commas : SynPat list * range List =
     match pats with
     | SynPat.Tuple (false, innerPats, innerCommas, _) :: rest ->

src/Compiler/SyntaxTree/SyntaxTreeOps.fsi

@@ -331,8 +331,6 @@ val prependIdentInLongIdentWithTrivia: ident: SynIdent -> mDot: range -> lid: Sy
 
 val mkDynamicArgExpr: expr: SynExpr -> SynExpr
 
-val normalizeTupleExpr: exprs: SynExpr list -> commas: range list -> SynExpr list * range List
-
 val normalizeTuplePat: pats: SynPat list -> commas: range list -> SynPat list * range List
 
 val desugarGetSetMembers: memberDefns: SynMemberDefns -> SynMemberDefns

src/Compiler/pars.fsy

@@ -169,6 +169,7 @@ let parse_error_rich = Some(fun (ctxt: ParseErrorContext<_>) ->
 %type <SynExpr> atomicExprAfterType
 %type <SynExpr> typedSequentialExprBlock
 %type <SynExpr * bool> atomicExpr
+%type <SynExpr list * range list> tupleExpr
 %type <SynTypeDefnSimpleRepr> tyconDefnOrSpfnSimpleRepr
 %type <Choice<SynEnumCase, SynUnionCase> list> unionTypeRepr
 %type <range * SynMemberDefns> tyconDefnAugmentation
@@ -4017,14 +4018,8 @@ declExpr:
 
   | tupleExpr %prec expr_tuple
      { let exprs, commas = $1
-       let exprs, commas =
-           // Nested non-struct tuple is only possible during error recovery,
-           // in other situations there are intermediate nodes.
-           match exprs with
-           | SynExpr.Tuple(false, _, _, _) :: _ -> normalizeTupleExpr exprs commas
-           | _ -> exprs, commas
-
-       SynExpr.Tuple(false, List.rev exprs, List.rev commas, (commas.Head, exprs) ||> unionRangeWithListBy (fun e -> e.Range)) }
+       let m = unionRanges exprs.Head.Range (List.last exprs).Range
+       SynExpr.Tuple(false, List.rev exprs, List.rev commas, m) }
 
   | declExpr JOIN_IN declExpr
      { SynExpr.JoinIn($1, rhs parseState 2, $3, unionRanges $1.Range $3.Range) }
@@ -4120,9 +4115,10 @@ declExpr:
      { reportParseErrorAt (rhs parseState 2) (FSComp.SR.parsUnfinishedExpression($2))
        exprFromParseError(mkSynInfix (rhs parseState 2) $1 $2 (arbExpr ("declExprInfix", (rhs parseState 3).StartRange))) }
 
-  | declExpr EQUALS OBLOCKEND_COMING_SOON
+  | declExpr EQUALS ends_coming_soon_or_recover
      { reportParseErrorAt (rhs parseState 2) (FSComp.SR.parsUnfinishedExpression("="))
-       exprFromParseError(mkSynInfix (rhs parseState 2) $1 "=" (arbExpr ("declExprInfix", (rhs parseState 3).StartRange))) }
+       let mEquals = rhs parseState 2
+       mkSynInfix mEquals $1 "=" (arbExpr ("declExprInfixEquals", mEquals.EndRange)) }
 
   | declExpr INFIX_COMPARE_OP OBLOCKEND_COMING_SOON
      { reportParseErrorAt (rhs parseState 2) (FSComp.SR.parsUnfinishedExpression($2))
@@ -4380,23 +4376,47 @@ tupleExpr:
         let exprs, commas = $1
         arbExpr ("tupleExpr1", commaRange.EndRange) :: exprs, commaRange :: commas }
 
+  | tupleExpr COMMA COMMA declExpr
+      { let exprs, commas = $1
+        let mComma1 = rhs parseState 2
+        let mComma2 = rhs parseState 3
+        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpressionInTuple ())
+        let expr = arbExpr ("tupleExpr2", mComma1.EndRange)
+        $4 :: expr :: exprs, (mComma2 :: mComma1 :: commas) }
+
+  | tupleExpr COMMA COMMA ends_coming_soon_or_recover
+      { let exprs, commas = $1
+        let mComma1 = rhs parseState 2
+        let mComma2 = rhs parseState 3
+        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpressionInTuple ())
+        if not $4 then reportParseErrorAt mComma2 (FSComp.SR.parsExpectedExpressionAfterToken ())
+        let expr1 = arbExpr ("tupleExpr3", mComma1.EndRange)
+        let expr2 = arbExpr ("tupleExpr4", mComma2.EndRange)
+        expr2 :: expr1 :: exprs, mComma2 :: mComma1 :: commas }
+
   | declExpr COMMA ends_coming_soon_or_recover
       { let commaRange = rhs parseState 2
         if not $3 then reportParseErrorAt commaRange (FSComp.SR.parsExpectedExpressionAfterToken ())
-        [arbExpr ("tupleExpr2", commaRange.EndRange); $1], [commaRange] }
+        [arbExpr ("tupleExpr5", commaRange.EndRange); $1], [commaRange] }
 
   | declExpr COMMA declExpr
       { [$3; $1], [rhs parseState 2] }
 
-  | COMMA declExpr
-      { let commaRange = rhs parseState 1
-        reportParseErrorAt commaRange (FSComp.SR.parsExpectingExpressionInTuple ())
-        [$2; arbExpr ("tupleExpr3", commaRange.StartRange)], [commaRange] }
-
-  | COMMA ends_coming_soon_or_recover
-      { let commaRange = rhs parseState 1
-        if not $2 then reportParseErrorAt commaRange (FSComp.SR.parsExpectedExpressionAfterToken ())
-        [arbExpr ("tupleExpr4", commaRange.EndRange); arbExpr ("tupleExpr5", commaRange.StartRange)], [commaRange] }
+  | declExpr COMMA COMMA ends_coming_soon_or_recover
+      { let mComma1 = rhs parseState 2
+        let mComma2 = rhs parseState 3
+        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpressionInTuple ())
+        if not $4 then reportParseErrorAt mComma2 (FSComp.SR.parsExpectedExpressionAfterToken ())
+        let expr1 = arbExpr ("tupleExpr6", mComma1.EndRange)
+        let expr2 = arbExpr ("tupleExpr7", mComma2.EndRange)
+        [expr2; expr1; $1], [mComma2; mComma1] }
+
+  | declExpr COMMA COMMA declExpr
+      { let mComma1 = rhs parseState 2
+        let mComma2 = rhs parseState 3
+        reportParseErrorAt mComma2 (FSComp.SR.parsExpectingExpressionInTuple ())
+        let expr = arbExpr ("tupleExpr8", mComma1.EndRange)
+        [$4; expr; $1], [mComma2; mComma1] }
 
 minusExpr:
   | INFIX_AT_HAT_OP minusExpr
@@ -4744,6 +4764,19 @@ parenExpr:
         //let mLhs = if $2 then unionRangeWithPos mLeftParen (rhs parseState 2).Start else mLeftParen
         //arbExpr ("parenExpr2", mLhs) }
 
+  | LPAREN COMMA declExpr rparen
+      { let mComma = rhs parseState 2
+        let mLparen = rhs parseState 1
+        let mRparen = rhs parseState 3
+        let errorExpr = arbExpr ("tupleExpr3", mComma.EndRange)
+        let mTuple = unionRanges mComma $3.Range
+        let tupleExpr =
+            match $3 with
+            | SynExpr.Tuple(false, exprs, commas, m) ->
+                SynExpr.Tuple(false, errorExpr :: exprs, mComma :: commas, mTuple)
+            | expr -> SynExpr.Tuple(false, [errorExpr; expr], [mComma], mTuple)
+        SynExpr.Paren(tupleExpr, mLparen, Some mRparen, rhs2 parseState 1 4) }
+
 parenExprBody:
   | typars COLON LPAREN classMemberSpfn rparen typedSequentialExpr
       { (fun m -> SynExpr.TraitCall($1, $4, $6, m)) } /* disambiguate: x $a.id(x) */

tests/service/ParserTests.fs

@@ -210,119 +210,6 @@ let checkRangeCountAndOrder commas =
         |> List.pairwise
         |> List.iter (assertIsBefore id))
 
-[<Test>]
-let ``Expr - Tuple 01`` () =
-    let parseResults = getParseResults """
-(,)
-(,,)
-(,,,)
-"""
-    let exprs = getSingleModuleMemberDecls parseResults |> List.map getSingleParenInnerExpr
-    match exprs with
-    | [ SynExpr.Tuple(_, [SynExpr.ArbitraryAfterError _ as e11; SynExpr.ArbitraryAfterError _ as e12], c1, _)
-        SynExpr.Tuple(_, [SynExpr.ArbitraryAfterError _ as e21; SynExpr.ArbitraryAfterError _ as e22; SynExpr.ArbitraryAfterError _ as e23], c2, _)
-        SynExpr.Tuple(_, [SynExpr.ArbitraryAfterError _ as e31; SynExpr.ArbitraryAfterError _ as e32; SynExpr.ArbitraryAfterError _ as e33; SynExpr.ArbitraryAfterError _ as e34], c3, _) ] ->
-            [ e11; e12; e21; e22; e23; e31; e32; e33; e34 ] |> checkNodeOrder
-            [ c1, 1; c2, 2; c3, 3 ] |> checkRangeCountAndOrder
-
-    | _ -> failwith "Unexpected tree"
-
-[<Test>]
-let ``Expr - Tuple 02`` () =
-    let parseResults = getParseResults """
-(1,)
-(,1)
-(1,1)
-"""
-    let exprs = getSingleModuleMemberDecls parseResults |> List.map getSingleParenInnerExpr    
-    match exprs with
-    | [ SynExpr.Tuple(_, [SynExpr.Const _ as e11; SynExpr.ArbitraryAfterError _ as e12], c1, _)
-        SynExpr.Tuple(_, [SynExpr.ArbitraryAfterError _  as e21; SynExpr.Const _  as e22], c2, _)
-        SynExpr.Tuple(_, [SynExpr.Const _  as e31; SynExpr.Const _  as e32], c3, _) ] ->
-            [ e11; e12; e21; e22; e31; e32 ] |> checkNodeOrder
-            [ c1, 1; c2, 1; c3, 1 ] |> checkRangeCountAndOrder
-
-    | _ -> failwith "Unexpected tree"
-
-[<Test>]
-let ``Expr - Tuple 03`` () =
-    let parseResults = getParseResults """
-(1,,)
-(,1,)
-(,,1)
-
-(1,1,)
-(,1,1)
-(1,,1)
-
-(1,1,1)
-"""
-    let exprs = getSingleModuleMemberDecls parseResults |> List.map getSingleParenInnerExpr    
-    match exprs with
-    | [ SynExpr.Tuple(_, [SynExpr.Const _ as e11; SynExpr.ArbitraryAfterError _  as e12; SynExpr.ArbitraryAfterError _  as e13], c1, _)
-        SynExpr.Tuple(_, [SynExpr.ArbitraryAfterError _ as e21; SynExpr.Const _ as e22; SynExpr.ArbitraryAfterError _ as e23], c2, _)
-        SynExpr.Tuple(_, [SynExpr.ArbitraryAfterError _ as e31; SynExpr.ArbitraryAfterError _ as e32; SynExpr.Const _ as e33], c3, _)
-
-        SynExpr.Tuple(_, [SynExpr.Const _ as e41; SynExpr.Const _ as e42; SynExpr.ArbitraryAfterError _ as e43], c4, _)
-        SynExpr.Tuple(_, [SynExpr.ArbitraryAfterError _ as e51; SynExpr.Const _ as e52; SynExpr.Const _ as e53], c5, _)
-        SynExpr.Tuple(_, [SynExpr.Const _ as e61; SynExpr.ArbitraryAfterError _ as e62; SynExpr.Const _ as e63], c6, _)
-
-        SynExpr.Tuple(_, [SynExpr.Const _ as e71; SynExpr.Const _ as e72; SynExpr.Const _ as e73], c7, _) ] ->
-            [ e11; e12; e13; e21; e22; e23; e31; e32; e33
-              e41; e42; e43; e51; e52; e53; e61; e62; e63
-              e71; e72; e73 ]
-            |> checkNodeOrder
-
-            [ c1, 2; c2, 2; c3, 2
-              c4, 2; c5, 2; c6, 2
-              c7, 2 ]
-            |> checkRangeCountAndOrder
-
-    | _ -> failwith "Unexpected tree"
-
-
-[<Test>]
-let ``Expr - Tuple 04`` () =
-    let parseResults = getParseResults """
-(,1,,2,3,,4,)
-"""
-    let exprs = getSingleModuleMemberDecls parseResults |> List.map getSingleParenInnerExpr
-    match exprs with
-    | [ SynExpr.Tuple(_, [ SynExpr.ArbitraryAfterError _ as e1
-                           SynExpr.Const _ as e2
-                           SynExpr.ArbitraryAfterError _ as e3
-                           SynExpr.Const _ as e4
-                           SynExpr.Const _ as e5
-                           SynExpr.ArbitraryAfterError _  as e6
-                           SynExpr.Const _ as e7
-                           SynExpr.ArbitraryAfterError _  as e8 ], c, _) ] ->
-            [ e1; e2; e3; e4; e5; e6; e7; e8 ] |> checkNodeOrder
-            [ c, 7 ] |> checkRangeCountAndOrder
-
-    | _ -> failwith "Unexpected tree"
-
-[<Test>]
-let ``Expr - Tuple 05`` () =
-    let parseResults = getParseResults """
-(1,
-"""
-    match getSingleModuleMemberDecls parseResults |> List.map getSingleParenInnerExpr with
-    | [ SynExpr.FromParseError(SynExpr.Tuple(_, [SynExpr.Const _; SynExpr.ArbitraryAfterError _], _, _), _) ] -> ()
-    | _ -> failwith "Unexpected tree"
-
-[<Test>]
-let ``Expr - Tuple 06`` () =
-    let parseResults = getParseResults """
-(1,,,2)
-"""
-    let synExprs = getSingleModuleMemberDecls parseResults |> List.map getSingleParenInnerExpr
-    match synExprs with
-    | [ SynExpr.Tuple(_, [ SynExpr.Const _
-                           SynExpr.ArbitraryAfterError _
-                           SynExpr.ArbitraryAfterError _
-                           SynExpr.Const _ ], _, _) ] -> ()
-    | _ -> failwith "Unexpected tree"
-
 [<Test>]
 let ``Expr - Tuple 07`` () =
     let parseResults = getParseResults """

tests/service/data/SyntaxTree/Expression/Binary - Eq 01.fs

@@ -0,0 +1,3 @@
+module Module
+
+a = b

tests/service/data/SyntaxTree/Expression/Binary - Eq 01.fs.bsl

@@ -0,0 +1,21 @@
+ImplFile
+  (ParsedImplFileInput
+     ("/root/Expression/Binary - Eq 01.fs", false, QualifiedNameOfFile Module,
+      [], [],
+      [SynModuleOrNamespace
+         ([Module], false, NamedModule,
+          [Expr
+             (App
+                (NonAtomic, false,
+                 App
+                   (NonAtomic, true,
+                    LongIdent
+                      (false,
+                       SynLongIdent
+                         ([op_Equality], [], [Some (OriginalNotation "=")]),
+                       None, (3,2--3,3)), Ident a, (3,0--3,3)), Ident b,
+                 (3,0--3,5)), (3,0--3,5))],
+          PreXmlDoc ((1,0), FSharp.Compiler.Xml.XmlDocCollector), [], None,
+          (1,0--3,5), { LeadingKeyword = Module (1,0--1,6) })], (true, true),
+      { ConditionalDirectives = []
+        CodeComments = [] }, set []))

tests/service/data/SyntaxTree/Expression/Binary - Eq 02.fs

@@ -0,0 +1,3 @@
+module Module
+
+M(a = b)

tests/service/data/SyntaxTree/Expression/Binary - Eq 02.fs.bsl

@@ -0,0 +1,25 @@
+ImplFile
+  (ParsedImplFileInput
+     ("/root/Expression/Binary - Eq 02.fs", false, QualifiedNameOfFile Module,
+      [], [],
+      [SynModuleOrNamespace
+         ([Module], false, NamedModule,
+          [Expr
+             (App
+                (Atomic, false, Ident M,
+                 Paren
+                   (App
+                      (NonAtomic, false,
+                       App
+                         (NonAtomic, true,
+                          LongIdent
+                            (false,
+                             SynLongIdent
+                               ([op_Equality], [], [Some (OriginalNotation "=")]),
+                             None, (3,4--3,5)), Ident a, (3,2--3,5)), Ident b,
+                       (3,2--3,7)), (3,1--3,2), Some (3,7--3,8), (3,1--3,8)),
+                 (3,0--3,8)), (3,0--3,8))],
+          PreXmlDoc ((1,0), FSharp.Compiler.Xml.XmlDocCollector), [], None,
+          (1,0--3,8), { LeadingKeyword = Module (1,0--1,6) })], (true, true),
+      { ConditionalDirectives = []
+        CodeComments = [] }, set []))

tests/service/data/SyntaxTree/Expression/Binary - Eq 03.fs

@@ -0,0 +1,3 @@
+module Module
+
+M(a = )

tests/service/data/SyntaxTree/Expression/Binary - Eq 03.fs.bsl

@@ -0,0 +1,28 @@
+ImplFile
+  (ParsedImplFileInput
+     ("/root/Expression/Binary - Eq 03.fs", false, QualifiedNameOfFile Module,
+      [], [],
+      [SynModuleOrNamespace
+         ([Module], false, NamedModule,
+          [Expr
+             (App
+                (Atomic, false, Ident M,
+                 Paren
+                   (App
+                      (NonAtomic, false,
+                       App
+                         (NonAtomic, true,
+                          LongIdent
+                            (false,
+                             SynLongIdent
+                               ([op_Equality], [], [Some (OriginalNotation "=")]),
+                             None, (3,4--3,5)), Ident a, (3,2--3,5)),
+                       ArbitraryAfterError ("declExprInfixEquals", (3,5--3,5)),
+                       (3,2--3,5)), (3,1--3,2), Some (3,6--3,7), (3,1--3,7)),
+                 (3,0--3,7)), (3,0--3,7))],
+          PreXmlDoc ((1,0), FSharp.Compiler.Xml.XmlDocCollector), [], None,
+          (1,0--3,7), { LeadingKeyword = Module (1,0--1,6) })], (true, true),
+      { ConditionalDirectives = []
+        CodeComments = [] }, set []))
+
+(3,4)-(3,5) parse error Unexpected token '=' or incomplete expression

tests/service/data/SyntaxTree/Expression/Binary - Eq 04.fs

@@ -0,0 +1,3 @@
+module Module
+
+M(a = 1, b =)

tests/service/data/SyntaxTree/Expression/Binary - Eq 04.fs.bsl

@@ -0,0 +1,43 @@
+ImplFile
+  (ParsedImplFileInput
+     ("/root/Expression/Binary - Eq 04.fs", false, QualifiedNameOfFile Module,
+      [], [],
+      [SynModuleOrNamespace
+         ([Module], false, NamedModule,
+          [Expr
+             (App
+                (Atomic, false, Ident M,
+                 Paren
+                   (Tuple
+                      (false,
+                       [App
+                          (NonAtomic, false,
+                           App
+                             (NonAtomic, true,
+                              LongIdent
+                                (false,
+                                 SynLongIdent
+                                   ([op_Equality], [],
+                                    [Some (OriginalNotation "=")]), None,
+                                 (3,4--3,5)), Ident a, (3,2--3,5)),
+                           Const (Int32 1, (3,6--3,7)), (3,2--3,7));
+                        App
+                          (NonAtomic, false,
+                           App
+                             (NonAtomic, true,
+                              LongIdent
+                                (false,
+                                 SynLongIdent
+                                   ([op_Equality], [],
+                                    [Some (OriginalNotation "=")]), None,
+                                 (3,11--3,12)), Ident b, (3,9--3,12)),
+                           ArbitraryAfterError
+                             ("declExprInfixEquals", (3,12--3,12)), (3,9--3,12))],
+                       [(3,7--3,8)], (3,2--3,12)), (3,1--3,2), Some (3,12--3,13),
+                    (3,1--3,13)), (3,0--3,13)), (3,0--3,13))],
+          PreXmlDoc ((1,0), FSharp.Compiler.Xml.XmlDocCollector), [], None,
+          (1,0--3,13), { LeadingKeyword = Module (1,0--1,6) })], (true, true),
+      { ConditionalDirectives = []
+        CodeComments = [] }, set []))
+
+(3,11)-(3,12) parse error Unexpected token '=' or incomplete expression