From f560e75aa687dffc1cc87040f3991e880049d8c7 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Mon, 4 Oct 2021 23:40:18 -0700 Subject: [PATCH 01/59] First pass implementation with a few tests as PoC --- .../StackFrameParserTests.Utilities.cs | 225 +++++++++++ .../StackFrame/StackFrameParserTests.cs | 103 +++++ .../StackFrame/IStackFrameNodeVisitor.cs | 23 ++ .../StackFrame/StackFrameCompilationUnit.cs | 26 ++ .../StackFrame/StackFrameKind.cs | 51 +++ .../StackFrame/StackFrameLexer.cs | 368 +++++++++++++++++ .../StackFrame/StackFrameNodeDefinitions.cs | 375 ++++++++++++++++++ .../StackFrame/StackFrameParser.cs | 293 ++++++++++++++ .../StackFrame/StackFrameTree.cs | 24 ++ 9 files changed, 1488 insertions(+) create mode 100644 src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs create mode 100644 src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs new file mode 100644 index 0000000000000..51d7cd256d129 --- /dev/null +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -0,0 +1,225 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; +using Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame; +using Roslyn.Test.Utilities; +using Xunit; + +namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame +{ + using StackFrameToken = EmbeddedSyntaxToken; + using StackFrameTrivia = EmbeddedSyntaxTrivia; + using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; + + public partial class StackFrameParserTests + { + private static StackFrameToken CreateToken(StackFrameKind kind, string s, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + => new( + kind, + leadingTrivia.IsDefaultOrEmpty ? ImmutableArray.Empty : leadingTrivia, + CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, s), + trailingTrivia.IsDefaultOrEmpty ? ImmutableArray.Empty : trailingTrivia, + ImmutableArray.Empty, + value: null!); + + private static StackFrameTrivia CreateTrivia(StackFrameKind kind, string s) + => new(kind, CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, s), ImmutableArray.Empty); + + private static ImmutableArray CreateTriviaArray(StackFrameTrivia trivia) + => ImmutableArray.Create(trivia); + + private static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); + private static readonly StackFrameToken CommaToken = CreateToken(StackFrameKind.CommaToken, ","); + private static readonly StackFrameToken OpenParenToken = CreateToken(StackFrameKind.OpenParenToken, "("); + private static readonly StackFrameToken CloseParenToken = CreateToken(StackFrameKind.CloseParenToken, ")"); + private static readonly StackFrameToken OpenBracketToken = CreateToken(StackFrameKind.OpenBracketToken, "["); + private static readonly StackFrameToken CloseBracketToken = CreateToken(StackFrameKind.CloseBracketToken, "]"); + private static readonly StackFrameToken SpaceToken = CreateToken(StackFrameKind.TextToken, " "); + + private static readonly StackFrameTrivia SpaceTrivia = CreateTrivia(StackFrameKind.WhitespaceTrivia, " "); + + private static void AssertEqual(StackFrameNodeOrToken expected, StackFrameNodeOrToken actual) + { + AssertEqual(expected.Node, actual.Node); + AssertEqual(expected.Token, actual.Token); + } + + private static void AssertEqual(StackFrameNode? expected, StackFrameNode? actual) + { + if (expected is null) + { + Assert.Null(actual); + return; + } + + AssertEx.NotNull(actual); + + Assert.Equal(expected.Kind, actual.Kind); + Assert.True(expected.ChildCount == actual.ChildCount, PrintChildDifference(expected, actual)); + + for (var i = 0; i < expected.ChildCount; i++) + { + AssertEqual(expected.ChildAt(i), actual.ChildAt(i)); + } + + static string PrintChildDifference(StackFrameNode expected, StackFrameNode actual) + { + var sb = new StringBuilder(); + sb.Append("Expected: "); + Print(expected, sb); + sb.AppendLine(); + + sb.Append("Actual: "); + Print(actual, sb); + + return sb.ToString(); + } + } + + private static void Print(StackFrameNode node, StringBuilder sb) + { + foreach (var child in node) + { + if (child.IsNode) + { + Print(child.Node, sb); + } + else + { + if (!child.Token.LeadingTrivia.IsDefaultOrEmpty) + { + Print(child.Token.LeadingTrivia, sb); + } + + sb.Append(child.Token.VirtualChars.CreateString()); + + if (!child.Token.TrailingTrivia.IsDefaultOrEmpty) + { + Print(child.Token.TrailingTrivia, sb); + } + } + } + } + + private static void Print(ImmutableArray triviaArray, StringBuilder sb) + { + if (triviaArray.IsDefault) + { + sb.Append(""); + return; + } + + if (triviaArray.IsEmpty) + { + sb.Append(""); + return; + } + + foreach (var trivia in triviaArray) + { + sb.Append(trivia.VirtualChars.CreateString()); + } + } + + private static void AssertEqual(StackFrameToken expected, StackFrameToken actual) + { + AssertEqual(expected.LeadingTrivia, actual.LeadingTrivia); + AssertEqual(expected.TrailingTrivia, actual.TrailingTrivia); + + Assert.Equal(expected.Kind, actual.Kind); + Assert.Equal(expected.IsMissing, actual.IsMissing); + Assert.Equal(expected.VirtualChars.CreateString(), actual.VirtualChars.CreateString()); + } + + private static void AssertEqual(ImmutableArray expected, ImmutableArray actual) + { + var diffMessage = PrintDiff(); + + if (expected.IsDefault) + { + Assert.True(actual.IsDefault, diffMessage); + return; + } + + Assert.False(actual.IsDefault, diffMessage); + Assert.True(expected.Length == actual.Length, diffMessage); + + for (var i = 0; i < expected.Length; i++) + { + AssertEqual(expected[i], actual[i]); + } + + string PrintDiff() + { + var sb = new StringBuilder(); + sb.Append("Expected: "); + + if (!expected.IsDefaultOrEmpty) + { + sb.Append('['); + } + + Print(expected, sb); + + if (expected.IsDefaultOrEmpty) + { + sb.AppendLine(); + } + else + { + sb.AppendLine("]"); + } + + sb.Append("Actual: "); + + if (!actual.IsDefaultOrEmpty) + { + sb.Append('['); + } + + Print(actual, sb); + + if (!actual.IsDefaultOrEmpty) + { + sb.Append(']'); + } + + return sb.ToString(); + } + } + + private static void AssertEqual(StackFrameTrivia expected, StackFrameTrivia actual) + { + Assert.Equal(expected.Kind, actual.Kind); + Assert.Equal(expected.VirtualChars.CreateString(), actual.VirtualChars.CreateString()); + } + + private static StackFrameArgumentList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) + { + var nodesWithParens = nodesOrTokens.Prepend(OpenParenToken).Append(CloseParenToken).ToImmutableArray(); + return new StackFrameArgumentList(nodesWithParens); + } + + private static StackFrameMethodDeclarationNode MethodDeclaration( + StackFrameMemberAccessExpressionNode memberAccessExpression, + StackFrameArgumentList argumentList, + StackFrameTypeArgumentList? typeArgumnets = null) + { + return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArgumnets, argumentList); + } + + private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameExpressionNode expressionNode, StackFrameBaseIdentifierNode identifierNode) + => new(expressionNode, DotToken, identifierNode); + + private static StackFrameIdentifierNode Identifier(string identifierName, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + => new(CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia)); + + private static StackFrameArrayExpressionNode ArrayExpression(StackFrameExpressionNode identifier, params StackFrameToken[] arrayTokens) + => new(identifier, arrayTokens.ToImmutableArray()); + } +} diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs new file mode 100644 index 0000000000000..275a141cc41af --- /dev/null +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -0,0 +1,103 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame; +using Roslyn.Test.Utilities; +using Xunit; + +namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame +{ + public partial class StackFrameParserTests + { + [Fact] + public void TestMethodOneParam() + { + var input = @"at ConsoleApp4.MyClass.M(string s)"; + var tree = StackFrameParser.TryParse(input); + AssertEx.NotNull(tree); + + Assert.True(tree.Root.AtTrivia.HasValue); + Assert.False(tree.Root.InTrivia.HasValue); + Assert.Null(tree.Root.FileInformationExpression); + Assert.False(tree.Root.TrailingTrivia.HasValue); + + var methodDeclaration = tree.Root.MethodDeclaration; + + var expectedMethodDeclaration = MethodDeclaration( + MemberAccessExpression( + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + + ArgumentList( + Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("s")) + ); + + AssertEqual(expectedMethodDeclaration, methodDeclaration); + } + + [Fact] + public void TestMethodTwoParam() + { + var input = @"at ConsoleApp4.MyClass.M(string s, string t)"; + var tree = StackFrameParser.TryParse(input); + AssertEx.NotNull(tree); + + Assert.True(tree.Root.AtTrivia.HasValue); + Assert.False(tree.Root.InTrivia.HasValue); + Assert.Null(tree.Root.FileInformationExpression); + Assert.False(tree.Root.TrailingTrivia.HasValue); + + var methodDeclaration = tree.Root.MethodDeclaration; + + var expectedMethodDeclaration = MethodDeclaration( + MemberAccessExpression( + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + + ArgumentList( + Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("s"), + CommaToken, + Identifier("string", leadingTrivia: CreateTriviaArray(SpaceTrivia), trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("t")) + ); + + AssertEqual(expectedMethodDeclaration, methodDeclaration); + } + + [Fact] + public void TestMethodArrayParam() + { + var input = @"at ConsoleApp4.MyClass.M(string[] s)"; + var tree = StackFrameParser.TryParse(input); + AssertEx.NotNull(tree); + + Assert.True(tree.Root.AtTrivia.HasValue); + Assert.False(tree.Root.InTrivia.HasValue); + Assert.Null(tree.Root.FileInformationExpression); + Assert.False(tree.Root.TrailingTrivia.HasValue); + + var methodDeclaration = tree.Root.MethodDeclaration; + + var expectedMethodDeclaration = MethodDeclaration( + MemberAccessExpression( + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + + ArgumentList( + ArrayExpression(Identifier("string"), OpenBracketToken, CloseBracketToken), + Identifier("s", leadingTrivia: CreateTriviaArray(SpaceTrivia))) + ); + + AssertEqual(expectedMethodDeclaration, methodDeclaration); + } + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs new file mode 100644 index 0000000000000..3b8a9d12d50cc --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + internal interface IStackFrameNodeVisitor + { + void Visit(StackFrameTextNode stackFrameTextNode); + void Visit(StackFrameMethodDeclarationNode stackFrameMethodDeclarationNode); + void Visit(StackFrameMemberAccessExpressionNode stackFrameMemberAccessExpressionNode); + void Visit(StackFrameTypeArgumentList stackFrameTypeArguments); + void Visit(StackFrameArgumentList stackFrameArgumentList); + void Visit(StackFrameIdentifierNode stackFrameIdentifierNode); + void Visit(StackFrameGenericTypeIdentifier stackFrameGenericTypeIdentifier); + void Visit(StackFrameTypeArgument stackFrameTypeArgument); + void Visit(StackFrameArrayExpressionNode stackFrameArrayExpressionNode); + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs new file mode 100644 index 0000000000000..a389386d0996f --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + internal class StackFrameCompilationUnit + { + public readonly EmbeddedSyntaxTrivia? AtTrivia; + public readonly StackFrameMethodDeclarationNode MethodDeclaration; + public readonly EmbeddedSyntaxTrivia? InTrivia; + public readonly StackFrameFileInformationNode? FileInformationExpression; + public readonly EmbeddedSyntaxTrivia? TrailingTrivia; + + public StackFrameCompilationUnit(EmbeddedSyntaxTrivia? atTrivia, StackFrameMethodDeclarationNode methodDeclaration, EmbeddedSyntaxTrivia? inTrivia, StackFrameFileInformationNode? fileInformationExpression, EmbeddedSyntaxTrivia? trailingTrivia) + { + AtTrivia = atTrivia; + MethodDeclaration = methodDeclaration; + InTrivia = inTrivia; + FileInformationExpression = fileInformationExpression; + TrailingTrivia = trailingTrivia; + } + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs new file mode 100644 index 0000000000000..936572069188f --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + internal enum StackFrameKind + { + None = 0, + EndOfLine, + Text, + MethodDeclaration, + MemberAccess, + Identifier, + GenericTypeIdentifier, + TypeArgument, + TypeIdentifier, + ArgumentList, + ArrayExpression, + + // Tokens + AmpersandToken, + OpenBracketToken, + CloseBracketToken, + OpenParenToken, + CloseParenToken, + DotToken, + TextToken, + PlusToken, + CommaToken, + ColonToken, + EqualsToken, + GreaterThanToken, + LessThanToken, + MinusToken, + SingleQuoteToken, + GraveAccentToken, // ` + BackslashToken, + ForwardSlashToken, + IdentifierToken, + WhitespaceToken, + + // Trivia + WhitespaceTrivia, + AtTrivia, // "at " portion of the stack frame + InTrivia, // optionsal " in " portion of the stack frame + TrailingTrivia, // All trailing text that is not syntactically relavent + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs new file mode 100644 index 0000000000000..a4d210874add8 --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -0,0 +1,368 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; +using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; +using Microsoft.CodeAnalysis.PooledObjects; +using Microsoft.CodeAnalysis.Text; +using Roslyn.Utilities; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + using StackFrameToken = EmbeddedSyntaxToken; + using StackFrameTrivia = EmbeddedSyntaxTrivia; + + internal struct StackFrameLexer + { + public readonly VirtualCharSequence Text; + public int Position; + + public StackFrameLexer(string text) + : this(VirtualCharSequence.Create(0, text)) + { + } + + public StackFrameLexer(VirtualCharSequence text) : this() + => Text = text; + + public VirtualChar CurrentChar => Position < Text.Length ? Text[Position] : default; + + public VirtualCharSequence GetSubPatternToCurrentPos(int start) + => GetSubPattern(start, Position); + + public VirtualCharSequence GetSubPattern(int start, int end) + => Text.GetSubSequence(TextSpan.FromBounds(start, end)); + + public StackFrameToken ScanNextToken(bool allowTrivia) + { + var trivia = ScanLeadingTrivia(allowTrivia); + if (Position == Text.Length) + { + return CreateToken(StackFrameKind.EndOfLine, trivia, VirtualCharSequence.Empty); + } + + var ch = CurrentChar; + Position++; + + return CreateToken(GetKind(ch), trivia, Text.GetSubSequence(new TextSpan(Position - 1, 1))); + } + + /// + /// Scans until EndOfLine is found, treating all text as trivia + /// + public StackFrameTrivia? ScanTrailingTrivia() + { + if (Position == Text.Length) + { + return null; + } + + var length = Text.Length - Position; + return CreateTrivia(StackFrameKind.TrailingTrivia, Text.GetSubSequence(new TextSpan(Position - 1, length))); + } + + public StackFrameToken? ScanIdentifier() + { + if (Position == Text.Length) + { + return null; + } + + var startPosition = Position; + + var ch = CurrentChar; + Position++; + + if (!UnicodeCharacterUtilities.IsIdentifierStartCharacter((char)ch.Value)) + { + return null; + } + + while (UnicodeCharacterUtilities.IsIdentifierPartCharacter((char)ch.Value)) + { + ch = CurrentChar; + Position++; + } + + var identifierEnd = Position - 1; + var identifierSpan = new TextSpan(startPosition, identifierEnd - startPosition); + var identifier = CreateToken(StackFrameKind.IdentifierToken, Text.GetSubSequence(identifierSpan)); + return identifier; + } + + public StackFrameToken ScanTypeArity() + { + if (Position == Text.Length) + { + return default; + } + + var startPosition = Position; + var ch = CurrentChar; + Position++; + + while (IsNumber(ch)) + { + ch = CurrentChar; + Position++; + } + + var aritySpan = new TextSpan(startPosition, (Position - 1) - startPosition); + var arityToken = CreateToken(StackFrameKind.TextToken, Text.GetSubSequence(aritySpan)); + return arityToken; + } + + internal ImmutableArray ScanArrayBrackets() + { + if (Position == Text.Length) + { + return default; + } + + using var _ = ArrayBuilder.GetInstance(out var builder); + + while (Position < Text.Length) + { + // Use PreviousCharAsToken here because we expect the + // start of the brackets to be the previous token that started + // the need to parse for an array + var token = PreviousCharAsToken(); + if (!IsArrayBracket(token)) + { + break; + } + + Position++; + builder.Add(token); + } + + Debug.Assert(builder.Count >= 2); + + return builder.ToImmutable(); + + static bool IsArrayBracket(StackFrameToken token) + => token.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.CloseBracketToken; + } + + public StackFrameToken PreviousCharAsToken() + { + var previousChar = Text[Position - 1]; + return CreateToken(GetKind(previousChar), Text.GetSubSequence(new TextSpan(Position - 1, 1))); + } + + internal StackFrameTrivia? ScanWhiteSpace(bool includePrevious) + { + if (Position == Text.Length) + { + return null; + } + + var startPosition = includePrevious ? Position - 1 : Position; + + while (IsBlank(CurrentChar)) + { + Position++; + } + + if (Position == startPosition) + { + return null; + } + + var whitespaceSpan = new TextSpan(startPosition, Position - startPosition); + return CreateTrivia(StackFrameKind.WhitespaceTrivia, Text.GetSubSequence(whitespaceSpan)); + } + + public StackFrameToken CurrentCharAsToken() + { + if (Position == Text.Length) + { + return CreateToken(StackFrameKind.EndOfLine, VirtualCharSequence.Empty); + } + + var previousChar = Text[Position]; + return CreateToken(GetKind(previousChar), Text.GetSubSequence(new TextSpan(Position, 1))); + } + + public bool IsAt(string val) + => TextAt(Position, val); + + private bool TextAt(int position, string val) + { + for (var i = 0; i < val.Length; i++) + { + if (position + i >= Text.Length || + Text[position + i] != val[i]) + { + return false; + } + } + + return true; + } + +#nullable disable + public static StackFrameToken CreateToken(StackFrameKind kind, VirtualCharSequence virtualChars) + => new(kind, ImmutableArray.Empty, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null); + + public static StackFrameToken CreateToken(StackFrameKind kind, ImmutableArray leadingTrivia, VirtualCharSequence virtualChars) + => new(kind, leadingTrivia, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null); + + public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars) + => CreateTrivia(kind, virtualChars, ImmutableArray.Empty); + + public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars, ImmutableArray diagnostics) + => new(kind, virtualChars, diagnostics); +#nullable enable + + private static StackFrameKind GetKind(VirtualChar ch) + => ch.Value switch + { + '\n' => StackFrameKind.EndOfLine, + '&' => StackFrameKind.AmpersandToken, + '[' => StackFrameKind.OpenBracketToken, + ']' => StackFrameKind.CloseBracketToken, + '(' => StackFrameKind.OpenParenToken, + ')' => StackFrameKind.CloseParenToken, + '.' => StackFrameKind.DotToken, + '+' => StackFrameKind.PlusToken, + ',' => StackFrameKind.CommaToken, + ':' => StackFrameKind.ColonToken, + '=' => StackFrameKind.EqualsToken, + '>' => StackFrameKind.GreaterThanToken, + '<' => StackFrameKind.LessThanToken, + '-' => StackFrameKind.MinusToken, + '\'' => StackFrameKind.SingleQuoteToken, + '`' => StackFrameKind.GraveAccentToken, + '\\' => StackFrameKind.BackslashToken, + '/' => StackFrameKind.ForwardSlashToken, + _ => IsBlank(ch) ? StackFrameKind.WhitespaceToken : StackFrameKind.TextToken, + }; + + private static bool IsNumber(VirtualChar ch) + => ch.Value switch + { + '0' or '1' or '2' or '3' or '4' or '5' or '6' or '7' or '8' or '9' + => true, + _ => false + }; + + private ImmutableArray ScanLeadingTrivia(bool allowTrivia) + { + if (!allowTrivia) + { + return ImmutableArray.Empty; + } + + using var _ = ArrayBuilder.GetInstance(out var result); + + while (Position < Text.Length) + { + var atTrivia = ScanAtTrivia(); + if (atTrivia != null) + { + result.Add(atTrivia.Value); + continue; + } + + var whitespace = ScanWhitespace(); + if (whitespace != null) + { + result.Add(whitespace.Value); + continue; + } + + var inTrivia = ScanInTrivia(); + if (inTrivia != null) + { + result.Add(inTrivia.Value); + continue; + } + + break; + } + + return result.ToImmutable(); + } + + public StackFrameTrivia? ScanAtTrivia() + { + if (Position >= Text.Length) + { + return null; + } + + // TODO: Handle multiple languages? Right now we're going to only parse english + const string AtString = "at "; + + if (IsAt(AtString)) + { + var start = Position; + Position += AtString.Length; + + return CreateTrivia(StackFrameKind.AtTrivia, GetSubPatternToCurrentPos(start)); + } + + return null; + } + + private StackFrameTrivia? ScanWhitespace() + { + var start = Position; + while (Position < Text.Length && IsBlank(Text[Position])) + { + Position++; + } + + if (Position > start) + { + return CreateTrivia(StackFrameKind.WhitespaceTrivia, GetSubPatternToCurrentPos(start)); + } + + return null; + } + + public StackFrameTrivia? ScanInTrivia() + { + if (Position >= Text.Length) + { + return null; + } + + // TODO: Handle multiple languages? Right now we're going to only parse english + const string InString = " in "; + + if (IsAt(InString)) + { + var start = Position; + Position += InString.Length; + + return CreateTrivia(StackFrameKind.AtTrivia, GetSubPatternToCurrentPos(start)); + } + + return null; + } + + public static bool IsBlank(VirtualChar ch) + { + // List taken from the native regex parser. + switch (ch.Value) + { + case '\u0009': + case '\u000A': + case '\u000C': + case '\u000D': + case ' ': + return true; + default: + return false; + } + } + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs new file mode 100644 index 0000000000000..58f4e97357012 --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -0,0 +1,375 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; +using Roslyn.Utilities; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; + using StackFrameToken = EmbeddedSyntaxToken; + using StackFrameTrivia = EmbeddedSyntaxTrivia; + + internal abstract class StackFrameNode : EmbeddedSyntaxNode + { + protected StackFrameNode(StackFrameKind kind) : base(kind) + { + } + + public abstract void Accept(IStackFrameNodeVisitor visitor); + } + + /// + /// Root of all expression nodes. + /// + internal abstract class StackFrameExpressionNode : StackFrameNode + { + protected StackFrameExpressionNode(StackFrameKind kind) : base(kind) + { + } + } + + internal sealed class StackFrameMethodDeclarationNode : StackFrameExpressionNode + { + public readonly StackFrameMemberAccessExpressionNode MemberAccessExpression; + public readonly StackFrameTypeArgumentList? TypeArguments; + public readonly StackFrameArgumentList ArgumentList; + + internal StackFrameMethodDeclarationNode( + StackFrameMemberAccessExpressionNode memberAccessExpression, + StackFrameTypeArgumentList? typeArguments, + StackFrameArgumentList argumentList) + : base(StackFrameKind.MethodDeclaration) + { + MemberAccessExpression = memberAccessExpression; + TypeArguments = typeArguments; + ArgumentList = argumentList; + } + + internal override int ChildCount => TypeArguments is null ? 2 : 3; + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => MemberAccessExpression, + 1 => TypeArguments is null ? ArgumentList : TypeArguments, + 2 => TypeArguments is null ? throw new InvalidOperationException() : ArgumentList, + _ => throw new InvalidOperationException(), + }; + } + + internal sealed class StackFrameMemberAccessExpressionNode : StackFrameExpressionNode + { + public readonly StackFrameExpressionNode Expression; + public readonly StackFrameToken Operator; + public readonly StackFrameBaseIdentifierNode Identifier; + + public StackFrameMemberAccessExpressionNode(StackFrameExpressionNode expression, StackFrameToken operatorToken, StackFrameBaseIdentifierNode identifier) : base(StackFrameKind.MemberAccess) + { + Expression = expression; + Operator = operatorToken; + Identifier = identifier; + } + + internal override int ChildCount => 3; + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => Expression, + 1 => Operator, + 2 => Identifier, + _ => throw new InvalidOperationException() + }; + + internal StackFrameMemberAccessExpressionNode WithTrailingTrivia(StackFrameTrivia trivia) + => new(Expression, Operator, Identifier.WithTrailingTrivia(trivia)); + } + + internal abstract class StackFrameBaseIdentifierNode : StackFrameExpressionNode + { + protected StackFrameBaseIdentifierNode(StackFrameKind kind) : base(kind) + { + } + + internal abstract StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTrivia trailingTrivia); + } + + internal sealed class StackFrameIdentifierNode : StackFrameBaseIdentifierNode + { + public readonly StackFrameToken Identifier; + + internal override int ChildCount => 1; + + internal StackFrameIdentifierNode(StackFrameToken identifier) + : base(StackFrameKind.Identifier) + { + Identifier = identifier; + } + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => Identifier, + _ => throw new InvalidOperationException() + }; + + public override string ToString() + => Identifier.VirtualChars.CreateString(); + + internal override StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTrivia trailingTrivia) + => new StackFrameIdentifierNode(Identifier.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))); + } + + internal sealed class StackFrameGenericTypeIdentifier : StackFrameBaseIdentifierNode + { + public readonly StackFrameToken Identifier; + public readonly StackFrameToken ArityToken; + public readonly StackFrameToken ArityNumericToken; + + internal override int ChildCount => 3; + + internal StackFrameGenericTypeIdentifier(StackFrameToken identifier, StackFrameToken arityToken, StackFrameToken arityNumericToken) + : base(StackFrameKind.GenericTypeIdentifier) + { + Identifier = identifier; + ArityToken = arityToken; + ArityNumericToken = arityNumericToken; + } + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => Identifier, + 1 => ArityToken, + 2 => ArityNumericToken, + _ => throw new InvalidOperationException() + }; + + internal override StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTrivia trailingTrivia) + => new StackFrameGenericTypeIdentifier( + Identifier, + ArityToken, + ArityNumericToken.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))); + } + + internal sealed class StackFrameArrayExpressionNode : StackFrameExpressionNode + { + private readonly StackFrameExpressionNode _identifier; + private readonly ImmutableArray _arrayBrackets; + + public StackFrameArrayExpressionNode(StackFrameExpressionNode identifier, ImmutableArray arrayBrackets) + : base(StackFrameKind.ArrayExpression) + { + _identifier = identifier; + _arrayBrackets = arrayBrackets; + + Debug.Assert(arrayBrackets.All(t => t.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.CloseBracketToken)); + } + + internal override int ChildCount => _arrayBrackets.Length + 1; + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => _identifier, + _ => _arrayBrackets[index - 1] + }; + } + + internal sealed class StackFrameTypeArgumentList : StackFrameNode + { + public static readonly StackFrameTypeArgumentList Empty = new(ImmutableArray.Empty); + public StackFrameTypeArgumentList(ImmutableArray childNodesOrTokens) : base(StackFrameKind.TypeArgument) + { +#if DEBUG + // The list should contain an open token, at least one identifier, and a close token + Debug.Assert(childNodesOrTokens.Length >= 3); + for (var i = 0; i < childNodesOrTokens.Length; i++) + { + var nodeOrToken = childNodesOrTokens[i]; + + if (i == 0) + { + Debug.Assert(nodeOrToken.Token.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken); + continue; + } + + if (i == childNodesOrTokens.Length - 1) + { + var openToken = childNodesOrTokens[0].Token; + switch (openToken.Kind) + { + case StackFrameKind.OpenBracketToken: Debug.Assert(nodeOrToken.Token.Kind == StackFrameKind.CloseBracketToken); break; + case StackFrameKind.LessThanToken: Debug.Assert(nodeOrToken.Token.Kind == StackFrameKind.GreaterThanToken); break; + default: throw ExceptionUtilities.UnexpectedValue(openToken.Kind); + } + + continue; + } + + if (nodeOrToken.IsNode) + { + Debug.Assert(nodeOrToken.Node is StackFrameTypeArgument); + } + else + { + Debug.Assert(nodeOrToken.Token.Kind == StackFrameKind.CommaToken); + } + } +#endif + _childNodesOrTokens = childNodesOrTokens; + } + + private readonly ImmutableArray _childNodesOrTokens; + + internal override int ChildCount => _childNodesOrTokens.Length; + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => _childNodesOrTokens[index]; + } + + internal sealed class StackFrameTypeArgument : StackFrameBaseIdentifierNode + { + public readonly StackFrameToken Identifier; + + internal override int ChildCount => 1; + + internal StackFrameTypeArgument(StackFrameToken identifier) + : base(StackFrameKind.TypeIdentifier) + { + Identifier = identifier; + } + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => Identifier, + _ => throw new InvalidOperationException() + }; + + internal override StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTrivia trailingTrivia) + => new StackFrameTypeArgument(Identifier.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))); + } + + internal sealed class StackFrameArgumentList : StackFrameNode + { + private readonly ImmutableArray _childNodesOrTokens; + + public StackFrameArgumentList(ImmutableArray childNodesOrTokens) : base(StackFrameKind.ArgumentList) + { +#if DEBUG + // The list should contain an open and close token, and optionally parameter definitions (each being type and name) + Debug.Assert(childNodesOrTokens.Length == 2 || childNodesOrTokens.Length >= 4); + for (var i = 0; i < childNodesOrTokens.Length; i++) + { + var nodeOrToken = childNodesOrTokens[i]; + + if (i == 0) + { + Debug.Assert(nodeOrToken.Token.Kind is StackFrameKind.OpenParenToken); + continue; + } + + if (i == childNodesOrTokens.Length - 1) + { + Debug.Assert(nodeOrToken.Token.Kind is StackFrameKind.CloseParenToken); + continue; + } + + if (nodeOrToken.IsNode) + { + Debug.Assert(nodeOrToken.Node is StackFrameIdentifierNode or StackFrameMemberAccessExpressionNode or StackFrameArrayExpressionNode); + } + else + { + Debug.Assert(nodeOrToken.Token.Kind is StackFrameKind.CommaToken or StackFrameKind.TextToken); + } + } +#endif + _childNodesOrTokens = childNodesOrTokens; + } + + internal override int ChildCount => _childNodesOrTokens.Length; + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => _childNodesOrTokens[index]; + } + + internal sealed class StackFrameFileInformationNode : StackFrameNode + { + public StackFrameFileInformationNode(StackFrameKind kind) : base(kind) + { + } + + internal override int ChildCount => throw new NotImplementedException(); + + public override void Accept(IStackFrameNodeVisitor visitor) + { + throw new NotImplementedException(); + } + + internal override StackFrameNodeOrToken ChildAt(int index) + { + throw new NotImplementedException(); + } + } + + /// + /// Represents a chunk of text (can be multiple characters) + /// + internal sealed class StackFrameTextNode : StackFrameExpressionNode + { + public StackFrameTextNode(StackFrameToken textToken) + : base(StackFrameKind.Text) + { + Debug.Assert(textToken.Kind == StackFrameKind.TextToken); + TextToken = textToken; + } + + public StackFrameToken TextToken { get; } + + internal override int ChildCount => 1; + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => TextToken, + _ => throw new InvalidOperationException(), + }; + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs new file mode 100644 index 0000000000000..2bb9cd21a88c5 --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -0,0 +1,293 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; +using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; +using Microsoft.CodeAnalysis.PooledObjects; +using Roslyn.Utilities; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; + using StackFrameToken = EmbeddedSyntaxToken; + using StackFrameTrivia = EmbeddedSyntaxTrivia; + + internal struct StackFrameParser + { + private StackFrameLexer _lexer; + private StackFrameToken CurrentToken => _lexer.PreviousCharAsToken(); + + private StackFrameParser(VirtualCharSequence text) + { + _lexer = new(text); + } + + /// + /// Given an input text, and set of options, parses out a fully representative syntax tree + /// and list of diagnostics. Parsing should always succeed, except in the case of the stack + /// overflowing. + /// + public static StackFrameTree? TryParse(VirtualCharSequence text) + { + if (text.IsDefault) + { + return null; + } + + try + { + return new StackFrameParser(text).ParseTree(); + } + catch (InsufficientExecutionStackException) + { + return null; + } + } + + /// + /// Constructs a and calls + /// + public static StackFrameTree? TryParse(string text) + => TryParse(VirtualCharSequence.Create(0, text)); + + private StackFrameTree? ParseTree() + { + var atTrivia = _lexer.ScanAtTrivia(); + + var methodDeclaration = ParseMethodDeclaration(addLeadingWhitespaceAsTrivia: !atTrivia.HasValue); + if (methodDeclaration is null) + { + return null; + } + + var inTrivia = _lexer.ScanInTrivia(); + var fileInformationExpression = inTrivia.HasValue + ? ParseFileInformation() + : null; + var trailingTrivia = _lexer.ScanTrailingTrivia(); + + Debug.Assert(_lexer.Position == _lexer.Text.Length); + Debug.Assert(_lexer.CurrentCharAsToken().Kind == StackFrameKind.EndOfLine); + + var root = new StackFrameCompilationUnit(atTrivia, methodDeclaration, inTrivia, fileInformationExpression, trailingTrivia); + + return new StackFrameTree( + _lexer.Text, root, ImmutableArray.Empty); + } + + private StackFrameMethodDeclarationNode? ParseMethodDeclaration(bool addLeadingWhitespaceAsTrivia) + { + var identifierExpression = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: addLeadingWhitespaceAsTrivia); + if (identifierExpression is not StackFrameMemberAccessExpressionNode memberAccessExpression) + { + return null; + } + + var typeArguments = ParseTypeArguments(); + var arguments = ParseMethodArguments(); + + if (arguments is null) + { + return null; + } + + return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, arguments); + } + + /// + /// Parses an identifier expression which could either be a or + /// + private StackFrameExpressionNode? ParseIdentifierExpression(bool addLeadingWhitespaceAsTrivia) + { + Queue<(StackFrameBaseIdentifierNode identifier, StackFrameToken separator)> typeIdentifierNodes = new(); + + // If allowed, add the leading whitespace as trivia to the first + // identifier token in the expression + var leadingTrivia = AllowWhitespace(CurrentToken) + ? _lexer.ScanWhiteSpace(includePrevious: addLeadingWhitespaceAsTrivia && CurrentToken.Kind == StackFrameKind.WhitespaceToken) + : null; + + var currentIdentifier = _lexer.ScanIdentifier(); + while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) + { + StackFrameToken? arity = null; + if (CurrentToken.Kind == StackFrameKind.GraveAccentToken) + { + arity = _lexer.ScanTypeArity(); + } + + if (leadingTrivia.HasValue) + { + currentIdentifier = currentIdentifier.Value.With(leadingTrivia: ImmutableArray.Create(leadingTrivia.Value)); + leadingTrivia = null; + } + + StackFrameBaseIdentifierNode identifierNode = arity.HasValue + ? new StackFrameGenericTypeIdentifier(currentIdentifier.Value, CurrentToken, arity.Value) + : new StackFrameIdentifierNode(currentIdentifier.Value); + + typeIdentifierNodes.Enqueue((identifierNode, CurrentToken)); + + if (CurrentToken.Kind != StackFrameKind.DotToken) + { + break; + } + + currentIdentifier = _lexer.ScanIdentifier(); + } + + if (typeIdentifierNodes.Count == 0) + { + return null; + } + + var trailingTrivia = CurrentToken.Kind == StackFrameKind.WhitespaceToken + ? _lexer.ScanWhiteSpace(includePrevious: true) + : null; + + if (typeIdentifierNodes.Count == 1) + { + var identifierNode = typeIdentifierNodes.Dequeue().identifier; + return trailingTrivia.HasValue + ? identifierNode.WithTrailingTrivia(trailingTrivia.Value) + : identifierNode; + } + + // Construct the member access expression from the identifiers + var (firstIdentifierNode, firstSeparator) = typeIdentifierNodes.Dequeue(); + var currentSeparator = firstSeparator; + + StackFrameMemberAccessExpressionNode? memberAccessExpression = null; + + while (typeIdentifierNodes.Count != 0) + { + var previousSeparator = currentSeparator; + (var currentIdentifierNode, currentSeparator) = typeIdentifierNodes.Dequeue(); + + StackFrameExpressionNode leftHandNode = memberAccessExpression is null + ? firstIdentifierNode + : memberAccessExpression; + + memberAccessExpression = new StackFrameMemberAccessExpressionNode(leftHandNode, previousSeparator, currentIdentifierNode); + } + + RoslynDebug.AssertNotNull(memberAccessExpression); + + return trailingTrivia.HasValue + ? memberAccessExpression.WithTrailingTrivia(trailingTrivia.Value) + : memberAccessExpression; + + static bool AllowWhitespace(StackFrameToken token) + => token.Kind + is StackFrameKind.WhitespaceTrivia + or StackFrameKind.CommaToken + or StackFrameKind.OpenBracketToken + or StackFrameKind.CloseBracketToken + or StackFrameKind.OpenParenToken + or StackFrameKind.CloseParenToken + or StackFrameKind.WhitespaceToken; + } + + private StackFrameTypeArgumentList? ParseTypeArguments() + { + if (CurrentToken.Kind is not StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken) + { + return null; + } + + var useCloseBracket = CurrentToken.Kind is StackFrameKind.OpenBracketToken; + + Func stopParsing = (token) => useCloseBracket ? token.Kind == StackFrameKind.CloseBracketToken : token.Kind == StackFrameKind.GreaterThanToken; + + using var _ = ArrayBuilder.GetInstance(out var builder); + builder.Add(CurrentToken); + + var currentIdentifier = _lexer.ScanIdentifier(); + + while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) + { + builder.Add(currentIdentifier.Value); + builder.Add(CurrentToken); + + if (stopParsing(CurrentToken)) + { + break; + } + + currentIdentifier = _lexer.ScanIdentifier(); + } + + return new StackFrameTypeArgumentList(builder.ToImmutable()); + } + + private StackFrameArgumentList? ParseMethodArguments() + { + if (CurrentToken.Kind != StackFrameKind.OpenParenToken) + { + return null; + } + + using var _ = ArrayBuilder.GetInstance(out var builder); + builder.Add(CurrentToken); + + var identifier = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: true); + while (identifier is not null) + { + // Check if there's an array type for the identifier + if (CurrentToken.Kind == StackFrameKind.OpenBracketToken) + { + builder.Add(ParseArrayIdentifier(identifier)); + } + else + { + builder.Add(identifier); + } + + // Let whitespace get added as trivia to the identifiers + // for the parameters + if (CurrentToken.Kind != StackFrameKind.WhitespaceToken) + { + builder.Add(CurrentToken); + } + + if (CurrentToken.Kind == StackFrameKind.CloseParenToken) + { + return new StackFrameArgumentList(builder.ToImmutable()); + } + + var addLeadingWhitespaceAsTrivia = identifier.ChildAt(identifier.ChildCount - 1).Token.TrailingTrivia.IsDefaultOrEmpty; + identifier = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: addLeadingWhitespaceAsTrivia); + } + + if (CurrentToken.Kind != StackFrameKind.CloseParenToken) + { + // If we parsed identifiers but never got to a closing portion for the argument list + // we need to bail and return null so we know that this isn't valid + return null; + } + + return new StackFrameArgumentList(builder.ToImmutable()); + } + + private StackFrameExpressionNode ParseArrayIdentifier(StackFrameExpressionNode identifier) + { + if (CurrentToken.Kind != StackFrameKind.OpenBracketToken) + { + throw new InvalidOperationException(); + } + + var arrayBrackets = _lexer.ScanArrayBrackets(); + return new StackFrameArrayExpressionNode(identifier, arrayBrackets); + } + + public StackFrameFileInformationNode ParseFileInformation() + { + throw new NotImplementedException(); + } + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs new file mode 100644 index 0000000000000..eaf73468d17fa --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; +using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + internal class StackFrameTree + { + public StackFrameTree(VirtualCharSequence text, StackFrameCompilationUnit root, ImmutableArray embeddedDiagnostics) + { + Text = text; + Root = root; + EmbeddedDiagnostics = embeddedDiagnostics; + } + + public VirtualCharSequence Text { get; } + public StackFrameCompilationUnit Root { get; } + public ImmutableArray EmbeddedDiagnostics { get; } + } +} From 872a90d18b0e1f4500147546b95e382db5de7fbe Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 5 Oct 2021 17:27:17 -0700 Subject: [PATCH 02/59] * Add generic method tests. * Fix some of the node type hierarchy issues * Make Open/Close tokens as requirements for TypeArgumentList and ParameterList * Add comments with simple input/output on methods * Make it so NodeOrToken can represent null --- .../StackFrameParserTests.Utilities.cs | 43 ++++- .../StackFrame/StackFrameParserTests.cs | 58 +++++++ .../StackFrame/IStackFrameNodeVisitor.cs | 2 +- .../StackFrame/StackFrameKind.cs | 2 +- .../StackFrame/StackFrameNodeDefinitions.cs | 159 +++++++++--------- .../StackFrame/StackFrameParser.cs | 97 ++++++++--- .../Common/EmbeddedSyntaxNodeOrToken.cs | 5 +- 7 files changed, 250 insertions(+), 116 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 51d7cd256d129..52210a0e21d8d 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -39,7 +39,9 @@ private static ImmutableArray CreateTriviaArray(StackFrameTriv private static readonly StackFrameToken CloseParenToken = CreateToken(StackFrameKind.CloseParenToken, ")"); private static readonly StackFrameToken OpenBracketToken = CreateToken(StackFrameKind.OpenBracketToken, "["); private static readonly StackFrameToken CloseBracketToken = CreateToken(StackFrameKind.CloseBracketToken, "]"); - private static readonly StackFrameToken SpaceToken = CreateToken(StackFrameKind.TextToken, " "); + private static readonly StackFrameToken LessThanToken = CreateToken(StackFrameKind.LessThanToken, "<"); + private static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); + private static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); private static readonly StackFrameTrivia SpaceTrivia = CreateTrivia(StackFrameKind.WhitespaceTrivia, " "); @@ -199,15 +201,12 @@ private static void AssertEqual(StackFrameTrivia expected, StackFrameTrivia actu Assert.Equal(expected.VirtualChars.CreateString(), actual.VirtualChars.CreateString()); } - private static StackFrameArgumentList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) - { - var nodesWithParens = nodesOrTokens.Prepend(OpenParenToken).Append(CloseParenToken).ToImmutableArray(); - return new StackFrameArgumentList(nodesWithParens); - } + private static StackFrameParameterList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) + => new(OpenParenToken, nodesOrTokens.ToImmutableArray(), CloseParenToken); private static StackFrameMethodDeclarationNode MethodDeclaration( StackFrameMemberAccessExpressionNode memberAccessExpression, - StackFrameArgumentList argumentList, + StackFrameParameterList argumentList, StackFrameTypeArgumentList? typeArgumnets = null) { return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArgumnets, argumentList); @@ -221,5 +220,35 @@ private static StackFrameIdentifierNode Identifier(string identifierName, Immuta private static StackFrameArrayExpressionNode ArrayExpression(StackFrameExpressionNode identifier, params StackFrameToken[] arrayTokens) => new(identifier, arrayTokens.ToImmutableArray()); + + private static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) + => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.TextToken, arity.ToString())); + + private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgument[] typeArguments) + { + using var _ = PooledObjects.ArrayBuilder.GetInstance(out var builder); + var openToken = useBrackets ? OpenBracketToken : LessThanToken; + var closeToken = useBrackets ? CloseBracketToken : GreaterThanToken; + + var isFirst = true; + foreach (var typeArgument in typeArguments) + { + if (isFirst) + { + isFirst = false; + } + else + { + builder.Add(CommaToken); + } + + builder.Add(typeArgument); + } + + return new(openToken, builder.ToImmutable(), closeToken); + } + + private static StackFrameTypeArgument TypeArgument(string identifier) + => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); } } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 275a141cc41af..3ddc0d0a8b286 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -99,5 +99,63 @@ public void TestMethodArrayParam() AssertEqual(expectedMethodDeclaration, methodDeclaration); } + + [Fact] + public void TestGenericMethod_Brackets() + { + var input = @"at ConsoleApp4.MyClass.M[T](T t)"; + var tree = StackFrameParser.TryParse(input); + AssertEx.NotNull(tree); + + Assert.True(tree.Root.AtTrivia.HasValue); + Assert.False(tree.Root.InTrivia.HasValue); + Assert.Null(tree.Root.FileInformationExpression); + Assert.False(tree.Root.TrailingTrivia.HasValue); + + var methodDeclaration = tree.Root.MethodDeclaration; + + var expectedMethodDeclaration = MethodDeclaration( + MemberAccessExpression( + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + typeArgumnets: TypeArgumentList(useBrackets: true, TypeArgument("T")), + argumentList: ArgumentList( + Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("t")) + ); + + AssertEqual(expectedMethodDeclaration, methodDeclaration); + } + + [Fact] + public void TestGenericMethod() + { + var input = @"at ConsoleApp4.MyClass.M(T t)"; + var tree = StackFrameParser.TryParse(input); + AssertEx.NotNull(tree); + + Assert.True(tree.Root.AtTrivia.HasValue); + Assert.False(tree.Root.InTrivia.HasValue); + Assert.Null(tree.Root.FileInformationExpression); + Assert.False(tree.Root.TrailingTrivia.HasValue); + + var methodDeclaration = tree.Root.MethodDeclaration; + + var expectedMethodDeclaration = MethodDeclaration( + MemberAccessExpression( + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + typeArgumnets: TypeArgumentList(useBrackets: false, TypeArgument("T")), + argumentList: ArgumentList( + Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("t")) + ); + + AssertEqual(expectedMethodDeclaration, methodDeclaration); + } } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index 3b8a9d12d50cc..31eaef511dc75 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -14,7 +14,7 @@ internal interface IStackFrameNodeVisitor void Visit(StackFrameMethodDeclarationNode stackFrameMethodDeclarationNode); void Visit(StackFrameMemberAccessExpressionNode stackFrameMemberAccessExpressionNode); void Visit(StackFrameTypeArgumentList stackFrameTypeArguments); - void Visit(StackFrameArgumentList stackFrameArgumentList); + void Visit(StackFrameParameterList stackFrameArgumentList); void Visit(StackFrameIdentifierNode stackFrameIdentifierNode); void Visit(StackFrameGenericTypeIdentifier stackFrameGenericTypeIdentifier); void Visit(StackFrameTypeArgument stackFrameTypeArgument); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 936572069188f..82caef5cb0eb8 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -17,7 +17,7 @@ internal enum StackFrameKind GenericTypeIdentifier, TypeArgument, TypeIdentifier, - ArgumentList, + ParameterList, ArrayExpression, // Tokens diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index 58f4e97357012..6b3455ec52f22 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -36,16 +36,16 @@ protected StackFrameExpressionNode(StackFrameKind kind) : base(kind) } } - internal sealed class StackFrameMethodDeclarationNode : StackFrameExpressionNode + internal sealed class StackFrameMethodDeclarationNode : StackFrameNode { public readonly StackFrameMemberAccessExpressionNode MemberAccessExpression; public readonly StackFrameTypeArgumentList? TypeArguments; - public readonly StackFrameArgumentList ArgumentList; + public readonly StackFrameParameterList ArgumentList; internal StackFrameMethodDeclarationNode( StackFrameMemberAccessExpressionNode memberAccessExpression, StackFrameTypeArgumentList? typeArguments, - StackFrameArgumentList argumentList) + StackFrameParameterList argumentList) : base(StackFrameKind.MethodDeclaration) { MemberAccessExpression = memberAccessExpression; @@ -53,19 +53,19 @@ internal StackFrameMethodDeclarationNode( ArgumentList = argumentList; } - internal override int ChildCount => TypeArguments is null ? 2 : 3; + internal override int ChildCount => 3; public override void Accept(IStackFrameNodeVisitor visitor) => visitor.Visit(this); internal override StackFrameNodeOrToken ChildAt(int index) - => index switch - { - 0 => MemberAccessExpression, - 1 => TypeArguments is null ? ArgumentList : TypeArguments, - 2 => TypeArguments is null ? throw new InvalidOperationException() : ArgumentList, - _ => throw new InvalidOperationException(), - }; + => index switch + { + 0 => MemberAccessExpression, + 1 => TypeArguments, + 2 => ArgumentList, + _ => throw new InvalidOperationException(), + }; } internal sealed class StackFrameMemberAccessExpressionNode : StackFrameExpressionNode @@ -201,57 +201,49 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameTypeArgumentList : StackFrameNode { - public static readonly StackFrameTypeArgumentList Empty = new(ImmutableArray.Empty); - public StackFrameTypeArgumentList(ImmutableArray childNodesOrTokens) : base(StackFrameKind.TypeArgument) + public readonly StackFrameToken OpenToken; + public readonly StackFrameToken CloseToken; + + public StackFrameTypeArgumentList(StackFrameToken openToken, ImmutableArray childNodesOrTokens, StackFrameToken closeToken) : base(StackFrameKind.TypeArgument) { -#if DEBUG - // The list should contain an open token, at least one identifier, and a close token - Debug.Assert(childNodesOrTokens.Length >= 3); - for (var i = 0; i < childNodesOrTokens.Length; i++) - { - var nodeOrToken = childNodesOrTokens[i]; - - if (i == 0) - { - Debug.Assert(nodeOrToken.Token.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken); - continue; - } - - if (i == childNodesOrTokens.Length - 1) - { - var openToken = childNodesOrTokens[0].Token; - switch (openToken.Kind) - { - case StackFrameKind.OpenBracketToken: Debug.Assert(nodeOrToken.Token.Kind == StackFrameKind.CloseBracketToken); break; - case StackFrameKind.LessThanToken: Debug.Assert(nodeOrToken.Token.Kind == StackFrameKind.GreaterThanToken); break; - default: throw ExceptionUtilities.UnexpectedValue(openToken.Kind); - } - - continue; - } - - if (nodeOrToken.IsNode) - { - Debug.Assert(nodeOrToken.Node is StackFrameTypeArgument); - } - else - { - Debug.Assert(nodeOrToken.Token.Kind == StackFrameKind.CommaToken); - } - } -#endif + Debug.Assert(openToken.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken); + Debug.Assert(openToken.Kind == StackFrameKind.OpenBracketToken ? closeToken.Kind == StackFrameKind.CloseBracketToken : closeToken.Kind == StackFrameKind.GreaterThanToken); + Debug.Assert(childNodesOrTokens.All(nodeOrToken => nodeOrToken.IsNode + ? nodeOrToken.Node is StackFrameTypeArgument + : nodeOrToken.Token.Kind == StackFrameKind.CommaToken)); + + OpenToken = openToken; + CloseToken = closeToken; _childNodesOrTokens = childNodesOrTokens; + ChildCount = childNodesOrTokens.Length + 2; } private readonly ImmutableArray _childNodesOrTokens; - internal override int ChildCount => _childNodesOrTokens.Length; + internal override int ChildCount { get; } public override void Accept(IStackFrameNodeVisitor visitor) => visitor.Visit(this); internal override StackFrameNodeOrToken ChildAt(int index) - => _childNodesOrTokens[index]; + { + if (index >= ChildCount) + { + throw new InvalidOperationException(); + } + + if (index == 0) + { + return OpenToken; + } + + if (index == ChildCount - 1) + { + return CloseToken; + } + + return _childNodesOrTokens[index - 1]; + } } internal sealed class StackFrameTypeArgument : StackFrameBaseIdentifierNode @@ -280,51 +272,50 @@ internal override StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTriv => new StackFrameTypeArgument(Identifier.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))); } - internal sealed class StackFrameArgumentList : StackFrameNode + internal sealed class StackFrameParameterList : StackFrameNode { + public readonly StackFrameToken OpenParen; + public readonly StackFrameToken CloseParen; private readonly ImmutableArray _childNodesOrTokens; - public StackFrameArgumentList(ImmutableArray childNodesOrTokens) : base(StackFrameKind.ArgumentList) + public StackFrameParameterList(StackFrameToken openParen, ImmutableArray childNodesOrTokens, StackFrameToken closeParen) : base(StackFrameKind.ParameterList) { -#if DEBUG - // The list should contain an open and close token, and optionally parameter definitions (each being type and name) - Debug.Assert(childNodesOrTokens.Length == 2 || childNodesOrTokens.Length >= 4); - for (var i = 0; i < childNodesOrTokens.Length; i++) - { - var nodeOrToken = childNodesOrTokens[i]; - - if (i == 0) - { - Debug.Assert(nodeOrToken.Token.Kind is StackFrameKind.OpenParenToken); - continue; - } - - if (i == childNodesOrTokens.Length - 1) - { - Debug.Assert(nodeOrToken.Token.Kind is StackFrameKind.CloseParenToken); - continue; - } - - if (nodeOrToken.IsNode) - { - Debug.Assert(nodeOrToken.Node is StackFrameIdentifierNode or StackFrameMemberAccessExpressionNode or StackFrameArrayExpressionNode); - } - else - { - Debug.Assert(nodeOrToken.Token.Kind is StackFrameKind.CommaToken or StackFrameKind.TextToken); - } - } -#endif + Debug.Assert(openParen.Kind == StackFrameKind.OpenParenToken); + Debug.Assert(closeParen.Kind == StackFrameKind.CloseParenToken); + Debug.Assert(childNodesOrTokens.All(nodeOrToken => nodeOrToken.IsNode + ? nodeOrToken.Node is StackFrameIdentifierNode or StackFrameMemberAccessExpressionNode or StackFrameArrayExpressionNode + : nodeOrToken.Token.Kind is StackFrameKind.CommaToken)); + + OpenParen = openParen; + CloseParen = closeParen; _childNodesOrTokens = childNodesOrTokens; + ChildCount = _childNodesOrTokens.Length + 2; } - internal override int ChildCount => _childNodesOrTokens.Length; + internal override int ChildCount { get; } public override void Accept(IStackFrameNodeVisitor visitor) => visitor.Visit(this); internal override StackFrameNodeOrToken ChildAt(int index) - => _childNodesOrTokens[index]; + { + if (index >= ChildCount) + { + throw new InvalidOperationException(); + } + + if (index == 0) + { + return OpenParen; + } + + if (index == ChildCount - 1) + { + return CloseParen; + } + + return _childNodesOrTokens[index - 1]; + } } internal sealed class StackFrameFileInformationNode : StackFrameNode diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 2bb9cd21a88c5..69fe71d84e96a 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -20,6 +20,11 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame internal struct StackFrameParser { private StackFrameLexer _lexer; + + /// + /// The current token that has been consumed by the parse. Note that the lexer position + /// will be one character ahead to represent the next token to be consumed. + /// private StackFrameToken CurrentToken => _lexer.PreviousCharAsToken(); private StackFrameParser(VirtualCharSequence text) @@ -55,6 +60,9 @@ private StackFrameParser(VirtualCharSequence text) public static StackFrameTree? TryParse(string text) => TryParse(VirtualCharSequence.Create(0, text)); + /// + /// Attempts to parse the full tree. Returns null on malformed data + /// private StackFrameTree? ParseTree() { var atTrivia = _lexer.ScanAtTrivia(); @@ -80,6 +88,12 @@ private StackFrameParser(VirtualCharSequence text) _lexer.Text, root, ImmutableArray.Empty); } + /// + /// Attempts to parse the full method declaration, optionally adding leading whitespace as trivia. Includes + /// all of the generic indicators for types, + /// + /// Ex: [|MyClass.MyMethod(string s)|] + /// private StackFrameMethodDeclarationNode? ParseMethodDeclaration(bool addLeadingWhitespaceAsTrivia) { var identifierExpression = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: addLeadingWhitespaceAsTrivia); @@ -89,7 +103,7 @@ private StackFrameParser(VirtualCharSequence text) } var typeArguments = ParseTypeArguments(); - var arguments = ParseMethodArguments(); + var arguments = ParseMethodParameters(); if (arguments is null) { @@ -100,8 +114,22 @@ private StackFrameParser(VirtualCharSequence text) } /// - /// Parses an identifier expression which could either be a or + /// Parses an identifier expression which could either be a or . Combines + /// identifiers that are separated by into . + /// + /// Identifiers will be parsed for arity but not generic type arguments. + /// + /// All of the following are valid identifiers, where "$$" marks the parsing starting point, and "[|" + "|]" mark the endpoints of the parsed identifier including trivia + /// * [|$$MyNamespace.MyClass.MyMethod|](string s) + /// * MyClass.MyMethod([|$$string |]s) + /// * MyClass.MyMethod(string [|$$s|]) // = false + /// * MyClass.MyMethod(string[| $$s|]) // = true + /// * [|$$MyClass`1.MyMethod|](string s) + /// * [|$$MyClass.MyMethod|][T](T t) /// + /// + /// Set to true if leading whitespaces before the identifier should be considered leading trivia. + /// private StackFrameExpressionNode? ParseIdentifierExpression(bool addLeadingWhitespaceAsTrivia) { Queue<(StackFrameBaseIdentifierNode identifier, StackFrameToken separator)> typeIdentifierNodes = new(); @@ -193,39 +221,63 @@ or StackFrameKind.CloseParenToken or StackFrameKind.WhitespaceToken; } + /// + /// Type arguments for stacks are only valid on method declarations, and can have either '[' or '<' as the + /// starting character depending on output source. + /// + /// ex: MyNamespace.MyClass.MyMethod[T](T t) + /// + /// Assumes the identifier "MyMethod" has already been parsed, and the type arguments will need to be parsed. + /// Returns null if no type arguments are found or if they are malformed. + /// private StackFrameTypeArgumentList? ParseTypeArguments() { - if (CurrentToken.Kind is not StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken) + if (CurrentToken.Kind is not (StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken)) { return null; } - var useCloseBracket = CurrentToken.Kind is StackFrameKind.OpenBracketToken; - - Func stopParsing = (token) => useCloseBracket ? token.Kind == StackFrameKind.CloseBracketToken : token.Kind == StackFrameKind.GreaterThanToken; + var openToken = CurrentToken; + var useCloseBracket = openToken.Kind is StackFrameKind.OpenBracketToken; using var _ = ArrayBuilder.GetInstance(out var builder); - builder.Add(CurrentToken); - var currentIdentifier = _lexer.ScanIdentifier(); + StackFrameToken? closeToken = null; while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { - builder.Add(currentIdentifier.Value); - builder.Add(CurrentToken); + builder.Add(new StackFrameTypeArgument(currentIdentifier.Value)); - if (stopParsing(CurrentToken)) + if ((useCloseBracket && CurrentToken.Kind == StackFrameKind.CloseBracketToken) + || CurrentToken.Kind == StackFrameKind.GreaterThanToken) { + closeToken = CurrentToken; + + // Consume the token and move on to the next one, since it is already added + // to the list of items for the TypeArgumentList + _lexer.Position++; break; } + builder.Add(CurrentToken); currentIdentifier = _lexer.ScanIdentifier(); } - return new StackFrameTypeArgumentList(builder.ToImmutable()); + if (!closeToken.HasValue) + { + return null; + } + + return new StackFrameTypeArgumentList(openToken, builder.ToImmutable(), closeToken.Value); } - private StackFrameArgumentList? ParseMethodArguments() + /// + /// MyNamespace.MyClass.MyMethod[|(string s1, string s2, int i1)|] + /// Takes parameter declarations from method text and parses them into a . + /// + /// Returns null in cases where the input is malformed. + /// + private StackFrameParameterList? ParseMethodParameters() { if (CurrentToken.Kind != StackFrameKind.OpenParenToken) { @@ -233,7 +285,7 @@ or StackFrameKind.CloseParenToken } using var _ = ArrayBuilder.GetInstance(out var builder); - builder.Add(CurrentToken); + var openParen = CurrentToken; var identifier = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: true); while (identifier is not null) @@ -248,6 +300,11 @@ or StackFrameKind.CloseParenToken builder.Add(identifier); } + if (CurrentToken.Kind == StackFrameKind.CloseParenToken) + { + return new StackFrameParameterList(openParen, builder.ToImmutable(), CurrentToken); + } + // Let whitespace get added as trivia to the identifiers // for the parameters if (CurrentToken.Kind != StackFrameKind.WhitespaceToken) @@ -255,11 +312,6 @@ or StackFrameKind.CloseParenToken builder.Add(CurrentToken); } - if (CurrentToken.Kind == StackFrameKind.CloseParenToken) - { - return new StackFrameArgumentList(builder.ToImmutable()); - } - var addLeadingWhitespaceAsTrivia = identifier.ChildAt(identifier.ChildCount - 1).Token.TrailingTrivia.IsDefaultOrEmpty; identifier = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: addLeadingWhitespaceAsTrivia); } @@ -271,9 +323,14 @@ or StackFrameKind.CloseParenToken return null; } - return new StackFrameArgumentList(builder.ToImmutable()); + return new StackFrameParameterList(openParen, builder.ToImmutable(), CurrentToken); } + /// + /// Given an input like "string[]" where "string" is the existing + /// passed in, converts it into by parsing the array portion + /// of the identifier. + /// private StackFrameExpressionNode ParseArrayIdentifier(StackFrameExpressionNode identifier) { if (CurrentToken.Kind != StackFrameKind.OpenBracketToken) diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxNodeOrToken.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxNodeOrToken.cs index 7861db94a9b91..8024776f059b4 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxNodeOrToken.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxNodeOrToken.cs @@ -15,9 +15,8 @@ internal struct EmbeddedSyntaxNodeOrToken public readonly TSyntaxNode? Node; public readonly EmbeddedSyntaxToken Token; - private EmbeddedSyntaxNodeOrToken(TSyntaxNode node) : this() + private EmbeddedSyntaxNodeOrToken(TSyntaxNode? node) : this() { - RoslynDebug.AssertNotNull(node); Node = node; } @@ -30,7 +29,7 @@ private EmbeddedSyntaxNodeOrToken(EmbeddedSyntaxToken token) : this [MemberNotNullWhen(true, nameof(Node))] public bool IsNode => Node != null; - public static implicit operator EmbeddedSyntaxNodeOrToken(TSyntaxNode node) + public static implicit operator EmbeddedSyntaxNodeOrToken(TSyntaxNode? node) => new(node); public static implicit operator EmbeddedSyntaxNodeOrToken(EmbeddedSyntaxToken token) From ba21c9ec6c0e8d416ba57d998745092fddfbfc41 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 5 Oct 2021 17:28:45 -0700 Subject: [PATCH 03/59] Update src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs Co-authored-by: CyrusNajmabadi --- .../Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 936572069188f..762f75f671e77 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -45,7 +45,7 @@ internal enum StackFrameKind // Trivia WhitespaceTrivia, AtTrivia, // "at " portion of the stack frame - InTrivia, // optionsal " in " portion of the stack frame + InTrivia, // optional " in " portion of the stack frame TrailingTrivia, // All trailing text that is not syntactically relavent } } From 9c04adbbbecdcc0d54d182b354f1fa581f9d57e8 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 5 Oct 2021 23:07:22 -0700 Subject: [PATCH 04/59] Restructure tests so that each tree is evaluated more thoroughly --- .../StackFrameParserTests.Generators.cs | 101 ++++++++ .../StackFrameParserTests.Utilities.cs | 215 +++++++++++------- .../StackFrame/StackFrameParserTests.cs | 190 ++++++---------- 3 files changed, 305 insertions(+), 201 deletions(-) create mode 100644 src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs new file mode 100644 index 0000000000000..9e33df93dfe1c --- /dev/null +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -0,0 +1,101 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; +using Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame; +using Roslyn.Test.Utilities; +using Xunit; +using System; + +namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame +{ + using StackFrameToken = EmbeddedSyntaxToken; + using StackFrameTrivia = EmbeddedSyntaxTrivia; + using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; + + public partial class StackFrameParserTests + { + private static StackFrameToken CreateToken(StackFrameKind kind, string s, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + => new( + kind, + leadingTrivia.IsDefaultOrEmpty ? ImmutableArray.Empty : leadingTrivia, + CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, s), + trailingTrivia.IsDefaultOrEmpty ? ImmutableArray.Empty : trailingTrivia, + ImmutableArray.Empty, + value: null!); + + private static StackFrameTrivia CreateTrivia(StackFrameKind kind, string s) + => new(kind, CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, s), ImmutableArray.Empty); + + private static ImmutableArray CreateTriviaArray(StackFrameTrivia trivia) + => ImmutableArray.Create(trivia); + + private static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); + private static readonly StackFrameToken CommaToken = CreateToken(StackFrameKind.CommaToken, ","); + private static readonly StackFrameToken OpenParenToken = CreateToken(StackFrameKind.OpenParenToken, "("); + private static readonly StackFrameToken CloseParenToken = CreateToken(StackFrameKind.CloseParenToken, ")"); + private static readonly StackFrameToken OpenBracketToken = CreateToken(StackFrameKind.OpenBracketToken, "["); + private static readonly StackFrameToken CloseBracketToken = CreateToken(StackFrameKind.CloseBracketToken, "]"); + private static readonly StackFrameToken LessThanToken = CreateToken(StackFrameKind.LessThanToken, "<"); + private static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); + private static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); + + private static readonly StackFrameTrivia SpaceTrivia = CreateTrivia(StackFrameKind.WhitespaceTrivia, " "); + + private static StackFrameParameterList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) + => new(OpenParenToken, nodesOrTokens.ToImmutableArray(), CloseParenToken); + + private static StackFrameMethodDeclarationNode MethodDeclaration( + StackFrameMemberAccessExpressionNode memberAccessExpression, + StackFrameParameterList argumentList, + StackFrameTypeArgumentList? typeArgumnets = null) + { + return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArgumnets, argumentList); + } + + private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameExpressionNode expressionNode, StackFrameBaseIdentifierNode identifierNode) + => new(expressionNode, DotToken, identifierNode); + + private static StackFrameIdentifierNode Identifier(string identifierName, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + => new(CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia)); + + private static StackFrameArrayExpressionNode ArrayExpression(StackFrameExpressionNode identifier, params StackFrameToken[] arrayTokens) + => new(identifier, arrayTokens.ToImmutableArray()); + + private static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) + => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.TextToken, arity.ToString())); + + private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgument[] typeArguments) + { + using var _ = PooledObjects.ArrayBuilder.GetInstance(out var builder); + var openToken = useBrackets ? OpenBracketToken : LessThanToken; + var closeToken = useBrackets ? CloseBracketToken : GreaterThanToken; + + var isFirst = true; + foreach (var typeArgument in typeArguments) + { + if (isFirst) + { + isFirst = false; + } + else + { + builder.Add(CommaToken); + } + + builder.Add(typeArgument); + } + + return new(openToken, builder.ToImmutable(), closeToken); + } + + private static StackFrameTypeArgument TypeArgument(string identifier) + => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); + } +} diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 52210a0e21d8d..bb93fe106a8a9 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -2,13 +2,16 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Text; +using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame; using Roslyn.Test.Utilities; using Xunit; +using System; namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame { @@ -18,33 +21,32 @@ namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame public partial class StackFrameParserTests { - private static StackFrameToken CreateToken(StackFrameKind kind, string s, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) - => new( - kind, - leadingTrivia.IsDefaultOrEmpty ? ImmutableArray.Empty : leadingTrivia, - CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, s), - trailingTrivia.IsDefaultOrEmpty ? ImmutableArray.Empty : trailingTrivia, - ImmutableArray.Empty, - value: null!); - - private static StackFrameTrivia CreateTrivia(StackFrameKind kind, string s) - => new(kind, CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, s), ImmutableArray.Empty); - - private static ImmutableArray CreateTriviaArray(StackFrameTrivia trivia) - => ImmutableArray.Create(trivia); - - private static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); - private static readonly StackFrameToken CommaToken = CreateToken(StackFrameKind.CommaToken, ","); - private static readonly StackFrameToken OpenParenToken = CreateToken(StackFrameKind.OpenParenToken, "("); - private static readonly StackFrameToken CloseParenToken = CreateToken(StackFrameKind.CloseParenToken, ")"); - private static readonly StackFrameToken OpenBracketToken = CreateToken(StackFrameKind.OpenBracketToken, "["); - private static readonly StackFrameToken CloseBracketToken = CreateToken(StackFrameKind.CloseBracketToken, "]"); - private static readonly StackFrameToken LessThanToken = CreateToken(StackFrameKind.LessThanToken, "<"); - private static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); - private static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); - - private static readonly StackFrameTrivia SpaceTrivia = CreateTrivia(StackFrameKind.WhitespaceTrivia, " "); + private static void Verify(string input, StackFrameMethodDeclarationNode? methodDeclaration = null, bool expectSuccess = true) + { + var tree = StackFrameParser.TryParse(input); + if (!expectSuccess) + { + Assert.Null(tree); + return; + } + + AssertEx.NotNull(tree); + VerifyCharacterSpans(input, tree); + + if (methodDeclaration is null) + { + Assert.Null(tree.Root.MethodDeclaration); + } + else + { + AssertEqual(methodDeclaration, tree.Root.MethodDeclaration); + } + Assert.True(tree.Root.AtTrivia.HasValue); + Assert.False(tree.Root.InTrivia.HasValue); + Assert.Null(tree.Root.FileInformationExpression); + Assert.False(tree.Root.TrailingTrivia.HasValue); + } private static void AssertEqual(StackFrameNodeOrToken expected, StackFrameNodeOrToken actual) { AssertEqual(expected.Node, actual.Node); @@ -138,6 +140,117 @@ private static void AssertEqual(StackFrameToken expected, StackFrameToken actual Assert.Equal(expected.VirtualChars.CreateString(), actual.VirtualChars.CreateString()); } + private static void VerifyCharacterSpans(string originalText, StackFrameTree tree) + { + var textSeq = VirtualCharSequence.Create(0, originalText); + var index = 0; + List enumeratedParsedCharacters = new(); + + foreach (var charSeq in EnumerateTree(tree)) + { + foreach (var ch in charSeq) + { + enumeratedParsedCharacters.Add(ch); + + if (textSeq[index++] != ch) + { + Assert.True(false, PrintDifference()); + } + } + } + + // Make sure we enumerated the total input + Assert.Equal(index, textSeq.Length); + + string PrintDifference() + { + var sb = new StringBuilder(); + + var start = Math.Max(0, index - 10); + var end = Math.Min(index, originalText.Length - 1); + + sb.Append("Expected: \t"); + PrintString(originalText, start, end, sb); + sb.AppendLine(); + + sb.Append("Actual: \t"); + var enumeratedString = new string(enumeratedParsedCharacters.Select(ch => (char)ch.Value).ToArray()); + PrintString(enumeratedString, start, end, sb); + sb.AppendLine(); + + return sb.ToString(); + + static void PrintString(string s, int start, int end, StringBuilder sb) + { + if (start > 0) + { + sb.Append("..."); + } + sb.Append(s[start..end]); + if (end < s.Length - 1) + { + sb.Append("..."); + } + } + } + } + + private static IEnumerable EnumerateTree(StackFrameTree tree) + { + var root = tree.Root; + + if (root.AtTrivia.HasValue) + { + yield return root.AtTrivia.Value.VirtualChars; + } + + var methodDeclaration = root.MethodDeclaration; + foreach (var seq in Enumerate(methodDeclaration)) + { + yield return seq; + } + + if (root.TrailingTrivia.HasValue) + { + yield return root.TrailingTrivia.Value.VirtualChars; + } + } + + private static IEnumerable Enumerate(StackFrameNode node) + { + foreach (var nodeOrToken in node) + { + if (nodeOrToken.IsNode) + { + foreach (var charSequence in Enumerate(nodeOrToken.Node).ToArray()) + { + yield return charSequence; + } + } + else if (!nodeOrToken.Token.VirtualChars.IsDefaultOrEmpty) + { + var token = nodeOrToken.Token; + if (!token.LeadingTrivia.IsDefault) + { + foreach (var trivia in token.LeadingTrivia) + { + yield return trivia.VirtualChars; + } + } + + yield return nodeOrToken.Token.VirtualChars; + + if (!token.TrailingTrivia.IsDefault) + { + foreach (var trivia in token.TrailingTrivia) + { + yield return trivia.VirtualChars; + } + } + } + } + } + private static void AssertEqual(ImmutableArray expected, ImmutableArray actual) { var diffMessage = PrintDiff(); @@ -200,55 +313,5 @@ private static void AssertEqual(StackFrameTrivia expected, StackFrameTrivia actu Assert.Equal(expected.Kind, actual.Kind); Assert.Equal(expected.VirtualChars.CreateString(), actual.VirtualChars.CreateString()); } - - private static StackFrameParameterList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) - => new(OpenParenToken, nodesOrTokens.ToImmutableArray(), CloseParenToken); - - private static StackFrameMethodDeclarationNode MethodDeclaration( - StackFrameMemberAccessExpressionNode memberAccessExpression, - StackFrameParameterList argumentList, - StackFrameTypeArgumentList? typeArgumnets = null) - { - return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArgumnets, argumentList); - } - - private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameExpressionNode expressionNode, StackFrameBaseIdentifierNode identifierNode) - => new(expressionNode, DotToken, identifierNode); - - private static StackFrameIdentifierNode Identifier(string identifierName, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) - => new(CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia)); - - private static StackFrameArrayExpressionNode ArrayExpression(StackFrameExpressionNode identifier, params StackFrameToken[] arrayTokens) - => new(identifier, arrayTokens.ToImmutableArray()); - - private static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) - => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.TextToken, arity.ToString())); - - private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgument[] typeArguments) - { - using var _ = PooledObjects.ArrayBuilder.GetInstance(out var builder); - var openToken = useBrackets ? OpenBracketToken : LessThanToken; - var closeToken = useBrackets ? CloseBracketToken : GreaterThanToken; - - var isFirst = true; - foreach (var typeArgument in typeArguments) - { - if (isFirst) - { - isFirst = false; - } - else - { - builder.Add(CommaToken); - } - - builder.Add(typeArgument); - } - - return new(openToken, builder.ToImmutable(), closeToken); - } - - private static StackFrameTypeArgument TypeArgument(string identifier) - => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); } } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 3ddc0d0a8b286..02fce0ec977af 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -12,150 +12,90 @@ public partial class StackFrameParserTests { [Fact] public void TestMethodOneParam() - { - var input = @"at ConsoleApp4.MyClass.M(string s)"; - var tree = StackFrameParser.TryParse(input); - AssertEx.NotNull(tree); - - Assert.True(tree.Root.AtTrivia.HasValue); - Assert.False(tree.Root.InTrivia.HasValue); - Assert.Null(tree.Root.FileInformationExpression); - Assert.False(tree.Root.TrailingTrivia.HasValue); - - var methodDeclaration = tree.Root.MethodDeclaration; - - var expectedMethodDeclaration = MethodDeclaration( - MemberAccessExpression( + => Verify( + @"at ConsoleApp4.MyClass.M(string s)", + methodDeclaration: MethodDeclaration( MemberAccessExpression( - Identifier("ConsoleApp4"), - Identifier("MyClass")), - Identifier("M")), - - ArgumentList( - Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), - Identifier("s")) + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + + ArgumentList( + Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("s")) + ) ); - AssertEqual(expectedMethodDeclaration, methodDeclaration); - } - [Fact] public void TestMethodTwoParam() - { - var input = @"at ConsoleApp4.MyClass.M(string s, string t)"; - var tree = StackFrameParser.TryParse(input); - AssertEx.NotNull(tree); - - Assert.True(tree.Root.AtTrivia.HasValue); - Assert.False(tree.Root.InTrivia.HasValue); - Assert.Null(tree.Root.FileInformationExpression); - Assert.False(tree.Root.TrailingTrivia.HasValue); - - var methodDeclaration = tree.Root.MethodDeclaration; - - var expectedMethodDeclaration = MethodDeclaration( - MemberAccessExpression( + => Verify( + @"at ConsoleApp4.MyClass.M(string s, string t)", + methodDeclaration: MethodDeclaration( MemberAccessExpression( - Identifier("ConsoleApp4"), - Identifier("MyClass")), - Identifier("M")), - - ArgumentList( - Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), - Identifier("s"), - CommaToken, - Identifier("string", leadingTrivia: CreateTriviaArray(SpaceTrivia), trailingTrivia: CreateTriviaArray(SpaceTrivia)), - Identifier("t")) + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + + ArgumentList( + Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("s"), + CommaToken, + Identifier("string", leadingTrivia: CreateTriviaArray(SpaceTrivia), trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("t")) + ) ); - AssertEqual(expectedMethodDeclaration, methodDeclaration); - } - [Fact] public void TestMethodArrayParam() - { - var input = @"at ConsoleApp4.MyClass.M(string[] s)"; - var tree = StackFrameParser.TryParse(input); - AssertEx.NotNull(tree); - - Assert.True(tree.Root.AtTrivia.HasValue); - Assert.False(tree.Root.InTrivia.HasValue); - Assert.Null(tree.Root.FileInformationExpression); - Assert.False(tree.Root.TrailingTrivia.HasValue); - - var methodDeclaration = tree.Root.MethodDeclaration; - - var expectedMethodDeclaration = MethodDeclaration( - MemberAccessExpression( + => Verify( + @"at ConsoleApp4.MyClass.M(string[] s)", + methodDeclaration: MethodDeclaration( MemberAccessExpression( - Identifier("ConsoleApp4"), - Identifier("MyClass")), - Identifier("M")), + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), - ArgumentList( - ArrayExpression(Identifier("string"), OpenBracketToken, CloseBracketToken), - Identifier("s", leadingTrivia: CreateTriviaArray(SpaceTrivia))) - ); - - AssertEqual(expectedMethodDeclaration, methodDeclaration); - } + ArgumentList( + ArrayExpression(Identifier("string"), OpenBracketToken, CloseBracketToken), + Identifier("s", leadingTrivia: CreateTriviaArray(SpaceTrivia))) + ) + ); [Fact] public void TestGenericMethod_Brackets() - { - var input = @"at ConsoleApp4.MyClass.M[T](T t)"; - var tree = StackFrameParser.TryParse(input); - AssertEx.NotNull(tree); - - Assert.True(tree.Root.AtTrivia.HasValue); - Assert.False(tree.Root.InTrivia.HasValue); - Assert.Null(tree.Root.FileInformationExpression); - Assert.False(tree.Root.TrailingTrivia.HasValue); - - var methodDeclaration = tree.Root.MethodDeclaration; - - var expectedMethodDeclaration = MethodDeclaration( - MemberAccessExpression( + => Verify( + @"at ConsoleApp4.MyClass.M[T](T t)", + methodDeclaration: MethodDeclaration( MemberAccessExpression( - Identifier("ConsoleApp4"), - Identifier("MyClass")), - Identifier("M")), - typeArgumnets: TypeArgumentList(useBrackets: true, TypeArgument("T")), - argumentList: ArgumentList( - Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), - Identifier("t")) - ); - - AssertEqual(expectedMethodDeclaration, methodDeclaration); - } + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + typeArgumnets: TypeArgumentList(useBrackets: true, TypeArgument("T")), + argumentList: ArgumentList( + Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("t")) + ) + ); [Fact] public void TestGenericMethod() - { - var input = @"at ConsoleApp4.MyClass.M(T t)"; - var tree = StackFrameParser.TryParse(input); - AssertEx.NotNull(tree); - - Assert.True(tree.Root.AtTrivia.HasValue); - Assert.False(tree.Root.InTrivia.HasValue); - Assert.Null(tree.Root.FileInformationExpression); - Assert.False(tree.Root.TrailingTrivia.HasValue); - - var methodDeclaration = tree.Root.MethodDeclaration; - - var expectedMethodDeclaration = MethodDeclaration( - MemberAccessExpression( + => Verify( + @"at ConsoleApp4.MyClass.M(T t)", + methodDeclaration: MethodDeclaration( MemberAccessExpression( - Identifier("ConsoleApp4"), - Identifier("MyClass")), - Identifier("M")), - typeArgumnets: TypeArgumentList(useBrackets: false, TypeArgument("T")), - argumentList: ArgumentList( - Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), - Identifier("t")) - ); - - AssertEqual(expectedMethodDeclaration, methodDeclaration); - } + MemberAccessExpression( + Identifier("ConsoleApp4"), + Identifier("MyClass")), + Identifier("M")), + typeArgumnets: TypeArgumentList(useBrackets: false, TypeArgument("T")), + argumentList: ArgumentList( + Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("t")) + ) + ); } } From c8b0bd3d026907f5d2cbd8b055beda476dff09c0 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 5 Oct 2021 23:38:22 -0700 Subject: [PATCH 05/59] Add more tests and clean up test framework making it easier to write more tests and validate things thorougly --- .../StackFrameParserTests.Generators.cs | 9 +++-- .../StackFrameParserTests.Utilities.cs | 4 +-- .../StackFrame/StackFrameParserTests.cs | 35 +++++++++++++++---- .../StackFrame/StackFrameNodeDefinitions.cs | 4 +-- 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index 9e33df93dfe1c..974cc021514c6 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -53,10 +53,10 @@ private static StackFrameParameterList ArgumentList(params StackFrameNodeOrToken private static StackFrameMethodDeclarationNode MethodDeclaration( StackFrameMemberAccessExpressionNode memberAccessExpression, - StackFrameParameterList argumentList, - StackFrameTypeArgumentList? typeArgumnets = null) + StackFrameTypeArgumentList? typeArguments = null, + StackFrameParameterList? argumentList = null) { - return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArgumnets, argumentList); + return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, argumentList ?? ArgumentList(OpenParenToken, CloseParenToken)); } private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameExpressionNode expressionNode, StackFrameBaseIdentifierNode identifierNode) @@ -71,6 +71,9 @@ private static StackFrameArrayExpressionNode ArrayExpression(StackFrameExpressio private static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.TextToken, arity.ToString())); + private static StackFrameTypeArgumentList TypeArgumentList(params StackFrameTypeArgument[] typeArguments) + => TypeArgumentList(useBrackets: true, typeArguments); + private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgument[] typeArguments) { using var _ = PooledObjects.ArrayBuilder.GetInstance(out var builder); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index bb93fe106a8a9..4bea0ae72eae0 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -21,10 +21,10 @@ namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame public partial class StackFrameParserTests { - private static void Verify(string input, StackFrameMethodDeclarationNode? methodDeclaration = null, bool expectSuccess = true) + private static void Verify(string input, StackFrameMethodDeclarationNode? methodDeclaration = null, bool expectFailure = false) { var tree = StackFrameParser.TryParse(input); - if (!expectSuccess) + if (expectFailure) { Assert.Null(tree); return; diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 02fce0ec977af..d10adff61e077 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -2,8 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame; -using Roslyn.Test.Utilities; using Xunit; namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame @@ -21,7 +19,7 @@ public void TestMethodOneParam() Identifier("MyClass")), Identifier("M")), - ArgumentList( + argumentList: ArgumentList( Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), Identifier("s")) ) @@ -38,7 +36,7 @@ public void TestMethodTwoParam() Identifier("MyClass")), Identifier("M")), - ArgumentList( + argumentList: ArgumentList( Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), Identifier("s"), CommaToken, @@ -58,7 +56,7 @@ public void TestMethodArrayParam() Identifier("MyClass")), Identifier("M")), - ArgumentList( + argumentList: ArgumentList( ArrayExpression(Identifier("string"), OpenBracketToken, CloseBracketToken), Identifier("s", leadingTrivia: CreateTriviaArray(SpaceTrivia))) ) @@ -74,7 +72,7 @@ public void TestGenericMethod_Brackets() Identifier("ConsoleApp4"), Identifier("MyClass")), Identifier("M")), - typeArgumnets: TypeArgumentList(useBrackets: true, TypeArgument("T")), + typeArguments: TypeArgumentList(TypeArgument("T")), argumentList: ArgumentList( Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), Identifier("t")) @@ -91,11 +89,34 @@ public void TestGenericMethod() Identifier("ConsoleApp4"), Identifier("MyClass")), Identifier("M")), - typeArgumnets: TypeArgumentList(useBrackets: false, TypeArgument("T")), + typeArguments: TypeArgumentList(useBrackets: false, TypeArgument("T")), argumentList: ArgumentList( Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), Identifier("t")) ) ); + + [Fact] + public void TestUnderscoreNames() + => Verify( + @"at _._[_](_ _)", + methodDeclaration: MethodDeclaration( + MemberAccessExpression( + Identifier("_"), + Identifier("_")), + typeArguments: TypeArgumentList(TypeArgument("_")), + argumentList: ArgumentList( + Identifier("_", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("_")) + ) + ); + + [Theory] + [InlineData(@"at M()")] // Method with no class is invalid + [InlineData(@"at M.1c()")] // Invalid start character for identifier + [InlineData(@"at 1M.C()")] + [InlineData(@"at M.C(string& s)")] // "string&" represents a reference (ref, out) and is not supported yet + public void TestInvalidInputs(string input) + => Verify(input, expectFailure: true); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index 6b3455ec52f22..b2509b6e308e8 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -282,14 +282,14 @@ public StackFrameParameterList(StackFrameToken openParen, ImmutableArray nodeOrToken.IsNode + Debug.Assert(childNodesOrTokens.IsDefaultOrEmpty || childNodesOrTokens.All(nodeOrToken => nodeOrToken.IsNode ? nodeOrToken.Node is StackFrameIdentifierNode or StackFrameMemberAccessExpressionNode or StackFrameArrayExpressionNode : nodeOrToken.Token.Kind is StackFrameKind.CommaToken)); OpenParen = openParen; CloseParen = closeParen; _childNodesOrTokens = childNodesOrTokens; - ChildCount = _childNodesOrTokens.Length + 2; + ChildCount = _childNodesOrTokens.IsDefault ? 2 : _childNodesOrTokens.Length + 2; } internal override int ChildCount { get; } From 6f8fcd4f7e91d3643f4e03ca4ba2b5b14acf525d Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 6 Oct 2021 01:09:20 -0700 Subject: [PATCH 06/59] Fix the parsing so there is no look behind. Make the "at " trivia be added to the method declaration and no longer be held onto at the root. --- .../StackFrameParserTests.Generators.cs | 1 + .../StackFrameParserTests.Utilities.cs | 8 +- .../StackFrame/StackFrameParserTests.cs | 12 +- .../StackFrame/IStackFrameNodeVisitor.cs | 1 - .../StackFrame/StackFrameCompilationUnit.cs | 4 +- .../StackFrame/StackFrameLexer.cs | 167 +++++++----------- .../StackFrame/StackFrameNodeDefinitions.cs | 87 +++++---- .../StackFrame/StackFrameParser.cs | 110 +++++------- 8 files changed, 170 insertions(+), 220 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index 974cc021514c6..eac3be336aa4f 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -47,6 +47,7 @@ private static ImmutableArray CreateTriviaArray(StackFrameTriv private static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); private static readonly StackFrameTrivia SpaceTrivia = CreateTrivia(StackFrameKind.WhitespaceTrivia, " "); + private static readonly StackFrameTrivia AtTrivia = CreateTrivia(StackFrameKind.AtTrivia, "at "); private static StackFrameParameterList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) => new(OpenParenToken, nodesOrTokens.ToImmutableArray(), CloseParenToken); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 4bea0ae72eae0..ec5181b8139dd 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -42,7 +42,6 @@ private static void Verify(string input, StackFrameMethodDeclarationNode? method AssertEqual(methodDeclaration, tree.Root.MethodDeclaration); } - Assert.True(tree.Root.AtTrivia.HasValue); Assert.False(tree.Root.InTrivia.HasValue); Assert.Null(tree.Root.FileInformationExpression); Assert.False(tree.Root.TrailingTrivia.HasValue); @@ -186,7 +185,9 @@ static void PrintString(string s, int start, int end, StringBuilder sb) { sb.Append("..."); } + sb.Append(s[start..end]); + if (end < s.Length - 1) { sb.Append("..."); @@ -199,11 +200,6 @@ private static IEnumerable EnumerateTree(StackFrameTree tre { var root = tree.Root; - if (root.AtTrivia.HasValue) - { - yield return root.AtTrivia.Value.VirtualChars; - } - var methodDeclaration = root.MethodDeclaration; foreach (var seq in Enumerate(methodDeclaration)) { diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index d10adff61e077..7aa6bb07340b2 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -15,7 +15,7 @@ public void TestMethodOneParam() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4"), + Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), Identifier("MyClass")), Identifier("M")), @@ -32,7 +32,7 @@ public void TestMethodTwoParam() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4"), + Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), Identifier("MyClass")), Identifier("M")), @@ -52,7 +52,7 @@ public void TestMethodArrayParam() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4"), + Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), Identifier("MyClass")), Identifier("M")), @@ -69,7 +69,7 @@ public void TestGenericMethod_Brackets() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4"), + Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), Identifier("MyClass")), Identifier("M")), typeArguments: TypeArgumentList(TypeArgument("T")), @@ -86,7 +86,7 @@ public void TestGenericMethod() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4"), + Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), Identifier("MyClass")), Identifier("M")), typeArguments: TypeArgumentList(useBrackets: false, TypeArgument("T")), @@ -102,7 +102,7 @@ public void TestUnderscoreNames() @"at _._[_](_ _)", methodDeclaration: MethodDeclaration( MemberAccessExpression( - Identifier("_"), + Identifier("_", leadingTrivia: CreateTriviaArray(AtTrivia)), Identifier("_")), typeArguments: TypeArgumentList(TypeArgument("_")), argumentList: ArgumentList( diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index 31eaef511dc75..4a1178fd27e5a 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -10,7 +10,6 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { internal interface IStackFrameNodeVisitor { - void Visit(StackFrameTextNode stackFrameTextNode); void Visit(StackFrameMethodDeclarationNode stackFrameMethodDeclarationNode); void Visit(StackFrameMemberAccessExpressionNode stackFrameMemberAccessExpressionNode); void Visit(StackFrameTypeArgumentList stackFrameTypeArguments); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs index a389386d0996f..bdfcba137b5c7 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs @@ -8,15 +8,13 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { internal class StackFrameCompilationUnit { - public readonly EmbeddedSyntaxTrivia? AtTrivia; public readonly StackFrameMethodDeclarationNode MethodDeclaration; public readonly EmbeddedSyntaxTrivia? InTrivia; public readonly StackFrameFileInformationNode? FileInformationExpression; public readonly EmbeddedSyntaxTrivia? TrailingTrivia; - public StackFrameCompilationUnit(EmbeddedSyntaxTrivia? atTrivia, StackFrameMethodDeclarationNode methodDeclaration, EmbeddedSyntaxTrivia? inTrivia, StackFrameFileInformationNode? fileInformationExpression, EmbeddedSyntaxTrivia? trailingTrivia) + public StackFrameCompilationUnit(StackFrameMethodDeclarationNode methodDeclaration, EmbeddedSyntaxTrivia? inTrivia, StackFrameFileInformationNode? fileInformationExpression, EmbeddedSyntaxTrivia? trailingTrivia) { - AtTrivia = atTrivia; MethodDeclaration = methodDeclaration; InTrivia = inTrivia; FileInformationExpression = fileInformationExpression; diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index a4d210874add8..0d89f8766a999 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -21,7 +21,7 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame internal struct StackFrameLexer { public readonly VirtualCharSequence Text; - public int Position; + public int Position { get; private set; } public StackFrameLexer(string text) : this(VirtualCharSequence.Create(0, text)) @@ -39,20 +39,6 @@ public VirtualCharSequence GetSubPatternToCurrentPos(int start) public VirtualCharSequence GetSubPattern(int start, int end) => Text.GetSubSequence(TextSpan.FromBounds(start, end)); - public StackFrameToken ScanNextToken(bool allowTrivia) - { - var trivia = ScanLeadingTrivia(allowTrivia); - if (Position == Text.Length) - { - return CreateToken(StackFrameKind.EndOfLine, trivia, VirtualCharSequence.Empty); - } - - var ch = CurrentChar; - Position++; - - return CreateToken(GetKind(ch), trivia, Text.GetSubSequence(new TextSpan(Position - 1, 1))); - } - /// /// Scans until EndOfLine is found, treating all text as trivia /// @@ -77,8 +63,6 @@ public StackFrameToken ScanNextToken(bool allowTrivia) var startPosition = Position; var ch = CurrentChar; - Position++; - if (!UnicodeCharacterUtilities.IsIdentifierStartCharacter((char)ch.Value)) { return null; @@ -86,12 +70,11 @@ public StackFrameToken ScanNextToken(bool allowTrivia) while (UnicodeCharacterUtilities.IsIdentifierPartCharacter((char)ch.Value)) { - ch = CurrentChar; Position++; + ch = CurrentChar; } - var identifierEnd = Position - 1; - var identifierSpan = new TextSpan(startPosition, identifierEnd - startPosition); + var identifierSpan = new TextSpan(startPosition, Position - startPosition); var identifier = CreateToken(StackFrameKind.IdentifierToken, Text.GetSubSequence(identifierSpan)); return identifier; } @@ -129,41 +112,32 @@ internal ImmutableArray ScanArrayBrackets() while (Position < Text.Length) { - // Use PreviousCharAsToken here because we expect the - // start of the brackets to be the previous token that started - // the need to parse for an array - var token = PreviousCharAsToken(); - if (!IsArrayBracket(token)) + var kind = GetKind(CurrentChar); + if (!IsArrayBracket(kind)) { break; } + builder.Add(CurrentCharAsToken()); Position++; - builder.Add(token); } Debug.Assert(builder.Count >= 2); return builder.ToImmutable(); - static bool IsArrayBracket(StackFrameToken token) - => token.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.CloseBracketToken; - } - - public StackFrameToken PreviousCharAsToken() - { - var previousChar = Text[Position - 1]; - return CreateToken(GetKind(previousChar), Text.GetSubSequence(new TextSpan(Position - 1, 1))); + static bool IsArrayBracket(StackFrameKind kind) + => kind is StackFrameKind.OpenBracketToken or StackFrameKind.CloseBracketToken; } - internal StackFrameTrivia? ScanWhiteSpace(bool includePrevious) + internal StackFrameTrivia? ScanWhiteSpace() { if (Position == Text.Length) { return null; } - var startPosition = includePrevious ? Position - 1 : Position; + var startPosition = Position; while (IsBlank(CurrentChar)) { @@ -190,7 +164,7 @@ public StackFrameToken CurrentCharAsToken() return CreateToken(GetKind(previousChar), Text.GetSubSequence(new TextSpan(Position, 1))); } - public bool IsAt(string val) + public bool IsStringAtPosition(string val) => TextAt(Position, val); private bool TextAt(int position, string val) @@ -207,19 +181,45 @@ private bool TextAt(int position, string val) return true; } -#nullable disable - public static StackFrameToken CreateToken(StackFrameKind kind, VirtualCharSequence virtualChars) - => new(kind, ImmutableArray.Empty, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null); + /// + /// Progress the position by one if the current character + /// matches the kind. + /// + /// + /// if the position was incremented + /// + internal bool ScanIfMatch(StackFrameKind kind, out StackFrameToken token) + { + if (GetKind(CurrentChar) == kind) + { + token = CurrentCharAsToken(); + Position++; + return true; + } - public static StackFrameToken CreateToken(StackFrameKind kind, ImmutableArray leadingTrivia, VirtualCharSequence virtualChars) - => new(kind, leadingTrivia, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null); + token = default; + return false; + } - public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars) - => CreateTrivia(kind, virtualChars, ImmutableArray.Empty); + /// + /// Progress the position by one if the current character + /// matches the kind. + /// + /// + /// if the position was incremented + /// + internal bool ScanIfMatch(Func matchFn, out StackFrameToken token) + { + if (matchFn(GetKind(CurrentChar))) + { + token = CurrentCharAsToken(); + Position++; + return true; + } - public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars, ImmutableArray diagnostics) - => new(kind, virtualChars, diagnostics); -#nullable enable + token = default; + return false; + } private static StackFrameKind GetKind(VirtualChar ch) => ch.Value switch @@ -248,49 +248,10 @@ private static StackFrameKind GetKind(VirtualChar ch) private static bool IsNumber(VirtualChar ch) => ch.Value switch { - '0' or '1' or '2' or '3' or '4' or '5' or '6' or '7' or '8' or '9' - => true, + >= '0' and <= '9' => true, _ => false }; - private ImmutableArray ScanLeadingTrivia(bool allowTrivia) - { - if (!allowTrivia) - { - return ImmutableArray.Empty; - } - - using var _ = ArrayBuilder.GetInstance(out var result); - - while (Position < Text.Length) - { - var atTrivia = ScanAtTrivia(); - if (atTrivia != null) - { - result.Add(atTrivia.Value); - continue; - } - - var whitespace = ScanWhitespace(); - if (whitespace != null) - { - result.Add(whitespace.Value); - continue; - } - - var inTrivia = ScanInTrivia(); - if (inTrivia != null) - { - result.Add(inTrivia.Value); - continue; - } - - break; - } - - return result.ToImmutable(); - } - public StackFrameTrivia? ScanAtTrivia() { if (Position >= Text.Length) @@ -301,7 +262,7 @@ private ImmutableArray ScanLeadingTrivia(bool allowTrivia) // TODO: Handle multiple languages? Right now we're going to only parse english const string AtString = "at "; - if (IsAt(AtString)) + if (IsStringAtPosition(AtString)) { var start = Position; Position += AtString.Length; @@ -312,22 +273,6 @@ private ImmutableArray ScanLeadingTrivia(bool allowTrivia) return null; } - private StackFrameTrivia? ScanWhitespace() - { - var start = Position; - while (Position < Text.Length && IsBlank(Text[Position])) - { - Position++; - } - - if (Position > start) - { - return CreateTrivia(StackFrameKind.WhitespaceTrivia, GetSubPatternToCurrentPos(start)); - } - - return null; - } - public StackFrameTrivia? ScanInTrivia() { if (Position >= Text.Length) @@ -338,7 +283,7 @@ private ImmutableArray ScanLeadingTrivia(bool allowTrivia) // TODO: Handle multiple languages? Right now we're going to only parse english const string InString = " in "; - if (IsAt(InString)) + if (IsStringAtPosition(InString)) { var start = Position; Position += InString.Length; @@ -364,5 +309,17 @@ public static bool IsBlank(VirtualChar ch) return false; } } + + public static StackFrameToken CreateToken(StackFrameKind kind, VirtualCharSequence virtualChars) + => new(kind, ImmutableArray.Empty, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null!); + + public static StackFrameToken CreateToken(StackFrameKind kind, ImmutableArray leadingTrivia, VirtualCharSequence virtualChars) + => new(kind, leadingTrivia, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null!); + + public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars) + => CreateTrivia(kind, virtualChars, ImmutableArray.Empty); + + public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars, ImmutableArray diagnostics) + => new(kind, virtualChars, diagnostics); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index b2509b6e308e8..266e8a6d3b74d 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -34,6 +34,9 @@ internal abstract class StackFrameExpressionNode : StackFrameNode protected StackFrameExpressionNode(StackFrameKind kind) : base(kind) { } + + internal abstract StackFrameExpressionNode WithLeadingTrivia(ImmutableArray trivia); + internal abstract StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trivia); } internal sealed class StackFrameMethodDeclarationNode : StackFrameNode @@ -66,6 +69,9 @@ internal override StackFrameNodeOrToken ChildAt(int index) 2 => ArgumentList, _ => throw new InvalidOperationException(), }; + + internal StackFrameMethodDeclarationNode WithLeadingTrivia(ImmutableArray trivia) + => new((StackFrameMemberAccessExpressionNode)MemberAccessExpression.WithLeadingTrivia(trivia), TypeArguments, ArgumentList); } internal sealed class StackFrameMemberAccessExpressionNode : StackFrameExpressionNode @@ -95,8 +101,17 @@ internal override StackFrameNodeOrToken ChildAt(int index) _ => throw new InvalidOperationException() }; - internal StackFrameMemberAccessExpressionNode WithTrailingTrivia(StackFrameTrivia trivia) - => new(Expression, Operator, Identifier.WithTrailingTrivia(trivia)); + internal StackFrameExpressionNode WithLeadingTrivia(StackFrameTrivia trivia) + => WithLeadingTrivia(ImmutableArray.Create(trivia)); + + internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray trivia) + => new StackFrameMemberAccessExpressionNode(Expression.WithLeadingTrivia(trivia), Operator, Identifier); + + internal StackFrameExpressionNode WithTrailingTrivia(StackFrameTrivia trivia) + => WithTrailingTrivia(ImmutableArray.Create(trivia)); + + internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trivia) + => new StackFrameMemberAccessExpressionNode(Expression, Operator, (StackFrameBaseIdentifierNode)Identifier.WithTrailingTrivia(trivia)); } internal abstract class StackFrameBaseIdentifierNode : StackFrameExpressionNode @@ -105,7 +120,11 @@ protected StackFrameBaseIdentifierNode(StackFrameKind kind) : base(kind) { } - internal abstract StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTrivia trailingTrivia); + internal StackFrameExpressionNode WithLeadingTrivia(StackFrameTrivia leadingTrivia) + => WithLeadingTrivia(ImmutableArray.Create(leadingTrivia)); + + internal StackFrameExpressionNode WithTrailingTrivia(StackFrameTrivia trailingTrivia) + => WithTrailingTrivia(ImmutableArray.Create(trailingTrivia)); } internal sealed class StackFrameIdentifierNode : StackFrameBaseIdentifierNode @@ -133,8 +152,11 @@ internal override StackFrameNodeOrToken ChildAt(int index) public override string ToString() => Identifier.VirtualChars.CreateString(); - internal override StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTrivia trailingTrivia) - => new StackFrameIdentifierNode(Identifier.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))); + internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trailingTrivia) + => new StackFrameIdentifierNode(Identifier.With(trailingTrivia: trailingTrivia)); + + internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray leadingTrivia) + => new StackFrameIdentifierNode(Identifier.With(leadingTrivia: leadingTrivia)); } internal sealed class StackFrameGenericTypeIdentifier : StackFrameBaseIdentifierNode @@ -165,11 +187,17 @@ internal override StackFrameNodeOrToken ChildAt(int index) _ => throw new InvalidOperationException() }; - internal override StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTrivia trailingTrivia) + internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray leadingTrivia) + => new StackFrameGenericTypeIdentifier( + Identifier.With(leadingTrivia: leadingTrivia), + ArityToken, + ArityNumericToken); + + internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trailingTrivia) => new StackFrameGenericTypeIdentifier( Identifier, ArityToken, - ArityNumericToken.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))); + ArityNumericToken.With(trailingTrivia: trailingTrivia)); } internal sealed class StackFrameArrayExpressionNode : StackFrameExpressionNode @@ -197,6 +225,17 @@ internal override StackFrameNodeOrToken ChildAt(int index) 0 => _identifier, _ => _arrayBrackets[index - 1] }; + + internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray trivia) + => new StackFrameArrayExpressionNode(_identifier.WithLeadingTrivia(trivia), _arrayBrackets); + + internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trivia) + { + var lastBracket = _arrayBrackets.Last().With(trailingTrivia: trivia); + var newBrackets = _arrayBrackets.RemoveAt(_arrayBrackets.Length - 1).Add(lastBracket); + + return new StackFrameArrayExpressionNode(_identifier, newBrackets); + } } internal sealed class StackFrameTypeArgumentList : StackFrameNode @@ -268,8 +307,11 @@ internal override StackFrameNodeOrToken ChildAt(int index) _ => throw new InvalidOperationException() }; - internal override StackFrameBaseIdentifierNode WithTrailingTrivia(StackFrameTrivia trailingTrivia) - => new StackFrameTypeArgument(Identifier.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))); + internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trailingTrivia) + => new StackFrameTypeArgument(Identifier.With(trailingTrivia: trailingTrivia)); + + internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray leadingTrivia) + => new StackFrameTypeArgument(Identifier.With(leadingTrivia: leadingTrivia)); } internal sealed class StackFrameParameterList : StackFrameNode @@ -336,31 +378,4 @@ internal override StackFrameNodeOrToken ChildAt(int index) throw new NotImplementedException(); } } - - /// - /// Represents a chunk of text (can be multiple characters) - /// - internal sealed class StackFrameTextNode : StackFrameExpressionNode - { - public StackFrameTextNode(StackFrameToken textToken) - : base(StackFrameKind.Text) - { - Debug.Assert(textToken.Kind == StackFrameKind.TextToken); - TextToken = textToken; - } - - public StackFrameToken TextToken { get; } - - internal override int ChildCount => 1; - - internal override StackFrameNodeOrToken ChildAt(int index) - => index switch - { - 0 => TextToken, - _ => throw new InvalidOperationException(), - }; - - public override void Accept(IStackFrameNodeVisitor visitor) - => visitor.Visit(this); - } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 69fe71d84e96a..71e3a3bd3bdf3 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Linq; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.PooledObjects; @@ -20,12 +21,7 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame internal struct StackFrameParser { private StackFrameLexer _lexer; - - /// - /// The current token that has been consumed by the parse. Note that the lexer position - /// will be one character ahead to represent the next token to be consumed. - /// - private StackFrameToken CurrentToken => _lexer.PreviousCharAsToken(); + private StackFrameToken CurrentToken => _lexer.CurrentCharAsToken(); private StackFrameParser(VirtualCharSequence text) { @@ -67,12 +63,22 @@ private StackFrameParser(VirtualCharSequence text) { var atTrivia = _lexer.ScanAtTrivia(); - var methodDeclaration = ParseMethodDeclaration(addLeadingWhitespaceAsTrivia: !atTrivia.HasValue); + var methodDeclaration = ParseMethodDeclaration(); if (methodDeclaration is null) { return null; } + if (atTrivia.HasValue) + { + var currentTrivia = methodDeclaration.ChildAt(0).Token.LeadingTrivia; + var newList = currentTrivia.IsDefaultOrEmpty + ? ImmutableArray.Create(atTrivia.Value) + : currentTrivia.Prepend(atTrivia.Value).ToImmutableArray(); + + methodDeclaration = methodDeclaration.WithLeadingTrivia(newList); + } + var inTrivia = _lexer.ScanInTrivia(); var fileInformationExpression = inTrivia.HasValue ? ParseFileInformation() @@ -82,7 +88,7 @@ private StackFrameParser(VirtualCharSequence text) Debug.Assert(_lexer.Position == _lexer.Text.Length); Debug.Assert(_lexer.CurrentCharAsToken().Kind == StackFrameKind.EndOfLine); - var root = new StackFrameCompilationUnit(atTrivia, methodDeclaration, inTrivia, fileInformationExpression, trailingTrivia); + var root = new StackFrameCompilationUnit(methodDeclaration, inTrivia, fileInformationExpression, trailingTrivia); return new StackFrameTree( _lexer.Text, root, ImmutableArray.Empty); @@ -94,9 +100,9 @@ private StackFrameParser(VirtualCharSequence text) /// /// Ex: [|MyClass.MyMethod(string s)|] /// - private StackFrameMethodDeclarationNode? ParseMethodDeclaration(bool addLeadingWhitespaceAsTrivia) + private StackFrameMethodDeclarationNode? ParseMethodDeclaration() { - var identifierExpression = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: addLeadingWhitespaceAsTrivia); + var identifierExpression = ParseIdentifierExpression(); if (identifierExpression is not StackFrameMemberAccessExpressionNode memberAccessExpression) { return null; @@ -122,25 +128,17 @@ private StackFrameParser(VirtualCharSequence text) /// All of the following are valid identifiers, where "$$" marks the parsing starting point, and "[|" + "|]" mark the endpoints of the parsed identifier including trivia /// * [|$$MyNamespace.MyClass.MyMethod|](string s) /// * MyClass.MyMethod([|$$string |]s) - /// * MyClass.MyMethod(string [|$$s|]) // = false - /// * MyClass.MyMethod(string[| $$s|]) // = true + /// * MyClass.MyMethod(string[| $$s|]) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) /// - /// - /// Set to true if leading whitespaces before the identifier should be considered leading trivia. - /// - private StackFrameExpressionNode? ParseIdentifierExpression(bool addLeadingWhitespaceAsTrivia) + private StackFrameExpressionNode? ParseIdentifierExpression() { Queue<(StackFrameBaseIdentifierNode identifier, StackFrameToken separator)> typeIdentifierNodes = new(); - // If allowed, add the leading whitespace as trivia to the first - // identifier token in the expression - var leadingTrivia = AllowWhitespace(CurrentToken) - ? _lexer.ScanWhiteSpace(includePrevious: addLeadingWhitespaceAsTrivia && CurrentToken.Kind == StackFrameKind.WhitespaceToken) - : null; - + var leadingTrivia = _lexer.ScanWhiteSpace(); var currentIdentifier = _lexer.ScanIdentifier(); + while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { StackFrameToken? arity = null; @@ -161,7 +159,9 @@ private StackFrameParser(VirtualCharSequence text) typeIdentifierNodes.Enqueue((identifierNode, CurrentToken)); - if (CurrentToken.Kind != StackFrameKind.DotToken) + // Progress the lexer if the current token is a dot token, which + // was already added to the list. + if (!_lexer.ScanIfMatch(StackFrameKind.DotToken, out var _)) { break; } @@ -174,9 +174,7 @@ private StackFrameParser(VirtualCharSequence text) return null; } - var trailingTrivia = CurrentToken.Kind == StackFrameKind.WhitespaceToken - ? _lexer.ScanWhiteSpace(includePrevious: true) - : null; + var trailingTrivia = _lexer.ScanWhiteSpace(); if (typeIdentifierNodes.Count == 1) { @@ -209,16 +207,6 @@ private StackFrameParser(VirtualCharSequence text) return trailingTrivia.HasValue ? memberAccessExpression.WithTrailingTrivia(trailingTrivia.Value) : memberAccessExpression; - - static bool AllowWhitespace(StackFrameToken token) - => token.Kind - is StackFrameKind.WhitespaceTrivia - or StackFrameKind.CommaToken - or StackFrameKind.OpenBracketToken - or StackFrameKind.CloseBracketToken - or StackFrameKind.OpenParenToken - or StackFrameKind.CloseParenToken - or StackFrameKind.WhitespaceToken; } /// @@ -232,30 +220,29 @@ or StackFrameKind.CloseParenToken /// private StackFrameTypeArgumentList? ParseTypeArguments() { - if (CurrentToken.Kind is not (StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken)) + if (!_lexer.ScanIfMatch( + kind => kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken, + out var openToken)) { return null; } - var openToken = CurrentToken; var useCloseBracket = openToken.Kind is StackFrameKind.OpenBracketToken; using var _ = ArrayBuilder.GetInstance(out var builder); var currentIdentifier = _lexer.ScanIdentifier(); - StackFrameToken? closeToken = null; + StackFrameToken closeToken = default; while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { builder.Add(new StackFrameTypeArgument(currentIdentifier.Value)); - if ((useCloseBracket && CurrentToken.Kind == StackFrameKind.CloseBracketToken) - || CurrentToken.Kind == StackFrameKind.GreaterThanToken) + if (useCloseBracket && _lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, out closeToken)) + { + break; + } + else if (_lexer.ScanIfMatch(StackFrameKind.GreaterThanToken, out closeToken)) { - closeToken = CurrentToken; - - // Consume the token and move on to the next one, since it is already added - // to the list of items for the TypeArgumentList - _lexer.Position++; break; } @@ -263,12 +250,12 @@ or StackFrameKind.CloseParenToken currentIdentifier = _lexer.ScanIdentifier(); } - if (!closeToken.HasValue) + if (closeToken.IsMissing) { return null; } - return new StackFrameTypeArgumentList(openToken, builder.ToImmutable(), closeToken.Value); + return new StackFrameTypeArgumentList(openToken, builder.ToImmutable(), closeToken); } /// @@ -279,15 +266,16 @@ or StackFrameKind.CloseParenToken /// private StackFrameParameterList? ParseMethodParameters() { - if (CurrentToken.Kind != StackFrameKind.OpenParenToken) + if (!_lexer.ScanIfMatch(StackFrameKind.OpenParenToken, out var openParen)) { return null; } + StackFrameToken closeParen; + using var _ = ArrayBuilder.GetInstance(out var builder); - var openParen = CurrentToken; + var identifier = ParseIdentifierExpression(); - var identifier = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: true); while (identifier is not null) { // Check if there's an array type for the identifier @@ -300,30 +288,26 @@ or StackFrameKind.CloseParenToken builder.Add(identifier); } - if (CurrentToken.Kind == StackFrameKind.CloseParenToken) + if (_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out closeParen)) { - return new StackFrameParameterList(openParen, builder.ToImmutable(), CurrentToken); + return new StackFrameParameterList(openParen, builder.ToImmutable(), closeParen); } - // Let whitespace get added as trivia to the identifiers - // for the parameters - if (CurrentToken.Kind != StackFrameKind.WhitespaceToken) + if (_lexer.ScanIfMatch(StackFrameKind.CommaToken, out var commaToken)) { - builder.Add(CurrentToken); + builder.Add(commaToken); } var addLeadingWhitespaceAsTrivia = identifier.ChildAt(identifier.ChildCount - 1).Token.TrailingTrivia.IsDefaultOrEmpty; - identifier = ParseIdentifierExpression(addLeadingWhitespaceAsTrivia: addLeadingWhitespaceAsTrivia); + identifier = ParseIdentifierExpression(); } - if (CurrentToken.Kind != StackFrameKind.CloseParenToken) + if (_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out closeParen)) { - // If we parsed identifiers but never got to a closing portion for the argument list - // we need to bail and return null so we know that this isn't valid - return null; + return new StackFrameParameterList(openParen, builder.ToImmutable(), closeParen); } - return new StackFrameParameterList(openParen, builder.ToImmutable(), CurrentToken); + return null; } /// From 541f3fafc6d31b3fd3ced7012ecf43748889b5fa Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 6 Oct 2021 02:25:13 -0700 Subject: [PATCH 07/59] Parse trivia for method parens --- .../StackFrameParserTests.Generators.cs | 35 ++++++++- .../StackFrame/StackFrameParserTests.cs | 74 ++++++++++++++++--- .../StackFrame/StackFrameParser.cs | 6 ++ 3 files changed, 103 insertions(+), 12 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index eac3be336aa4f..a47141d3048b2 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -33,7 +33,7 @@ private static StackFrameToken CreateToken(StackFrameKind kind, string s, Immuta private static StackFrameTrivia CreateTrivia(StackFrameKind kind, string s) => new(kind, CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, s), ImmutableArray.Empty); - private static ImmutableArray CreateTriviaArray(StackFrameTrivia trivia) + private static ImmutableArray CreateTriviaArray(params StackFrameTrivia[] trivia) => ImmutableArray.Create(trivia); private static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); @@ -46,12 +46,14 @@ private static ImmutableArray CreateTriviaArray(StackFrameTriv private static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); private static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); - private static readonly StackFrameTrivia SpaceTrivia = CreateTrivia(StackFrameKind.WhitespaceTrivia, " "); private static readonly StackFrameTrivia AtTrivia = CreateTrivia(StackFrameKind.AtTrivia, "at "); private static StackFrameParameterList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) => new(OpenParenToken, nodesOrTokens.ToImmutableArray(), CloseParenToken); + private static StackFrameParameterList ArgumentList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameNodeOrToken[] nodesOrTokens) + => new(openToken, nodesOrTokens.ToImmutableArray(), closeToken); + private static StackFrameMethodDeclarationNode MethodDeclaration( StackFrameMemberAccessExpressionNode memberAccessExpression, StackFrameTypeArgumentList? typeArguments = null, @@ -60,6 +62,35 @@ private static StackFrameMethodDeclarationNode MethodDeclaration( return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, argumentList ?? ArgumentList(OpenParenToken, CloseParenToken)); } + private static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + { + StackFrameExpressionNode? current = null; + var identifiers = s.Split('.'); + for (var i = 0; i < identifiers.Length; i++) + { + var identifier = identifiers[i]; + + if (current is null) + { + current = Identifier(identifier, leadingTrivia: leadingTrivia); + } + else if (i == identifiers.Length - 1) + { + current = MemberAccessExpression(current, Identifier(identifier, trailingTrivia: trailingTrivia)); + } + else + { + current = MemberAccessExpression(current, Identifier(identifier)); + } + } + + AssertEx.NotNull(current); + return (StackFrameMemberAccessExpressionNode)current; + } + + private static StackFrameTrivia SpaceTrivia(int count = 1) + => CreateTrivia(StackFrameKind.WhitespaceTrivia, new string(' ', count)); + private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameExpressionNode expressionNode, StackFrameBaseIdentifierNode identifierNode) => new(expressionNode, DotToken, identifierNode); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 7aa6bb07340b2..3e58b46ab483b 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -8,6 +8,53 @@ namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame { public partial class StackFrameParserTests { + [Fact] + public void TestNoParams() + => Verify( + @"at ConsoleApp4.MyClass.M()", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(AtTrivia)), + argumentList: ArgumentList()) + ); + + [Fact] + public void TestNoParams_NoAtTrivia() + => Verify( + @"ConsoleApp4.MyClass.M()", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("ConsoleApp4.MyClass.M"), + argumentList: ArgumentList()) + ); + + [Fact] + public void TestNoParams_SpaceInParams_NoAtTrivia() + => Verify( + @"ConsoleApp4.MyClass.M( )", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("ConsoleApp4.MyClass.M"), + argumentList: ArgumentList( + OpenParenToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia(2))), + CloseParenToken)) + ); + + [Fact] + public void TestNoParams_SpaceTrivia() + => Verify( + @" ConsoleApp4.MyClass.M()", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(SpaceTrivia())), + argumentList: ArgumentList()) + ); + + [Fact] + public void TestNoParams_SpaceTrivia2() + => Verify( + @" ConsoleApp4.MyClass.M()", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(SpaceTrivia(2))), + argumentList: ArgumentList()) + ); + [Fact] public void TestMethodOneParam() => Verify( @@ -20,7 +67,7 @@ public void TestMethodOneParam() Identifier("M")), argumentList: ArgumentList( - Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia())), Identifier("s")) ) ); @@ -37,10 +84,10 @@ public void TestMethodTwoParam() Identifier("M")), argumentList: ArgumentList( - Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia())), Identifier("s"), CommaToken, - Identifier("string", leadingTrivia: CreateTriviaArray(SpaceTrivia), trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("string", leadingTrivia: CreateTriviaArray(SpaceTrivia()), trailingTrivia: CreateTriviaArray(SpaceTrivia())), Identifier("t")) ) ); @@ -58,7 +105,7 @@ public void TestMethodArrayParam() argumentList: ArgumentList( ArrayExpression(Identifier("string"), OpenBracketToken, CloseBracketToken), - Identifier("s", leadingTrivia: CreateTriviaArray(SpaceTrivia))) + Identifier("s", leadingTrivia: CreateTriviaArray(SpaceTrivia()))) ) ); @@ -74,7 +121,7 @@ public void TestGenericMethod_Brackets() Identifier("M")), typeArguments: TypeArgumentList(TypeArgument("T")), argumentList: ArgumentList( - Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia())), Identifier("t")) ) ); @@ -91,7 +138,7 @@ public void TestGenericMethod() Identifier("M")), typeArguments: TypeArgumentList(useBrackets: false, TypeArgument("T")), argumentList: ArgumentList( - Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia())), Identifier("t")) ) ); @@ -101,16 +148,23 @@ public void TestUnderscoreNames() => Verify( @"at _._[_](_ _)", methodDeclaration: MethodDeclaration( - MemberAccessExpression( - Identifier("_", leadingTrivia: CreateTriviaArray(AtTrivia)), - Identifier("_")), + MemberAccessExpression("_._", leadingTrivia: CreateTriviaArray(AtTrivia)), typeArguments: TypeArgumentList(TypeArgument("_")), argumentList: ArgumentList( - Identifier("_", trailingTrivia: CreateTriviaArray(SpaceTrivia)), + Identifier("_", trailingTrivia: CreateTriviaArray(SpaceTrivia())), Identifier("_")) ) ); + [Fact] + public void TestAnonymousMethod() + => Verify( + @"Microsoft.VisualStudio.DesignTools.SurfaceDesigner.Tools.EventRouter.ScopeElement_MouseUp.AnonymousMethod__0()", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("Microsoft.VisualStudio.DesignTools.SurfaceDesigner.Tools.EventRouter.ScopeElement_MouseUp.AnonymousMethod__0"), + argumentList: ArgumentList()) + ); + [Theory] [InlineData(@"at M()")] // Method with no class is invalid [InlineData(@"at M.1c()")] // Invalid start character for identifier diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 71e3a3bd3bdf3..48addced642b6 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -271,6 +271,12 @@ private StackFrameParser(VirtualCharSequence text) return null; } + var spaceTrivia = _lexer.ScanWhiteSpace(); + if (spaceTrivia.HasValue) + { + openParen = openParen.With(trailingTrivia: ImmutableArray.Create(spaceTrivia.Value)); + } + StackFrameToken closeParen; using var _ = ArrayBuilder.GetInstance(out var builder); From 1a0252c0499c81776da49a14e4598b2eef41413d Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 6 Oct 2021 15:45:22 -0700 Subject: [PATCH 08/59] Fix so trailing trivia is on the EOL token. Make sure to use TextTrivia for the trivia. Add tests --- .../StackFrameParserTests.Generators.cs | 4 ++ .../StackFrameParserTests.Utilities.cs | 49 +++++++++++-------- .../StackFrame/StackFrameParserTests.cs | 12 +++++ .../StackFrame/StackFrameCompilationUnit.cs | 12 +++-- .../StackFrame/StackFrameKind.cs | 2 +- .../StackFrame/StackFrameLexer.cs | 14 +++--- .../StackFrame/StackFrameParser.cs | 13 +++-- 7 files changed, 69 insertions(+), 37 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index a47141d3048b2..563d9c151fd48 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -36,6 +36,9 @@ private static StackFrameTrivia CreateTrivia(StackFrameKind kind, string s) private static ImmutableArray CreateTriviaArray(params StackFrameTrivia[] trivia) => ImmutableArray.Create(trivia); + private static ImmutableArray CreateTriviaArray(string s) + => CreateTriviaArray(CreateTrivia(StackFrameKind.TextTrivia, s)); + private static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); private static readonly StackFrameToken CommaToken = CreateToken(StackFrameKind.CommaToken, ","); private static readonly StackFrameToken OpenParenToken = CreateToken(StackFrameKind.OpenParenToken, "("); @@ -45,6 +48,7 @@ private static ImmutableArray CreateTriviaArray(params StackFr private static readonly StackFrameToken LessThanToken = CreateToken(StackFrameKind.LessThanToken, "<"); private static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); private static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); + private static readonly StackFrameToken EOLToken = CreateToken(StackFrameKind.EndOfLine, ""); private static readonly StackFrameTrivia AtTrivia = CreateTrivia(StackFrameKind.AtTrivia, "at "); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index ec5181b8139dd..406afdf6497cb 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -21,7 +21,7 @@ namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame public partial class StackFrameParserTests { - private static void Verify(string input, StackFrameMethodDeclarationNode? methodDeclaration = null, bool expectFailure = false) + private static void Verify(string input, StackFrameMethodDeclarationNode? methodDeclaration = null, bool expectFailure = false, StackFrameToken? eolTokenOpt = null) { var tree = StackFrameParser.TryParse(input); if (expectFailure) @@ -42,9 +42,11 @@ private static void Verify(string input, StackFrameMethodDeclarationNode? method AssertEqual(methodDeclaration, tree.Root.MethodDeclaration); } - Assert.False(tree.Root.InTrivia.HasValue); - Assert.Null(tree.Root.FileInformationExpression); - Assert.False(tree.Root.TrailingTrivia.HasValue); + var eolToken = eolTokenOpt.HasValue + ? eolTokenOpt.Value + : CreateToken(StackFrameKind.EndOfLine, ""); + + AssertEqual(eolToken, tree.Root.EndOfLineToken); } private static void AssertEqual(StackFrameNodeOrToken expected, StackFrameNodeOrToken actual) { @@ -206,9 +208,9 @@ private static IEnumerable EnumerateTree(StackFrameTree tre yield return seq; } - if (root.TrailingTrivia.HasValue) + foreach (var charSequence in Enumerate(root.EndOfLineToken)) { - yield return root.TrailingTrivia.Value.VirtualChars; + yield return charSequence; } } @@ -225,24 +227,31 @@ private static IEnumerable Enumerate(StackFrameNode node) } else if (!nodeOrToken.Token.VirtualChars.IsDefaultOrEmpty) { - var token = nodeOrToken.Token; - if (!token.LeadingTrivia.IsDefault) + foreach (var charSequence in Enumerate(nodeOrToken.Token)) { - foreach (var trivia in token.LeadingTrivia) - { - yield return trivia.VirtualChars; - } + yield return charSequence; } + } + } + } - yield return nodeOrToken.Token.VirtualChars; + private static IEnumerable Enumerate(StackFrameToken token) + { + if (!token.LeadingTrivia.IsDefault) + { + foreach (var trivia in token.LeadingTrivia) + { + yield return trivia.VirtualChars; + } + } - if (!token.TrailingTrivia.IsDefault) - { - foreach (var trivia in token.TrailingTrivia) - { - yield return trivia.VirtualChars; - } - } + yield return token.VirtualChars; + + if (!token.TrailingTrivia.IsDefault) + { + foreach (var trivia in token.TrailingTrivia) + { + yield return trivia.VirtualChars; } } } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 3e58b46ab483b..e6cfd7ed2e6aa 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -17,6 +17,17 @@ public void TestNoParams() argumentList: ArgumentList()) ); + [Fact] + public void TestTrailingTrivia() + => Verify( + @"at ConsoleApp4.MyClass.M() some other text", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(AtTrivia)), + argumentList: ArgumentList()), + + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" some other text")) + ); + [Fact] public void TestNoParams_NoAtTrivia() => Verify( @@ -170,6 +181,7 @@ public void TestAnonymousMethod() [InlineData(@"at M.1c()")] // Invalid start character for identifier [InlineData(@"at 1M.C()")] [InlineData(@"at M.C(string& s)")] // "string&" represents a reference (ref, out) and is not supported yet + [InlineData(@"at StreamJsonRpc.JsonRpc.d__139`1.MoveNext()")] // Generated/Inline methods are not supported yet public void TestInvalidInputs(string input) => Verify(input, expectFailure: true); } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs index bdfcba137b5c7..cc8b4b4113e59 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs @@ -6,19 +6,21 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { + using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; + using StackFrameToken = EmbeddedSyntaxToken; + using StackFrameTrivia = EmbeddedSyntaxTrivia; + internal class StackFrameCompilationUnit { public readonly StackFrameMethodDeclarationNode MethodDeclaration; - public readonly EmbeddedSyntaxTrivia? InTrivia; public readonly StackFrameFileInformationNode? FileInformationExpression; - public readonly EmbeddedSyntaxTrivia? TrailingTrivia; + public readonly StackFrameToken EndOfLineToken; - public StackFrameCompilationUnit(StackFrameMethodDeclarationNode methodDeclaration, EmbeddedSyntaxTrivia? inTrivia, StackFrameFileInformationNode? fileInformationExpression, EmbeddedSyntaxTrivia? trailingTrivia) + public StackFrameCompilationUnit(StackFrameMethodDeclarationNode methodDeclaration, StackFrameFileInformationNode? fileInformationExpression, StackFrameToken endOfLineToken) { MethodDeclaration = methodDeclaration; - InTrivia = inTrivia; FileInformationExpression = fileInformationExpression; - TrailingTrivia = trailingTrivia; + EndOfLineToken = endOfLineToken; } } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index ab8b303851fa9..8b0bd460132ac 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -46,6 +46,6 @@ internal enum StackFrameKind WhitespaceTrivia, AtTrivia, // "at " portion of the stack frame InTrivia, // optional " in " portion of the stack frame - TrailingTrivia, // All trailing text that is not syntactically relavent + TextTrivia, } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 0d89f8766a999..74a2d6ea74157 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -39,18 +39,17 @@ public VirtualCharSequence GetSubPatternToCurrentPos(int start) public VirtualCharSequence GetSubPattern(int start, int end) => Text.GetSubSequence(TextSpan.FromBounds(start, end)); - /// - /// Scans until EndOfLine is found, treating all text as trivia - /// - public StackFrameTrivia? ScanTrailingTrivia() + public StackFrameTrivia? ScanRemainingTrivia() { if (Position == Text.Length) { return null; } - var length = Text.Length - Position; - return CreateTrivia(StackFrameKind.TrailingTrivia, Text.GetSubSequence(new TextSpan(Position - 1, length))); + var start = Position; + Position = Text.Length; + + return CreateTrivia(StackFrameKind.TextTrivia, GetSubPatternToCurrentPos(start)); } public StackFrameToken? ScanIdentifier() @@ -74,8 +73,7 @@ public VirtualCharSequence GetSubPattern(int start, int end) ch = CurrentChar; } - var identifierSpan = new TextSpan(startPosition, Position - startPosition); - var identifier = CreateToken(StackFrameKind.IdentifierToken, Text.GetSubSequence(identifierSpan)); + var identifier = CreateToken(StackFrameKind.IdentifierToken, GetSubPatternToCurrentPos(startPosition)); return identifier; } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 48addced642b6..152fe12326c9f 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -83,12 +83,19 @@ private StackFrameParser(VirtualCharSequence text) var fileInformationExpression = inTrivia.HasValue ? ParseFileInformation() : null; - var trailingTrivia = _lexer.ScanTrailingTrivia(); + + var trailingTrivia = _lexer.ScanRemainingTrivia(); + var eolToken = CurrentToken; Debug.Assert(_lexer.Position == _lexer.Text.Length); - Debug.Assert(_lexer.CurrentCharAsToken().Kind == StackFrameKind.EndOfLine); + Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); + + if (trailingTrivia.HasValue) + { + eolToken = eolToken.With(leadingTrivia: ImmutableArray.Create(trailingTrivia.Value)); + } - var root = new StackFrameCompilationUnit(methodDeclaration, inTrivia, fileInformationExpression, trailingTrivia); + var root = new StackFrameCompilationUnit(methodDeclaration, fileInformationExpression, eolToken); return new StackFrameTree( _lexer.Text, root, ImmutableArray.Empty); From 35099cc83709966f327d19b3a9e720023833cac2 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Thu, 7 Oct 2021 00:45:02 -0700 Subject: [PATCH 09/59] Get file parsing working and add a simple test --- .../StackFrameParserTests.Generators.cs | 12 ++ .../StackFrameParserTests.Utilities.cs | 32 ++++- .../StackFrame/StackFrameParserTests.cs | 43 +++++- .../StackFrame/IStackFrameNodeVisitor.cs | 1 + .../StackFrame/StackFrameKind.cs | 5 +- .../StackFrame/StackFrameLexer.cs | 99 +++++++++++++- .../StackFrame/StackFrameNodeDefinitions.cs | 30 +++-- .../StackFrame/StackFrameParser.cs | 122 ++++++++++++++++-- 8 files changed, 309 insertions(+), 35 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index 563d9c151fd48..d6a06b89d4699 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -49,8 +49,11 @@ private static ImmutableArray CreateTriviaArray(string s) private static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); private static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); private static readonly StackFrameToken EOLToken = CreateToken(StackFrameKind.EndOfLine, ""); + private static readonly StackFrameToken ColonToken = CreateToken(StackFrameKind.ColonToken, ":"); private static readonly StackFrameTrivia AtTrivia = CreateTrivia(StackFrameKind.AtTrivia, "at "); + private static readonly StackFrameTrivia LineTrivia = CreateTrivia(StackFrameKind.LineTrivia, "line "); + private static readonly StackFrameTrivia InTrivia = CreateTrivia(StackFrameKind.InTrivia, " in "); private static StackFrameParameterList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) => new(OpenParenToken, nodesOrTokens.ToImmutableArray(), CloseParenToken); @@ -136,5 +139,14 @@ private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, par private static StackFrameTypeArgument TypeArgument(string identifier) => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); + + private static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken? colon = null, StackFrameToken? line = null) + => new(path.With(leadingTrivia: CreateTriviaArray(InTrivia)), colon, line); + + private static StackFrameToken Path(string path) + => CreateToken(StackFrameKind.PathToken, path); + + private static StackFrameToken Line(int lineNumber) + => CreateToken(StackFrameKind.NumberToken, lineNumber.ToString(), leadingTrivia: ImmutableArray.Create(LineTrivia)); } } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 406afdf6497cb..55280da4736a7 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -21,7 +21,7 @@ namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame public partial class StackFrameParserTests { - private static void Verify(string input, StackFrameMethodDeclarationNode? methodDeclaration = null, bool expectFailure = false, StackFrameToken? eolTokenOpt = null) + private static void Verify(string input, StackFrameMethodDeclarationNode? methodDeclaration = null, bool expectFailure = false, StackFrameFileInformationNode? fileInformation = null, StackFrameToken? eolTokenOpt = null) { var tree = StackFrameParser.TryParse(input); if (expectFailure) @@ -42,6 +42,15 @@ private static void Verify(string input, StackFrameMethodDeclarationNode? method AssertEqual(methodDeclaration, tree.Root.MethodDeclaration); } + if (fileInformation is null) + { + Assert.Null(tree.Root.FileInformationExpression); + } + else + { + AssertEqual(fileInformation, tree.Root.FileInformationExpression); + } + var eolToken = eolTokenOpt.HasValue ? eolTokenOpt.Value : CreateToken(StackFrameKind.EndOfLine, ""); @@ -50,8 +59,15 @@ private static void Verify(string input, StackFrameMethodDeclarationNode? method } private static void AssertEqual(StackFrameNodeOrToken expected, StackFrameNodeOrToken actual) { - AssertEqual(expected.Node, actual.Node); - AssertEqual(expected.Token, actual.Token); + Assert.Equal(expected.IsNode, actual.IsNode); + if (expected.IsNode) + { + AssertEqual(expected.Node, actual.Node); + } + else + { + AssertEqual(expected.Token, actual.Token); + } } private static void AssertEqual(StackFrameNode? expected, StackFrameNode? actual) @@ -161,7 +177,7 @@ private static void VerifyCharacterSpans(string originalText, StackFrameTree tre } // Make sure we enumerated the total input - Assert.Equal(index, textSeq.Length); + Assert.Equal(textSeq.Length, index); string PrintDifference() { @@ -208,6 +224,14 @@ private static IEnumerable EnumerateTree(StackFrameTree tre yield return seq; } + if (root.FileInformationExpression is not null) + { + foreach (var seq in Enumerate(root.FileInformationExpression)) + { + yield return seq; + } + } + foreach (var charSequence in Enumerate(root.EndOfLineToken)) { yield return charSequence; diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index e6cfd7ed2e6aa..506c14dd1823e 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -154,16 +154,24 @@ public void TestGenericMethod() ) ); - [Fact] - public void TestUnderscoreNames() + [Theory] + [InlineData("_")] + [InlineData("_s")] + [InlineData("S0m3th1ng")] + [InlineData("ü")] // Unicode character + [InlineData("uʶ")] // character and modifier character + [InlineData("a\u00AD")] // Soft hyphen formatting character + [InlineData("a‿")] // Connecting punctuation (combining character + + public void TestIdentifierNames(string identifierName) => Verify( - @"at _._[_](_ _)", + @$"at {identifierName}.{identifierName}[{identifierName}]({identifierName} {identifierName})", methodDeclaration: MethodDeclaration( - MemberAccessExpression("_._", leadingTrivia: CreateTriviaArray(AtTrivia)), - typeArguments: TypeArgumentList(TypeArgument("_")), + MemberAccessExpression($"{identifierName}.{identifierName}", leadingTrivia: CreateTriviaArray(AtTrivia)), + typeArguments: TypeArgumentList(TypeArgument(identifierName)), argumentList: ArgumentList( - Identifier("_", trailingTrivia: CreateTriviaArray(SpaceTrivia())), - Identifier("_")) + Identifier(identifierName, trailingTrivia: CreateTriviaArray(SpaceTrivia())), + Identifier(identifierName)) ) ); @@ -176,12 +184,33 @@ public void TestAnonymousMethod() argumentList: ArgumentList()) ); + [Fact] + public void TestFileInformation() + => Verify( + @"M.M() in C:\folder\m.cs:line 1", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("M.M"), + argumentList: ArgumentList()), + + fileInformation: FileInformation( + Path(@"C:\folder\m.cs"), + ColonToken, + Line(1)) + ); + [Theory] [InlineData(@"at M()")] // Method with no class is invalid [InlineData(@"at M.1c()")] // Invalid start character for identifier [InlineData(@"at 1M.C()")] [InlineData(@"at M.C(string& s)")] // "string&" represents a reference (ref, out) and is not supported yet [InlineData(@"at StreamJsonRpc.JsonRpc.d__139`1.MoveNext()")] // Generated/Inline methods are not supported yet + [InlineData(@"at M(")] // Missing closing paren + [InlineData(@"at M)")] // MIssing open paren + [InlineData(@"at M.M[T>(T t)")] // Mismatched generic opening/close + [InlineData(@"at M.M Verify(input, expectFailure: true); } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index 4a1178fd27e5a..75528a5a7df64 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -18,5 +18,6 @@ internal interface IStackFrameNodeVisitor void Visit(StackFrameGenericTypeIdentifier stackFrameGenericTypeIdentifier); void Visit(StackFrameTypeArgument stackFrameTypeArgument); void Visit(StackFrameArrayExpressionNode stackFrameArrayExpressionNode); + void Visit(StackFrameFileInformationNode stackFrameFileInformationNode); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 8b0bd460132ac..97ae963086713 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -19,6 +19,7 @@ internal enum StackFrameKind TypeIdentifier, ParameterList, ArrayExpression, + FileInformation, // Tokens AmpersandToken, @@ -40,12 +41,14 @@ internal enum StackFrameKind BackslashToken, ForwardSlashToken, IdentifierToken, - WhitespaceToken, + PathToken, + NumberToken, // Trivia WhitespaceTrivia, AtTrivia, // "at " portion of the stack frame InTrivia, // optional " in " portion of the stack frame + LineTrivia, // optional "line " string indicating the line number of a file TextTrivia, } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 74a2d6ea74157..c9210e3265fc2 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -6,7 +6,9 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.IO; using System.Linq; +using System.Text; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.PooledObjects; @@ -240,7 +242,7 @@ private static StackFrameKind GetKind(VirtualChar ch) '`' => StackFrameKind.GraveAccentToken, '\\' => StackFrameKind.BackslashToken, '/' => StackFrameKind.ForwardSlashToken, - _ => IsBlank(ch) ? StackFrameKind.WhitespaceToken : StackFrameKind.TextToken, + _ => IsBlank(ch) ? StackFrameKind.WhitespaceTrivia : StackFrameKind.TextToken, }; private static bool IsNumber(VirtualChar ch) @@ -252,7 +254,7 @@ private static bool IsNumber(VirtualChar ch) public StackFrameTrivia? ScanAtTrivia() { - if (Position >= Text.Length) + if (Position == Text.Length) { return null; } @@ -273,7 +275,7 @@ private static bool IsNumber(VirtualChar ch) public StackFrameTrivia? ScanInTrivia() { - if (Position >= Text.Length) + if (Position == Text.Length) { return null; } @@ -286,12 +288,101 @@ private static bool IsNumber(VirtualChar ch) var start = Position; Position += InString.Length; - return CreateTrivia(StackFrameKind.AtTrivia, GetSubPatternToCurrentPos(start)); + return CreateTrivia(StackFrameKind.InTrivia, GetSubPatternToCurrentPos(start)); + } + + return null; + } + + /// + /// Attempts to parse a path following https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names + /// + internal StackFrameToken? ScanPath() + { + if (Position == Text.Length) + { + return null; + } + + var startPosition = Position; + var invalidChars = Path.GetInvalidPathChars(); + var isRooted = false; + + while (!invalidChars.Contains((char)CurrentChar.Value)) + { + // If the path is rooted then we no longer accept + if (isRooted && IsInvalidCharForRootedPath((char)CurrentChar.Value)) + { + break; + } + + Position++; + + if (!isRooted) + { + var str = GetSubPatternToCurrentPos(startPosition).CreateString(); + isRooted = Path.IsPathRooted(str); + } + } + + if (startPosition == Position) + { + return null; + } + + return CreateToken(StackFrameKind.PathToken, GetSubPatternToCurrentPos(startPosition)); + + static bool IsInvalidCharForRootedPath(char c) + => c switch + { + '<' or + '>' or + '"' or + '|' or + '?' or + '*' or + ':' => true, + + _ => false + }; + } + + internal StackFrameTrivia? ScanLineTrivia() + { + if (Position == Text.Length) + { + return null; + } + + // TODO: Handle multiple languages? Right now we're going to only parse english + const string LineString = "line "; + if (IsStringAtPosition(LineString)) + { + var start = Position; + Position += LineString.Length; + + return CreateTrivia(StackFrameKind.LineTrivia, GetSubPatternToCurrentPos(start)); } return null; } + internal StackFrameToken? ScanNumbers() + { + if (Position == Text.Length) + { + return null; + } + + var start = Position; + while (IsNumber(CurrentChar)) + { + Position++; + } + + return CreateToken(StackFrameKind.NumberToken, GetSubPatternToCurrentPos(start)); + } + public static bool IsBlank(VirtualChar ch) { // List taken from the native regex parser. diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index 266e8a6d3b74d..fe7f4d789b6a1 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -362,20 +362,34 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameFileInformationNode : StackFrameNode { - public StackFrameFileInformationNode(StackFrameKind kind) : base(kind) + public readonly StackFrameToken Path; + public readonly StackFrameToken? Colon; + public readonly StackFrameToken? Line; + + public StackFrameFileInformationNode(StackFrameToken path, StackFrameToken? colon = null, StackFrameToken? line = null) : base(StackFrameKind.FileInformation) { + Path = path; + Colon = colon; + Line = line; } - internal override int ChildCount => throw new NotImplementedException(); + internal override int ChildCount => 3; public override void Accept(IStackFrameNodeVisitor visitor) - { - throw new NotImplementedException(); - } + => visitor.Visit(this); internal override StackFrameNodeOrToken ChildAt(int index) - { - throw new NotImplementedException(); - } + => index switch + { + 0 => Path, + 1 => Colon.HasValue ? Colon.Value : null, + 2 => Line.HasValue ? Line.Value : null, + _ => throw new InvalidOperationException() + }; + + internal StackFrameFileInformationNode WithLeadingTrivia(StackFrameTrivia inTrivia) + => new(Path.With(leadingTrivia: ImmutableArray.Create(inTrivia)), + Colon, + Line); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 152fe12326c9f..6d72dab4a782f 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -6,10 +6,13 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.IO; using System.Linq; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.PooledObjects; +using Microsoft.CodeAnalysis.Shared.Utilities; +using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame @@ -80,22 +83,50 @@ private StackFrameParser(VirtualCharSequence text) } var inTrivia = _lexer.ScanInTrivia(); - var fileInformationExpression = inTrivia.HasValue + var (fileInformation, isFilePathValid) = inTrivia.HasValue ? ParseFileInformation() - : null; + : (null, false); - var trailingTrivia = _lexer.ScanRemainingTrivia(); - var eolToken = CurrentToken; + using var _ = ArrayBuilder.GetInstance(out var trailingTriviaBuilder); - Debug.Assert(_lexer.Position == _lexer.Text.Length); - Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); + if (inTrivia.HasValue) + { + if (isFilePathValid) + { + // If the path is valid, just add the inTrivia to the file information + RoslynDebug.AssertNotNull(fileInformation); + fileInformation = fileInformation.WithLeadingTrivia(inTrivia.Value); + } + else + { + // If we parsed a path but it's not valid for the file system, + // combine both with the remaining trivia as text + trailingTriviaBuilder.Add(inTrivia.Value); + + if (fileInformation is not null) + { + // If the path isn't valid we don't expect the line or colon trivia + // to exist on the expression + Debug.Assert(!fileInformation.Line.HasValue); + Debug.Assert(!fileInformation.Colon.HasValue); + + trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, fileInformation.Path.VirtualChars)); + } + } + } - if (trailingTrivia.HasValue) + var remainingTrivia = _lexer.ScanRemainingTrivia(); + if (remainingTrivia.HasValue) { - eolToken = eolToken.With(leadingTrivia: ImmutableArray.Create(trailingTrivia.Value)); + trailingTriviaBuilder.Add(remainingTrivia.Value); } - var root = new StackFrameCompilationUnit(methodDeclaration, fileInformationExpression, eolToken); + var eolToken = CurrentToken.With(leadingTrivia: trailingTriviaBuilder.ToImmutable()); + + Debug.Assert(_lexer.Position == _lexer.Text.Length); + Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); + + var root = new StackFrameCompilationUnit(methodDeclaration, fileInformation, eolToken); return new StackFrameTree( _lexer.Text, root, ImmutableArray.Empty); @@ -339,9 +370,78 @@ private StackFrameExpressionNode ParseArrayIdentifier(StackFrameExpressionNode i return new StackFrameArrayExpressionNode(identifier, arrayBrackets); } - public StackFrameFileInformationNode ParseFileInformation() + /// + /// Parses text for a valid file path using valid file characters. It's very possible this includes a path that doesn't exist but + /// forms a valid path identifier. + /// + public (StackFrameFileInformationNode? fileInformation, bool isPathValid) ParseFileInformation() { - throw new NotImplementedException(); + var path = _lexer.ScanPath(); + if (!path.HasValue) + { + return (null, false); + } + + // Make sure all the parts are valid as a file path + var isValidFile = IOUtilities.PerformIO(() => + { + var pathStr = path.Value.VirtualChars.CreateString(); + var file = new FileInfo(pathStr); + var invalidFileChars = Path.GetInvalidFileNameChars(); + if (file.Name.Any(c => invalidFileChars.Contains(c))) + { + return false; + } + + var directory = file.Directory; + var invalidDirectoryChars = Path.GetInvalidPathChars(); + if (directory.FullName.Any(c => invalidDirectoryChars.Contains(c))) + { + return false; + } + + return true; + }); + + if (!isValidFile) + { + return (new(path.Value), false); + } + + if (!_lexer.ScanIfMatch(StackFrameKind.ColonToken, out var colonToken)) + { + return (new(path.Value), true); + } + + var lineIdentifier = _lexer.ScanLineTrivia(); + if (!lineIdentifier.HasValue) + { + // malformed, we have a ": " with no "line " trivia + // add the colonToken as trivia to the valid path and return it + var colonTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.Text, colonToken.VirtualChars); + return + (new(path.Value.With(trailingTrivia: ImmutableArray.Create(colonTrivia))) + , true); + } + + var numbers = _lexer.ScanNumbers(); + if (!numbers.HasValue) + { + // malformed, we have a ":line " but no following number. + // Add the colon and line trivia as trailing trivia + var jointTriviaSpan = new TextSpan(colonToken.GetSpan().Start, colonToken.VirtualChars.Length + lineIdentifier.Value.VirtualChars.Length); + var trailingTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.Text, _lexer.Text.GetSubSequence(jointTriviaSpan)); + return + (new(path.Value.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))) + , true); + } + + return + (new( + path.Value, + colonToken, + numbers.Value.With(leadingTrivia: ImmutableArray.Create(lineIdentifier.Value))) + , true); } } } From 59e54aa5f5f63858456f596360618a28a84ec40c Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Thu, 7 Oct 2021 00:56:48 -0700 Subject: [PATCH 10/59] change to TryParse* naming --- .../StackFrame/StackFrameParser.cs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 6d72dab4a782f..ecaf552eefeff 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -45,7 +45,7 @@ private StackFrameParser(VirtualCharSequence text) try { - return new StackFrameParser(text).ParseTree(); + return new StackFrameParser(text).TryParseTree(); } catch (InsufficientExecutionStackException) { @@ -62,11 +62,11 @@ private StackFrameParser(VirtualCharSequence text) /// /// Attempts to parse the full tree. Returns null on malformed data /// - private StackFrameTree? ParseTree() + private StackFrameTree? TryParseTree() { var atTrivia = _lexer.ScanAtTrivia(); - var methodDeclaration = ParseMethodDeclaration(); + var methodDeclaration = TryParseMethodDeclaration(); if (methodDeclaration is null) { return null; @@ -84,7 +84,7 @@ private StackFrameParser(VirtualCharSequence text) var inTrivia = _lexer.ScanInTrivia(); var (fileInformation, isFilePathValid) = inTrivia.HasValue - ? ParseFileInformation() + ? TryParseFileInformation() : (null, false); using var _ = ArrayBuilder.GetInstance(out var trailingTriviaBuilder); @@ -138,16 +138,16 @@ private StackFrameParser(VirtualCharSequence text) /// /// Ex: [|MyClass.MyMethod(string s)|] /// - private StackFrameMethodDeclarationNode? ParseMethodDeclaration() + private StackFrameMethodDeclarationNode? TryParseMethodDeclaration() { - var identifierExpression = ParseIdentifierExpression(); + var identifierExpression = TryParseIdentifierExpression(); if (identifierExpression is not StackFrameMemberAccessExpressionNode memberAccessExpression) { return null; } - var typeArguments = ParseTypeArguments(); - var arguments = ParseMethodParameters(); + var typeArguments = TryParseTypeArguments(); + var arguments = TryParseMethodParameters(); if (arguments is null) { @@ -170,7 +170,7 @@ private StackFrameParser(VirtualCharSequence text) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) /// - private StackFrameExpressionNode? ParseIdentifierExpression() + private StackFrameExpressionNode? TryParseIdentifierExpression() { Queue<(StackFrameBaseIdentifierNode identifier, StackFrameToken separator)> typeIdentifierNodes = new(); @@ -256,7 +256,7 @@ private StackFrameParser(VirtualCharSequence text) /// Assumes the identifier "MyMethod" has already been parsed, and the type arguments will need to be parsed. /// Returns null if no type arguments are found or if they are malformed. /// - private StackFrameTypeArgumentList? ParseTypeArguments() + private StackFrameTypeArgumentList? TryParseTypeArguments() { if (!_lexer.ScanIfMatch( kind => kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken, @@ -302,7 +302,7 @@ private StackFrameParser(VirtualCharSequence text) /// /// Returns null in cases where the input is malformed. /// - private StackFrameParameterList? ParseMethodParameters() + private StackFrameParameterList? TryParseMethodParameters() { if (!_lexer.ScanIfMatch(StackFrameKind.OpenParenToken, out var openParen)) { @@ -318,14 +318,14 @@ private StackFrameParser(VirtualCharSequence text) StackFrameToken closeParen; using var _ = ArrayBuilder.GetInstance(out var builder); - var identifier = ParseIdentifierExpression(); + var identifier = TryParseIdentifierExpression(); while (identifier is not null) { // Check if there's an array type for the identifier if (CurrentToken.Kind == StackFrameKind.OpenBracketToken) { - builder.Add(ParseArrayIdentifier(identifier)); + builder.Add(TryParseArrayIdentifier(identifier)); } else { @@ -343,7 +343,7 @@ private StackFrameParser(VirtualCharSequence text) } var addLeadingWhitespaceAsTrivia = identifier.ChildAt(identifier.ChildCount - 1).Token.TrailingTrivia.IsDefaultOrEmpty; - identifier = ParseIdentifierExpression(); + identifier = TryParseIdentifierExpression(); } if (_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out closeParen)) @@ -359,7 +359,7 @@ private StackFrameParser(VirtualCharSequence text) /// passed in, converts it into by parsing the array portion /// of the identifier. /// - private StackFrameExpressionNode ParseArrayIdentifier(StackFrameExpressionNode identifier) + private StackFrameExpressionNode TryParseArrayIdentifier(StackFrameExpressionNode identifier) { if (CurrentToken.Kind != StackFrameKind.OpenBracketToken) { @@ -374,7 +374,7 @@ private StackFrameExpressionNode ParseArrayIdentifier(StackFrameExpressionNode i /// Parses text for a valid file path using valid file characters. It's very possible this includes a path that doesn't exist but /// forms a valid path identifier. /// - public (StackFrameFileInformationNode? fileInformation, bool isPathValid) ParseFileInformation() + public (StackFrameFileInformationNode? fileInformation, bool isPathValid) TryParseFileInformation() { var path = _lexer.ScanPath(); if (!path.HasValue) From 7d6ab41d44c5f069f0a3ac47d6c2b314ce20b74e Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Thu, 7 Oct 2021 16:04:09 -0700 Subject: [PATCH 11/59] Fix array bracket scanning --- .../StackFrame/StackFrameLexer.cs | 29 ---------- .../StackFrame/StackFrameParser.cs | 53 ++++++++++++++++--- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index c9210e3265fc2..efe4ac45c26cf 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -101,35 +101,6 @@ public StackFrameToken ScanTypeArity() return arityToken; } - internal ImmutableArray ScanArrayBrackets() - { - if (Position == Text.Length) - { - return default; - } - - using var _ = ArrayBuilder.GetInstance(out var builder); - - while (Position < Text.Length) - { - var kind = GetKind(CurrentChar); - if (!IsArrayBracket(kind)) - { - break; - } - - builder.Add(CurrentCharAsToken()); - Position++; - } - - Debug.Assert(builder.Count >= 2); - - return builder.ToImmutable(); - - static bool IsArrayBracket(StackFrameKind kind) - => kind is StackFrameKind.OpenBracketToken or StackFrameKind.CloseBracketToken; - } - internal StackFrameTrivia? ScanWhiteSpace() { if (Position == Text.Length) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index ecaf552eefeff..d01d7946ea9ec 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -325,7 +325,15 @@ private StackFrameParser(VirtualCharSequence text) // Check if there's an array type for the identifier if (CurrentToken.Kind == StackFrameKind.OpenBracketToken) { - builder.Add(TryParseArrayIdentifier(identifier)); + if (TryParseArrayIdentifier(identifier, out var parsedTokens)) + { + builder.Add(new StackFrameArrayExpressionNode(identifier, parsedTokens)); + } + else + { + // Invalid array identifiers, bail parsing parameters + return null; + } } else { @@ -359,22 +367,42 @@ private StackFrameParser(VirtualCharSequence text) /// passed in, converts it into by parsing the array portion /// of the identifier. /// - private StackFrameExpressionNode TryParseArrayIdentifier(StackFrameExpressionNode identifier) + private bool TryParseArrayIdentifier(StackFrameExpressionNode identifier, out ImmutableArray arrayTokens) { - if (CurrentToken.Kind != StackFrameKind.OpenBracketToken) + using var _ = ArrayBuilder.GetInstance(out var builder); + + while (true) { - throw new InvalidOperationException(); + if (!_lexer.ScanIfMatch(StackFrameKind.OpenBracketToken, out var openBracket)) + { + arrayTokens = builder.ToImmutable(); + return arrayTokens.Length > 0 && arrayTokens[^1].Kind == StackFrameKind.CloseBracketToken; + } + + builder.Add(AddTrailingWhitespace(openBracket)); + + if (_lexer.ScanIfMatch(StackFrameKind.CommaToken, out var commaToken)) + { + builder.Add(AddTrailingWhitespace(commaToken)); + } + + if (!_lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, out var closeBracket)) + { + arrayTokens = builder.ToImmutable(); + return arrayTokens.Length > 0 && arrayTokens.Last().Kind == StackFrameKind.CloseBracketToken; + } + + builder.Add(AddTrailingWhitespace(closeBracket)); } - var arrayBrackets = _lexer.ScanArrayBrackets(); - return new StackFrameArrayExpressionNode(identifier, arrayBrackets); + throw ExceptionUtilities.Unreachable; } /// /// Parses text for a valid file path using valid file characters. It's very possible this includes a path that doesn't exist but /// forms a valid path identifier. /// - public (StackFrameFileInformationNode? fileInformation, bool isPathValid) TryParseFileInformation() + private (StackFrameFileInformationNode? fileInformation, bool isPathValid) TryParseFileInformation() { var path = _lexer.ScanPath(); if (!path.HasValue) @@ -443,5 +471,16 @@ private StackFrameExpressionNode TryParseArrayIdentifier(StackFrameExpressionNod numbers.Value.With(leadingTrivia: ImmutableArray.Create(lineIdentifier.Value))) , true); } + + private StackFrameToken AddTrailingWhitespace(StackFrameToken token) + { + var whitespace = _lexer.ScanWhiteSpace(); + if (whitespace.HasValue) + { + return token.With(trailingTrivia: ImmutableArray.Create(whitespace.Value)); + } + + return token; + } } } From 1830c574835276e77d37a774e87bd80b34a23582 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Fri, 8 Oct 2021 11:57:38 -0700 Subject: [PATCH 12/59] More file information tests. Clean up array and arity parsing. --- .../StackFrameParserTests.Generators.cs | 4 +- .../StackFrameParserTests.Utilities.cs | 9 +-- .../StackFrame/StackFrameParserTests.cs | 68 ++++++++++++++++++- .../StackFrame/StackFrameKind.cs | 1 - .../StackFrame/StackFrameLexer.cs | 22 ------ .../StackFrame/StackFrameNodeDefinitions.cs | 2 +- .../StackFrame/StackFrameParser.cs | 48 +++++++++++-- 7 files changed, 114 insertions(+), 40 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index d6a06b89d4699..93769f085c7eb 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -36,8 +36,8 @@ private static StackFrameTrivia CreateTrivia(StackFrameKind kind, string s) private static ImmutableArray CreateTriviaArray(params StackFrameTrivia[] trivia) => ImmutableArray.Create(trivia); - private static ImmutableArray CreateTriviaArray(string s) - => CreateTriviaArray(CreateTrivia(StackFrameKind.TextTrivia, s)); + private static ImmutableArray CreateTriviaArray(params string[] strings) + => strings.Select(s => CreateTrivia(StackFrameKind.TextTrivia, s)).ToImmutableArray(); private static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); private static readonly StackFrameToken CommaToken = CreateToken(StackFrameKind.CommaToken, ","); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 55280da4736a7..be2aabe4ad804 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -149,12 +149,12 @@ private static void Print(ImmutableArray triviaArray, StringBu private static void AssertEqual(StackFrameToken expected, StackFrameToken actual) { - AssertEqual(expected.LeadingTrivia, actual.LeadingTrivia); - AssertEqual(expected.TrailingTrivia, actual.TrailingTrivia); - Assert.Equal(expected.Kind, actual.Kind); Assert.Equal(expected.IsMissing, actual.IsMissing); Assert.Equal(expected.VirtualChars.CreateString(), actual.VirtualChars.CreateString()); + + AssertEqual(expected.LeadingTrivia, actual.LeadingTrivia, expected); + AssertEqual(expected.TrailingTrivia, actual.TrailingTrivia, expected); } private static void VerifyCharacterSpans(string originalText, StackFrameTree tree) @@ -280,7 +280,7 @@ private static IEnumerable Enumerate(StackFrameToken token) } } - private static void AssertEqual(ImmutableArray expected, ImmutableArray actual) + private static void AssertEqual(ImmutableArray expected, ImmutableArray actual, StackFrameToken token) { var diffMessage = PrintDiff(); @@ -301,6 +301,7 @@ private static void AssertEqual(ImmutableArray expected, Immut string PrintDiff() { var sb = new StringBuilder(); + sb.AppendLine($"Trivia is different on {token.Kind}"); sb.Append("Expected: "); if (!expected.IsDefaultOrEmpty) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 506c14dd1823e..31739974c4d1c 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -115,8 +115,25 @@ public void TestMethodArrayParam() Identifier("M")), argumentList: ArgumentList( - ArrayExpression(Identifier("string"), OpenBracketToken, CloseBracketToken), - Identifier("s", leadingTrivia: CreateTriviaArray(SpaceTrivia()))) + ArrayExpression(Identifier("string"), OpenBracketToken, CloseBracketToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia()))), + Identifier("s")) + ) + ); + + [Fact] + public void TestCommaArrayParam() + => Verify( + @"at ConsoleApp4.MyClass.M(string[,] s)", + methodDeclaration: MethodDeclaration( + MemberAccessExpression( + MemberAccessExpression( + Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), + Identifier("MyClass")), + Identifier("M")), + + argumentList: ArgumentList( + ArrayExpression(Identifier("string"), OpenBracketToken, CommaToken, CloseBracketToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia()))), + Identifier("s")) ) ); @@ -198,6 +215,50 @@ public void TestFileInformation() Line(1)) ); + [Fact] + public void TestFileInformation_PartialPath() + => Verify( + @"M.M() in C:\folder\m.cs:line", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("M.M"), + argumentList: ArgumentList()), + + fileInformation: FileInformation( + Path(@"C:\folder\m.cs").With(trailingTrivia: CreateTriviaArray(":"))), + + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray("line") + ) + ); + + [Fact] + public void TestFileInformation_PartialPath2() + => Verify( + @"M.M() in C:\folder\m.cs:", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("M.M"), + argumentList: ArgumentList()), + + fileInformation: FileInformation( + Path(@"C:\folder\m.cs").With(trailingTrivia: CreateTriviaArray(":")) + ) + ); + + [Fact] + public void TestFileInformation_TrailingTrivia() + => Verify( + @"M.M() in C:\folder\m.cs:line 1[trailingtrivia]", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("M.M"), + argumentList: ArgumentList()), + + fileInformation: FileInformation( + Path(@"C:\folder\m.cs"), + ColonToken, + Line(1)), + + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray("[trailingtrivia]")) + ); + [Theory] [InlineData(@"at M()")] // Method with no class is invalid [InlineData(@"at M.1c()")] // Invalid start character for identifier @@ -210,7 +271,8 @@ public void TestFileInformation() [InlineData(@"at M.M Verify(input, expectFailure: true); } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 97ae963086713..569f7d29b858b 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -10,7 +10,6 @@ internal enum StackFrameKind { None = 0, EndOfLine, - Text, MethodDeclaration, MemberAccess, Identifier, diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index efe4ac45c26cf..c25e0b968414b 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -79,28 +79,6 @@ public VirtualCharSequence GetSubPattern(int start, int end) return identifier; } - public StackFrameToken ScanTypeArity() - { - if (Position == Text.Length) - { - return default; - } - - var startPosition = Position; - var ch = CurrentChar; - Position++; - - while (IsNumber(ch)) - { - ch = CurrentChar; - Position++; - } - - var aritySpan = new TextSpan(startPosition, (Position - 1) - startPosition); - var arityToken = CreateToken(StackFrameKind.TextToken, Text.GetSubSequence(aritySpan)); - return arityToken; - } - internal StackFrameTrivia? ScanWhiteSpace() { if (Position == Text.Length) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index fe7f4d789b6a1..d1b7e0590e993 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -211,7 +211,7 @@ public StackFrameArrayExpressionNode(StackFrameExpressionNode identifier, Immuta _identifier = identifier; _arrayBrackets = arrayBrackets; - Debug.Assert(arrayBrackets.All(t => t.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.CloseBracketToken)); + Debug.Assert(arrayBrackets.All(t => t.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.CloseBracketToken or StackFrameKind.CommaToken)); } internal override int ChildCount => _arrayBrackets.Length + 1; diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index d01d7946ea9ec..3dc4d282b0a25 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -17,10 +17,19 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { + using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; using StackFrameToken = EmbeddedSyntaxToken; using StackFrameTrivia = EmbeddedSyntaxTrivia; + internal class StackFrameParseException : Exception + { + public StackFrameParseException(StackFrameKind expectedKind, StackFrameToken actual) + : base($"Expected {expectedKind} instead of '{actual.VirtualChars.CreateString()}' at {actual.GetSpan().Start}") + { + } + } + internal struct StackFrameParser { private StackFrameLexer _lexer; @@ -47,6 +56,11 @@ private StackFrameParser(VirtualCharSequence text) { return new StackFrameParser(text).TryParseTree(); } + catch (StackFrameParseException) + { + // Should we report why parsing failed here? + return null; + } catch (InsufficientExecutionStackException) { return null; @@ -180,9 +194,16 @@ private StackFrameParser(VirtualCharSequence text) while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { StackFrameToken? arity = null; - if (CurrentToken.Kind == StackFrameKind.GraveAccentToken) + if (_lexer.ScanIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { - arity = _lexer.ScanTypeArity(); + var numbers = _lexer.ScanNumbers(); + if (!numbers.HasValue) + { + throw new StackFrameParseException(StackFrameKind.NumberToken, CurrentToken); + } + + var arityChars = VirtualCharSequence.FromBounds(graveAccentToken.VirtualChars, numbers.Value.VirtualChars); + arity = StackFrameLexer.CreateToken(StackFrameKind.ArrayExpression, arityChars); } if (leadingTrivia.HasValue) @@ -275,12 +296,25 @@ private StackFrameParser(VirtualCharSequence text) { builder.Add(new StackFrameTypeArgument(currentIdentifier.Value)); - if (useCloseBracket && _lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, out closeToken)) + if (_lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, out var closeBracket)) { - break; + if (useCloseBracket) + { + closeToken = closeBracket; + break; + } + + throw new StackFrameParseException(StackFrameKind.GreaterThanToken, closeBracket); } - else if (_lexer.ScanIfMatch(StackFrameKind.GreaterThanToken, out closeToken)) + + if (_lexer.ScanIfMatch(StackFrameKind.GreaterThanToken, out var greaterThanToken)) { + if (useCloseBracket) + { + throw new StackFrameParseException(StackFrameKind.CloseBracketToken, greaterThanToken); + } + + closeToken = greaterThanToken; break; } @@ -446,7 +480,7 @@ private bool TryParseArrayIdentifier(StackFrameExpressionNode identifier, out Im { // malformed, we have a ": " with no "line " trivia // add the colonToken as trivia to the valid path and return it - var colonTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.Text, colonToken.VirtualChars); + var colonTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, colonToken.VirtualChars); return (new(path.Value.With(trailingTrivia: ImmutableArray.Create(colonTrivia))) , true); @@ -458,7 +492,7 @@ private bool TryParseArrayIdentifier(StackFrameExpressionNode identifier, out Im // malformed, we have a ":line " but no following number. // Add the colon and line trivia as trailing trivia var jointTriviaSpan = new TextSpan(colonToken.GetSpan().Start, colonToken.VirtualChars.Length + lineIdentifier.Value.VirtualChars.Length); - var trailingTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.Text, _lexer.Text.GetSubSequence(jointTriviaSpan)); + var trailingTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, _lexer.Text.GetSubSequence(jointTriviaSpan)); return (new(path.Value.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))) , true); From b171e55a6e77a5f9634010b78067f809a85b365b Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Fri, 8 Oct 2021 15:03:43 -0700 Subject: [PATCH 13/59] Fix invalid path parsing and add tests --- .../StackFrame/StackFrameParserTests.cs | 49 +++++++++++++++++++ .../StackFrame/StackFrameCompilationUnit.cs | 19 +++++++ .../StackFrame/StackFrameLexer.cs | 5 ++ .../StackFrame/StackFrameParser.cs | 26 ++++++++-- 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 31739974c4d1c..e2dced4e983a9 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -28,6 +28,28 @@ public void TestTrailingTrivia() eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" some other text")) ); + [Fact] + public void TestTrailingTrivia_InTriviaNoSpace() + => Verify( + @"at ConsoleApp4.MyClass.M() inC:\My\Path\C.cs:line 26", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(AtTrivia)), + argumentList: ArgumentList()), + + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(@" inC:\My\Path\C.cs:line 26")) + ); + + [Fact] + public void TestTrailingTrivia_InTriviaNoSpace2() + => Verify( + @"at ConsoleApp4.MyClass.M()in C:\My\Path\C.cs:line 26", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(AtTrivia)), + argumentList: ArgumentList()), + + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(@"in C:\My\Path\C.cs:line 26")) + ); + [Fact] public void TestNoParams_NoAtTrivia() => Verify( @@ -259,6 +281,31 @@ public void TestFileInformation_TrailingTrivia() eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray("[trailingtrivia]")) ); + [Fact] + public void TestFileInformation_TrailingTrivia2() + => Verify( + @"M.M() in C:\folder\m.cs:[trailingtrivia]", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("M.M"), + argumentList: ArgumentList()), + + fileInformation: FileInformation( + Path(@"C:\folder\m.cs").With(trailingTrivia: CreateTriviaArray(":"))), + + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray("[trailingtrivia]")) + ); + + [Fact] + public void TestFileInformation_InvalidDirectory() + => Verify( + @"M.M() in C:\<\m.cs", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("M.M"), + argumentList: ArgumentList()), + + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\", @"<\m.cs")) + ); + [Theory] [InlineData(@"at M()")] // Method with no class is invalid [InlineData(@"at M.1c()")] // Invalid start character for identifier @@ -273,6 +320,8 @@ public void TestFileInformation_TrailingTrivia() [InlineData(@"at M.M(string] s)")] // Close only array bracket [InlineData(@"at M.M(string[][][ s)")] [InlineData(@"at M.M(string[[]] s)")] + [InlineData(@"at M.N`.P()")] // Missing numeric for arity + [InlineData(@"at M.N`9N.P()")] // Invalid character after arity public void TestInvalidInputs(string input) => Verify(input, expectFailure: true); } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs index cc8b4b4113e59..a6c3bccfe452a 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs @@ -10,10 +10,29 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame using StackFrameToken = EmbeddedSyntaxToken; using StackFrameTrivia = EmbeddedSyntaxTrivia; + /// + /// The root unit for a stackframe. Includes the method declaration for the stack frame and optional file information. + /// Any leading "at " is considered trivia of , and " in " is put as trivia for the . + /// Remaining unparsable text is put as leading trivia on the + /// internal class StackFrameCompilationUnit { + /// + /// Represents the method declaration for a stack frame. Requires at least a member + /// access and argument list with no parameters to be considered valid + /// public readonly StackFrameMethodDeclarationNode MethodDeclaration; + + /// + /// File information for a stack frame. May be optionally contained. If available, represents + /// the file path of a stackframe and optionally the line number. This is available as hint information + /// and may be useful for a user, but is not always accurate when mapping back to source. + /// public readonly StackFrameFileInformationNode? FileInformationExpression; + + /// + /// The end token of a frame. Any trailing text is added as leading trivia of this token. + /// public readonly StackFrameToken EndOfLineToken; public StackFrameCompilationUnit(StackFrameMethodDeclarationNode methodDeclaration, StackFrameFileInformationNode? fileInformationExpression, StackFrameToken endOfLineToken) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index c25e0b968414b..1b93f312f9a7b 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -329,6 +329,11 @@ static bool IsInvalidCharForRootedPath(char c) Position++; } + if (start == Position) + { + return null; + } + return CreateToken(StackFrameKind.NumberToken, GetSubPatternToCurrentPos(start)); } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 3dc4d282b0a25..8b614196658ea 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -30,6 +30,11 @@ public StackFrameParseException(StackFrameKind expectedKind, StackFrameToken act } } + /// + /// Attempts to parse a stack frame line from given input. StackFrame is generally + /// defined as a string line in a StackTrace. See https://docs.microsoft.com/en-us/dotnet/api/system.environment.stacktrace for + /// more documentation on dotnet stack traces. + /// internal struct StackFrameParser { private StackFrameLexer _lexer; @@ -115,7 +120,7 @@ private StackFrameParser(VirtualCharSequence text) { // If we parsed a path but it's not valid for the file system, // combine both with the remaining trivia as text - trailingTriviaBuilder.Add(inTrivia.Value); + trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, inTrivia.Value.VirtualChars)); if (fileInformation is not null) { @@ -140,7 +145,7 @@ private StackFrameParser(VirtualCharSequence text) Debug.Assert(_lexer.Position == _lexer.Text.Length); Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); - var root = new StackFrameCompilationUnit(methodDeclaration, fileInformation, eolToken); + var root = new StackFrameCompilationUnit(methodDeclaration, isFilePathValid ? fileInformation : null, eolToken); return new StackFrameTree( _lexer.Text, root, ImmutableArray.Empty); @@ -449,13 +454,28 @@ private bool TryParseArrayIdentifier(StackFrameExpressionNode identifier, out Im { var pathStr = path.Value.VirtualChars.CreateString(); var file = new FileInfo(pathStr); + if (file.FullName.Length != pathStr.Length) + { + // Somewhere in the file info construction the path + // was invalid and only partially parsed + return false; + } + var invalidFileChars = Path.GetInvalidFileNameChars(); - if (file.Name.Any(c => invalidFileChars.Contains(c))) + + // File name is a requirement for a successful lookup + if (file.Name.IsEmpty() || file.Name.Any(c => invalidFileChars.Contains(c))) { return false; } + // A file path may not have a directory, which is okay for our purposes var directory = file.Directory; + if (directory is null) + { + return true; + } + var invalidDirectoryChars = Path.GetInvalidPathChars(); if (directory.FullName.Any(c => invalidDirectoryChars.Contains(c))) { From 87964e6c129c9b9890a150d5f7223e751168d8e3 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Fri, 8 Oct 2021 20:54:12 -0700 Subject: [PATCH 14/59] Add more file path tests. Require that a path is fully formed with line number, otherwise it's just trivia --- .../StackFrameParserTests.Generators.cs | 2 +- .../StackFrame/StackFrameParserTests.cs | 36 ++++-- .../StackFrame/StackFrameCompilationUnit.cs | 2 +- .../StackFrame/StackFrameLexer.cs | 39 +++---- .../StackFrame/StackFrameNodeDefinitions.cs | 10 +- .../StackFrame/StackFrameParser.cs | 103 +++++------------- 6 files changed, 75 insertions(+), 117 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index 93769f085c7eb..5463639ac41b7 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -140,7 +140,7 @@ private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, par private static StackFrameTypeArgument TypeArgument(string identifier) => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); - private static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken? colon = null, StackFrameToken? line = null) + private static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken colon, StackFrameToken line) => new(path.With(leadingTrivia: CreateTriviaArray(InTrivia)), colon, line); private static StackFrameToken Path(string path) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index e2dced4e983a9..bacc93c596538 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -245,10 +245,7 @@ public void TestFileInformation_PartialPath() MemberAccessExpression("M.M"), argumentList: ArgumentList()), - fileInformation: FileInformation( - Path(@"C:\folder\m.cs").With(trailingTrivia: CreateTriviaArray(":"))), - - eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray("line") + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":", "line") ) ); @@ -260,11 +257,29 @@ public void TestFileInformation_PartialPath2() MemberAccessExpression("M.M"), argumentList: ArgumentList()), - fileInformation: FileInformation( - Path(@"C:\folder\m.cs").With(trailingTrivia: CreateTriviaArray(":")) + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":") ) ); + [Theory] + [InlineData(@"C:\folder\m.cs", 1)] + [InlineData(@"m.cs", 1)] + [InlineData(@"C:\folder\m.cs", 123456789)] + [InlineData(@"..\m.cs", 1)] + [InlineData(@".\m.cs", 1)] + public void TestFilePaths(string path, int line) + => Verify( + $"M.M() in {path}:line {line}", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("M.M"), + argumentList: ArgumentList()), + + fileInformation: FileInformation( + Path(path), + ColonToken, + Line(line)) + ); + [Fact] public void TestFileInformation_TrailingTrivia() => Verify( @@ -289,10 +304,7 @@ public void TestFileInformation_TrailingTrivia2() MemberAccessExpression("M.M"), argumentList: ArgumentList()), - fileInformation: FileInformation( - Path(@"C:\folder\m.cs").With(trailingTrivia: CreateTriviaArray(":"))), - - eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray("[trailingtrivia]")) + eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":", "[trailingtrivia]")) ); [Fact] @@ -307,6 +319,10 @@ public void TestFileInformation_InvalidDirectory() ); [Theory] + [InlineData("")] + [InlineData("lkasjdlkfjalskdfj")] + [InlineData("\n")] + [InlineData("at ")] [InlineData(@"at M()")] // Method with no class is invalid [InlineData(@"at M.1c()")] // Invalid start character for identifier [InlineData(@"at 1M.C()")] diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs index a6c3bccfe452a..ca8424363d4e1 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs @@ -12,7 +12,7 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame /// /// The root unit for a stackframe. Includes the method declaration for the stack frame and optional file information. - /// Any leading "at " is considered trivia of , and " in " is put as trivia for the . + /// Any leading "at " is considered trivia of , and " in " is put as trivia for the . /// Remaining unparsable text is put as leading trivia on the /// internal class StackFrameCompilationUnit diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 1b93f312f9a7b..1db689c2f2253 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -12,6 +12,7 @@ using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.PooledObjects; +using Microsoft.CodeAnalysis.Shared.Utilities; using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; @@ -245,6 +246,7 @@ private static bool IsNumber(VirtualChar ch) /// /// Attempts to parse a path following https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names + /// Uses as a tool to determine if the path is correct for returning. /// internal StackFrameToken? ScanPath() { @@ -254,24 +256,25 @@ private static bool IsNumber(VirtualChar ch) } var startPosition = Position; - var invalidChars = Path.GetInvalidPathChars(); - var isRooted = false; - while (!invalidChars.Contains((char)CurrentChar.Value)) + while (Position < Text.Length) { - // If the path is rooted then we no longer accept - if (isRooted && IsInvalidCharForRootedPath((char)CurrentChar.Value)) + // Path needs to do a look ahead to determine if adding the next character + // invalidates the path. Break if it does + var str = GetSubPattern(startPosition, Position + 1).CreateString(); + + var isValidPath = IOUtilities.PerformIO(() => + { + var fileInfo = new FileInfo(str); + return true; + }, false); + + if (!isValidPath) { break; } Position++; - - if (!isRooted) - { - var str = GetSubPatternToCurrentPos(startPosition).CreateString(); - isRooted = Path.IsPathRooted(str); - } } if (startPosition == Position) @@ -280,20 +283,6 @@ private static bool IsNumber(VirtualChar ch) } return CreateToken(StackFrameKind.PathToken, GetSubPatternToCurrentPos(startPosition)); - - static bool IsInvalidCharForRootedPath(char c) - => c switch - { - '<' or - '>' or - '"' or - '|' or - '?' or - '*' or - ':' => true, - - _ => false - }; } internal StackFrameTrivia? ScanLineTrivia() diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index d1b7e0590e993..1e2527aff33e1 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -363,10 +363,10 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameFileInformationNode : StackFrameNode { public readonly StackFrameToken Path; - public readonly StackFrameToken? Colon; - public readonly StackFrameToken? Line; + public readonly StackFrameToken Colon; + public readonly StackFrameToken Line; - public StackFrameFileInformationNode(StackFrameToken path, StackFrameToken? colon = null, StackFrameToken? line = null) : base(StackFrameKind.FileInformation) + public StackFrameFileInformationNode(StackFrameToken path, StackFrameToken colon, StackFrameToken line) : base(StackFrameKind.FileInformation) { Path = path; Colon = colon; @@ -382,8 +382,8 @@ internal override StackFrameNodeOrToken ChildAt(int index) => index switch { 0 => Path, - 1 => Colon.HasValue ? Colon.Value : null, - 2 => Line.HasValue ? Line.Value : null, + 1 => Colon, + 2 => Line, _ => throw new InvalidOperationException() }; diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 8b614196658ea..3a0cf5fe2cec0 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -102,35 +102,35 @@ private StackFrameParser(VirtualCharSequence text) } var inTrivia = _lexer.ScanInTrivia(); - var (fileInformation, isFilePathValid) = inTrivia.HasValue + var fileInformationNodeOrToken = inTrivia.HasValue ? TryParseFileInformation() - : (null, false); + : null; using var _ = ArrayBuilder.GetInstance(out var trailingTriviaBuilder); - if (inTrivia.HasValue) + var fileInformation = fileInformationNodeOrToken.Node as StackFrameFileInformationNode; + + if (fileInformation is not null) + { + Debug.Assert(inTrivia.HasValue); + fileInformation = fileInformation.WithLeadingTrivia(inTrivia.Value); + } + else if (inTrivia.HasValue) { - if (isFilePathValid) + // If the file path wasn't valid make sure to add the consumed tokens to the trailing trivia + trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, inTrivia.Value.VirtualChars)); + + var fileToken = fileInformationNodeOrToken.Token; + if (!fileToken.LeadingTrivia.IsDefaultOrEmpty) { - // If the path is valid, just add the inTrivia to the file information - RoslynDebug.AssertNotNull(fileInformation); - fileInformation = fileInformation.WithLeadingTrivia(inTrivia.Value); + trailingTriviaBuilder.AddRange(fileToken.LeadingTrivia); } - else - { - // If we parsed a path but it's not valid for the file system, - // combine both with the remaining trivia as text - trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, inTrivia.Value.VirtualChars)); - if (fileInformation is not null) - { - // If the path isn't valid we don't expect the line or colon trivia - // to exist on the expression - Debug.Assert(!fileInformation.Line.HasValue); - Debug.Assert(!fileInformation.Colon.HasValue); + trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, fileToken.VirtualChars)); - trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, fileInformation.Path.VirtualChars)); - } + if (!fileToken.TrailingTrivia.IsDefaultOrEmpty) + { + trailingTriviaBuilder.AddRange(fileToken.TrailingTrivia); } } @@ -145,7 +145,7 @@ private StackFrameParser(VirtualCharSequence text) Debug.Assert(_lexer.Position == _lexer.Text.Length); Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); - var root = new StackFrameCompilationUnit(methodDeclaration, isFilePathValid ? fileInformation : null, eolToken); + var root = new StackFrameCompilationUnit(methodDeclaration, fileInformation, eolToken); return new StackFrameTree( _lexer.Text, root, ImmutableArray.Empty); @@ -441,58 +441,17 @@ private bool TryParseArrayIdentifier(StackFrameExpressionNode identifier, out Im /// Parses text for a valid file path using valid file characters. It's very possible this includes a path that doesn't exist but /// forms a valid path identifier. /// - private (StackFrameFileInformationNode? fileInformation, bool isPathValid) TryParseFileInformation() + private StackFrameNodeOrToken TryParseFileInformation() { var path = _lexer.ScanPath(); if (!path.HasValue) { - return (null, false); - } - - // Make sure all the parts are valid as a file path - var isValidFile = IOUtilities.PerformIO(() => - { - var pathStr = path.Value.VirtualChars.CreateString(); - var file = new FileInfo(pathStr); - if (file.FullName.Length != pathStr.Length) - { - // Somewhere in the file info construction the path - // was invalid and only partially parsed - return false; - } - - var invalidFileChars = Path.GetInvalidFileNameChars(); - - // File name is a requirement for a successful lookup - if (file.Name.IsEmpty() || file.Name.Any(c => invalidFileChars.Contains(c))) - { - return false; - } - - // A file path may not have a directory, which is okay for our purposes - var directory = file.Directory; - if (directory is null) - { - return true; - } - - var invalidDirectoryChars = Path.GetInvalidPathChars(); - if (directory.FullName.Any(c => invalidDirectoryChars.Contains(c))) - { - return false; - } - - return true; - }); - - if (!isValidFile) - { - return (new(path.Value), false); + return null; } if (!_lexer.ScanIfMatch(StackFrameKind.ColonToken, out var colonToken)) { - return (new(path.Value), true); + return path.Value; } var lineIdentifier = _lexer.ScanLineTrivia(); @@ -501,9 +460,7 @@ private bool TryParseArrayIdentifier(StackFrameExpressionNode identifier, out Im // malformed, we have a ": " with no "line " trivia // add the colonToken as trivia to the valid path and return it var colonTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, colonToken.VirtualChars); - return - (new(path.Value.With(trailingTrivia: ImmutableArray.Create(colonTrivia))) - , true); + return path.Value.With(trailingTrivia: ImmutableArray.Create(colonTrivia)); } var numbers = _lexer.ScanNumbers(); @@ -513,17 +470,13 @@ private bool TryParseArrayIdentifier(StackFrameExpressionNode identifier, out Im // Add the colon and line trivia as trailing trivia var jointTriviaSpan = new TextSpan(colonToken.GetSpan().Start, colonToken.VirtualChars.Length + lineIdentifier.Value.VirtualChars.Length); var trailingTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, _lexer.Text.GetSubSequence(jointTriviaSpan)); - return - (new(path.Value.With(trailingTrivia: ImmutableArray.Create(trailingTrivia))) - , true); + return path.Value.With(trailingTrivia: ImmutableArray.Create(trailingTrivia)); } - return - (new( + return new StackFrameFileInformationNode( path.Value, colonToken, - numbers.Value.With(leadingTrivia: ImmutableArray.Create(lineIdentifier.Value))) - , true); + numbers.Value.With(leadingTrivia: ImmutableArray.Create(lineIdentifier.Value))); } private StackFrameToken AddTrailingWhitespace(StackFrameToken token) From 15479df6c0cdd47baa221efd2d7642efa1fd250f Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Fri, 8 Oct 2021 20:59:00 -0700 Subject: [PATCH 15/59] Get rid of TextToken. It doesn't make sense and only causes potential problems --- .../StackFrame/StackFrameParserTests.Generators.cs | 2 +- .../EmbeddedLanguages/StackFrame/StackFrameKind.cs | 1 - .../EmbeddedLanguages/StackFrame/StackFrameLexer.cs | 11 +++++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index 5463639ac41b7..573c7873e11ec 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -108,7 +108,7 @@ private static StackFrameArrayExpressionNode ArrayExpression(StackFrameExpressio => new(identifier, arrayTokens.ToImmutableArray()); private static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) - => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.TextToken, arity.ToString())); + => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.NumberToken, arity.ToString())); private static StackFrameTypeArgumentList TypeArgumentList(params StackFrameTypeArgument[] typeArguments) => TypeArgumentList(useBrackets: true, typeArguments); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 569f7d29b858b..3fe722829e975 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -27,7 +27,6 @@ internal enum StackFrameKind OpenParenToken, CloseParenToken, DotToken, - TextToken, PlusToken, CommaToken, ColonToken, diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 1db689c2f2253..68bf06ad1882b 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -3,15 +3,10 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Generic; using System.Collections.Immutable; -using System.Diagnostics; using System.IO; -using System.Linq; -using System.Text; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; -using Microsoft.CodeAnalysis.PooledObjects; using Microsoft.CodeAnalysis.Shared.Utilities; using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; @@ -192,7 +187,11 @@ private static StackFrameKind GetKind(VirtualChar ch) '`' => StackFrameKind.GraveAccentToken, '\\' => StackFrameKind.BackslashToken, '/' => StackFrameKind.ForwardSlashToken, - _ => IsBlank(ch) ? StackFrameKind.WhitespaceTrivia : StackFrameKind.TextToken, + _ => IsBlank(ch) + ? StackFrameKind.WhitespaceTrivia + : IsNumber(ch) + ? StackFrameKind.NumberToken + : StackFrameKind.TextTrivia }; private static bool IsNumber(VirtualChar ch) From b0e6e6ef9a2bfef8534d2c57594bddad2b20ffe9 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Mon, 11 Oct 2021 16:00:48 -0700 Subject: [PATCH 16/59] Update to remove scan arity from lexer and instead scan numbers with parser --- .../StackFrameParserTests.Generators.cs | 16 ++++++++++++++-- .../StackFrame/StackFrameParserTests.cs | 18 ++++++++++++++++++ .../StackFrame/StackFrameParser.cs | 9 +++------ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index 573c7873e11ec..4f98a4d5929a3 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -98,8 +98,20 @@ private static StackFrameMemberAccessExpressionNode MemberAccessExpression(strin private static StackFrameTrivia SpaceTrivia(int count = 1) => CreateTrivia(StackFrameKind.WhitespaceTrivia, new string(' ', count)); - private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameExpressionNode expressionNode, StackFrameBaseIdentifierNode identifierNode) - => new(expressionNode, DotToken, identifierNode); + private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameExpressionNode expressionNode, StackFrameBaseIdentifierNode identifierNode, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + { + if (!leadingTrivia.IsDefaultOrEmpty) + { + expressionNode = expressionNode.WithLeadingTrivia(leadingTrivia); + } + + if (!trailingTrivia.IsDefaultOrEmpty) + { + identifierNode = (StackFrameBaseIdentifierNode)identifierNode.WithTrailingTrivia(trailingTrivia); + } + + return new(expressionNode, DotToken, identifierNode); + } private static StackFrameIdentifierNode Identifier(string identifierName, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) => new(CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia)); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index bacc93c596538..b5481c816dab2 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -17,6 +17,24 @@ public void TestNoParams() argumentList: ArgumentList()) ); + [Theory] + [InlineData("C", 1)] + [InlineData("C", 100)] + [InlineData("a‿", 5)] + public void TestArity(string typeName, int arity) + => Verify( + $"at ConsoleApp4.{typeName}`{arity}.M()", + methodDeclaration: MethodDeclaration( + MemberAccessExpression( + MemberAccessExpression( + Identifier("ConsoleApp4"), + GenericType(typeName, arity)), + Identifier("M"), + leadingTrivia: CreateTriviaArray(AtTrivia)), + + argumentList: ArgumentList()) + ); + [Fact] public void TestTrailingTrivia() => Verify( diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 3a0cf5fe2cec0..14d2fba521c4a 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -201,14 +201,11 @@ private StackFrameParser(VirtualCharSequence text) StackFrameToken? arity = null; if (_lexer.ScanIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { - var numbers = _lexer.ScanNumbers(); - if (!numbers.HasValue) + arity = _lexer.ScanNumbers(); + if (!arity.HasValue) { throw new StackFrameParseException(StackFrameKind.NumberToken, CurrentToken); } - - var arityChars = VirtualCharSequence.FromBounds(graveAccentToken.VirtualChars, numbers.Value.VirtualChars); - arity = StackFrameLexer.CreateToken(StackFrameKind.ArrayExpression, arityChars); } if (leadingTrivia.HasValue) @@ -218,7 +215,7 @@ private StackFrameParser(VirtualCharSequence text) } StackFrameBaseIdentifierNode identifierNode = arity.HasValue - ? new StackFrameGenericTypeIdentifier(currentIdentifier.Value, CurrentToken, arity.Value) + ? new StackFrameGenericTypeIdentifier(currentIdentifier.Value, graveAccentToken, arity.Value) : new StackFrameIdentifierNode(currentIdentifier.Value); typeIdentifierNodes.Enqueue((identifierNode, CurrentToken)); From 0866c7b23b5b5e46624e80dcb36816b3cdd30aae Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Mon, 25 Oct 2021 16:48:46 -0700 Subject: [PATCH 17/59] Fix so parameters are better handled. Make identifier a token and not a node. Fix a lot of trivia parsing problems. --- .../StackFrameParserTests.Generators.cs | 100 ++++-- .../StackFrame/StackFrameParserTests.cs | 138 ++++---- .../StackFrame/IStackFrameNodeVisitor.cs | 23 +- .../StackFrame/StackFrameKind.cs | 5 +- .../StackFrame/StackFrameLexer.cs | 50 ++- .../StackFrame/StackFrameNodeDefinitions.cs | 310 ++++++++++-------- .../StackFrame/StackFrameParser.cs | 222 ++++++------- 7 files changed, 481 insertions(+), 367 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index 4f98a4d5929a3..96d4b0c17bbb4 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -2,16 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System.Collections.Generic; +using System; using System.Collections.Immutable; using System.Linq; -using System.Text; -using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame; using Roslyn.Test.Utilities; using Xunit; -using System; namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame { @@ -36,6 +33,9 @@ private static StackFrameTrivia CreateTrivia(StackFrameKind kind, string s) private static ImmutableArray CreateTriviaArray(params StackFrameTrivia[] trivia) => ImmutableArray.Create(trivia); + private static ImmutableArray CreateTriviaArray(StackFrameTrivia? trivia) + => trivia.HasValue ? ImmutableArray.Create(trivia.Value) : ImmutableArray.Empty; + private static ImmutableArray CreateTriviaArray(params string[] strings) => strings.Select(s => CreateTrivia(StackFrameKind.TextTrivia, s)).ToImmutableArray(); @@ -55,77 +55,103 @@ private static ImmutableArray CreateTriviaArray(params string[ private static readonly StackFrameTrivia LineTrivia = CreateTrivia(StackFrameKind.LineTrivia, "line "); private static readonly StackFrameTrivia InTrivia = CreateTrivia(StackFrameKind.InTrivia, " in "); - private static StackFrameParameterList ArgumentList(params StackFrameNodeOrToken[] nodesOrTokens) - => new(OpenParenToken, nodesOrTokens.ToImmutableArray(), CloseParenToken); + private static readonly StackFrameParameterList EmptyParams = ParameterList(OpenParenToken, CloseParenToken); - private static StackFrameParameterList ArgumentList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameNodeOrToken[] nodesOrTokens) - => new(openToken, nodesOrTokens.ToImmutableArray(), closeToken); + private static StackFrameParameterNode Parameter(StackFrameNodeOrToken type, StackFrameToken identifier) + => new(type, identifier); + + private static StackFrameParameterList ParameterList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameParameterNode[] parameters) + { + var separatedList = parameters.Length == 0 + ? SeparatedStackFrameNodeList.Empty + : new(CommaSeparateList(parameters)); + + return new(openToken, closeToken, separatedList); + + static ImmutableArray CommaSeparateList(StackFrameParameterNode[] parameters) + { + var builder = ImmutableArray.CreateBuilder(); + builder.Add(parameters[0]); + + for (var i = 1; i < parameters.Length; i++) + { + builder.Add(CommaToken); + builder.Add(parameters[i]); + } + + return builder.ToImmutable(); + } + } private static StackFrameMethodDeclarationNode MethodDeclaration( StackFrameMemberAccessExpressionNode memberAccessExpression, StackFrameTypeArgumentList? typeArguments = null, StackFrameParameterList? argumentList = null) { - return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, argumentList ?? ArgumentList(OpenParenToken, CloseParenToken)); + return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, argumentList ?? ParameterList(OpenParenToken, CloseParenToken)); } - private static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + private static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + => MemberAccessExpression(s, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); + + private static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) { - StackFrameExpressionNode? current = null; + StackFrameNodeOrToken? current = null; var identifiers = s.Split('.'); for (var i = 0; i < identifiers.Length; i++) { var identifier = identifiers[i]; - if (current is null) + if (!current.HasValue) { - current = Identifier(identifier, leadingTrivia: leadingTrivia); + current = Identifier(identifier, leadingTrivia: leadingTrivia, trailingTrivia: ImmutableArray.Empty); } else if (i == identifiers.Length - 1) { - current = MemberAccessExpression(current, Identifier(identifier, trailingTrivia: trailingTrivia)); + current = MemberAccessExpression(current.Value, Identifier(identifier, leadingTrivia: ImmutableArray.Empty, trailingTrivia: trailingTrivia)); } else { - current = MemberAccessExpression(current, Identifier(identifier)); + current = MemberAccessExpression(current.Value, Identifier(identifier)); } } - AssertEx.NotNull(current); - return (StackFrameMemberAccessExpressionNode)current; + Assert.True(current.HasValue); + Assert.True(current!.Value.IsNode); + + var node = current.Value.Node; + AssertEx.NotNull(node); + return (StackFrameMemberAccessExpressionNode)node; } private static StackFrameTrivia SpaceTrivia(int count = 1) => CreateTrivia(StackFrameKind.WhitespaceTrivia, new string(' ', count)); - private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameExpressionNode expressionNode, StackFrameBaseIdentifierNode identifierNode, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) - { - if (!leadingTrivia.IsDefaultOrEmpty) - { - expressionNode = expressionNode.WithLeadingTrivia(leadingTrivia); - } + private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameNodeOrToken left, StackFrameNodeOrToken right) + => new(left, DotToken, right); - if (!trailingTrivia.IsDefaultOrEmpty) - { - identifierNode = (StackFrameBaseIdentifierNode)identifierNode.WithTrailingTrivia(trailingTrivia); - } + private static StackFrameToken Identifier(string identifierName) + => Identifier(identifierName, leadingTrivia: null, trailingTrivia: null); - return new(expressionNode, DotToken, identifierNode); - } + private static StackFrameToken Identifier(string identifierName, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + => Identifier(identifierName, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); + + private static StackFrameToken Identifier(string identifierName, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) + => CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia); - private static StackFrameIdentifierNode Identifier(string identifierName, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) - => new(CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia)); + private static StackFrameArrayRankSpecifier ArrayRankSpecifier(int commaCount = 0, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + => new(OpenBracketToken.With(leadingTrivia: CreateTriviaArray(leadingTrivia)), CloseBracketToken.With(trailingTrivia: CreateTriviaArray(trailingTrivia)), Enumerable.Repeat(CommaToken, commaCount).ToImmutableArray()); - private static StackFrameArrayExpressionNode ArrayExpression(StackFrameExpressionNode identifier, params StackFrameToken[] arrayTokens) + private static StackFrameArrayTypeExpression ArrayExpression(StackFrameNodeOrToken identifier, params StackFrameArrayRankSpecifier[] arrayTokens) => new(identifier, arrayTokens.ToImmutableArray()); private static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.NumberToken, arity.ToString())); - private static StackFrameTypeArgumentList TypeArgumentList(params StackFrameTypeArgument[] typeArguments) + private static StackFrameTypeArgumentList TypeArgumentList(params StackFrameTypeArgumentNode[] typeArguments) => TypeArgumentList(useBrackets: true, typeArguments); - private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgument[] typeArguments) + private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgumentNode[] typeArguments) { using var _ = PooledObjects.ArrayBuilder.GetInstance(out var builder); var openToken = useBrackets ? OpenBracketToken : LessThanToken; @@ -146,10 +172,12 @@ private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, par builder.Add(typeArgument); } - return new(openToken, builder.ToImmutable(), closeToken); + var typeArgumentsList = new SeparatedStackFrameNodeList(builder.ToImmutable()); + + return new(openToken, typeArgumentsList, closeToken); } - private static StackFrameTypeArgument TypeArgument(string identifier) + private static StackFrameTypeArgumentNode TypeArgument(string identifier) => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); private static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken colon, StackFrameToken line) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index b5481c816dab2..6ac8892e692a4 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -13,26 +13,26 @@ public void TestNoParams() => Verify( @"at ConsoleApp4.MyClass.M()", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(AtTrivia)), - argumentList: ArgumentList()) + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), + argumentList: EmptyParams) ); [Theory] [InlineData("C", 1)] [InlineData("C", 100)] - [InlineData("a‿", 5)] + [InlineData("a‿", 5)] // Unicode character with connection + [InlineData("abcdefg", 99999)] public void TestArity(string typeName, int arity) => Verify( $"at ConsoleApp4.{typeName}`{arity}.M()", methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4"), + Identifier("ConsoleApp4", leadingTrivia: AtTrivia), GenericType(typeName, arity)), - Identifier("M"), - leadingTrivia: CreateTriviaArray(AtTrivia)), + Identifier("M")), - argumentList: ArgumentList()) + argumentList: EmptyParams) ); [Fact] @@ -40,8 +40,8 @@ public void TestTrailingTrivia() => Verify( @"at ConsoleApp4.MyClass.M() some other text", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(AtTrivia)), - argumentList: ArgumentList()), + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), + argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" some other text")) ); @@ -51,8 +51,8 @@ public void TestTrailingTrivia_InTriviaNoSpace() => Verify( @"at ConsoleApp4.MyClass.M() inC:\My\Path\C.cs:line 26", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(AtTrivia)), - argumentList: ArgumentList()), + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), + argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(@" inC:\My\Path\C.cs:line 26")) ); @@ -62,8 +62,8 @@ public void TestTrailingTrivia_InTriviaNoSpace2() => Verify( @"at ConsoleApp4.MyClass.M()in C:\My\Path\C.cs:line 26", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(AtTrivia)), - argumentList: ArgumentList()), + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), + argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(@"in C:\My\Path\C.cs:line 26")) ); @@ -74,7 +74,7 @@ public void TestNoParams_NoAtTrivia() @"ConsoleApp4.MyClass.M()", methodDeclaration: MethodDeclaration( MemberAccessExpression("ConsoleApp4.MyClass.M"), - argumentList: ArgumentList()) + argumentList: EmptyParams) ); [Fact] @@ -83,7 +83,7 @@ public void TestNoParams_SpaceInParams_NoAtTrivia() @"ConsoleApp4.MyClass.M( )", methodDeclaration: MethodDeclaration( MemberAccessExpression("ConsoleApp4.MyClass.M"), - argumentList: ArgumentList( + argumentList: ParameterList( OpenParenToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia(2))), CloseParenToken)) ); @@ -93,8 +93,8 @@ public void TestNoParams_SpaceTrivia() => Verify( @" ConsoleApp4.MyClass.M()", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(SpaceTrivia())), - argumentList: ArgumentList()) + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: SpaceTrivia()), + argumentList: EmptyParams) ); [Fact] @@ -102,8 +102,8 @@ public void TestNoParams_SpaceTrivia2() => Verify( @" ConsoleApp4.MyClass.M()", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: CreateTriviaArray(SpaceTrivia(2))), - argumentList: ArgumentList()) + MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: SpaceTrivia(2)), + argumentList: EmptyParams) ); [Fact] @@ -113,13 +113,16 @@ public void TestMethodOneParam() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), + Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), - argumentList: ArgumentList( - Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia())), - Identifier("s")) + argumentList: ParameterList( + OpenParenToken, + CloseParenToken, + Parameter( + Identifier("string"), + Identifier("s", leadingTrivia: SpaceTrivia()))) ) ); @@ -130,16 +133,19 @@ public void TestMethodTwoParam() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), + Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), - argumentList: ArgumentList( - Identifier("string", trailingTrivia: CreateTriviaArray(SpaceTrivia())), - Identifier("s"), - CommaToken, - Identifier("string", leadingTrivia: CreateTriviaArray(SpaceTrivia()), trailingTrivia: CreateTriviaArray(SpaceTrivia())), - Identifier("t")) + argumentList: ParameterList( + OpenParenToken, + CloseParenToken, + Parameter( + Identifier("string"), + Identifier("s", leadingTrivia: SpaceTrivia())), + Parameter( + Identifier("string", leadingTrivia: SpaceTrivia()), + Identifier("t", leadingTrivia: SpaceTrivia()))) ) ); @@ -150,13 +156,15 @@ public void TestMethodArrayParam() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), + Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), - argumentList: ArgumentList( - ArrayExpression(Identifier("string"), OpenBracketToken, CloseBracketToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia()))), - Identifier("s")) + argumentList: ParameterList( + OpenParenToken, + CloseParenToken, + Parameter(ArrayExpression(Identifier("string"), ArrayRankSpecifier(trailingTrivia: SpaceTrivia())), + Identifier("s"))) ) ); @@ -167,13 +175,16 @@ public void TestCommaArrayParam() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), + Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), - argumentList: ArgumentList( - ArrayExpression(Identifier("string"), OpenBracketToken, CommaToken, CloseBracketToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia()))), - Identifier("s")) + argumentList: ParameterList( + OpenParenToken, + CloseParenToken, + Parameter( + ArrayExpression(Identifier("string"), ArrayRankSpecifier(1, trailingTrivia: SpaceTrivia())), + Identifier("s"))) ) ); @@ -184,13 +195,16 @@ public void TestGenericMethod_Brackets() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), + Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), typeArguments: TypeArgumentList(TypeArgument("T")), - argumentList: ArgumentList( - Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia())), - Identifier("t")) + argumentList: ParameterList( + OpenParenToken, + CloseParenToken, + Parameter( + Identifier("T"), + Identifier("t", leadingTrivia: SpaceTrivia()))) ) ); @@ -201,13 +215,16 @@ public void TestGenericMethod() methodDeclaration: MethodDeclaration( MemberAccessExpression( MemberAccessExpression( - Identifier("ConsoleApp4", leadingTrivia: CreateTriviaArray(AtTrivia)), + Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), typeArguments: TypeArgumentList(useBrackets: false, TypeArgument("T")), - argumentList: ArgumentList( - Identifier("T", trailingTrivia: CreateTriviaArray(SpaceTrivia())), - Identifier("t")) + argumentList: ParameterList( + OpenParenToken, + CloseParenToken, + Parameter( + Identifier("T"), + Identifier("t", leadingTrivia: SpaceTrivia()))) ) ); @@ -218,17 +235,20 @@ public void TestGenericMethod() [InlineData("ü")] // Unicode character [InlineData("uʶ")] // character and modifier character [InlineData("a\u00AD")] // Soft hyphen formatting character - [InlineData("a‿")] // Connecting punctuation (combining character + [InlineData("a‿")] // Connecting punctuation (combining character) public void TestIdentifierNames(string identifierName) => Verify( @$"at {identifierName}.{identifierName}[{identifierName}]({identifierName} {identifierName})", methodDeclaration: MethodDeclaration( - MemberAccessExpression($"{identifierName}.{identifierName}", leadingTrivia: CreateTriviaArray(AtTrivia)), + MemberAccessExpression($"{identifierName}.{identifierName}", leadingTrivia: AtTrivia), typeArguments: TypeArgumentList(TypeArgument(identifierName)), - argumentList: ArgumentList( - Identifier(identifierName, trailingTrivia: CreateTriviaArray(SpaceTrivia())), - Identifier(identifierName)) + argumentList: ParameterList( + OpenParenToken, + CloseParenToken, + Parameter( + Identifier(identifierName), + Identifier(identifierName, leadingTrivia: SpaceTrivia()))) ) ); @@ -238,7 +258,7 @@ public void TestAnonymousMethod() @"Microsoft.VisualStudio.DesignTools.SurfaceDesigner.Tools.EventRouter.ScopeElement_MouseUp.AnonymousMethod__0()", methodDeclaration: MethodDeclaration( MemberAccessExpression("Microsoft.VisualStudio.DesignTools.SurfaceDesigner.Tools.EventRouter.ScopeElement_MouseUp.AnonymousMethod__0"), - argumentList: ArgumentList()) + argumentList: EmptyParams) ); [Fact] @@ -247,7 +267,7 @@ public void TestFileInformation() @"M.M() in C:\folder\m.cs:line 1", methodDeclaration: MethodDeclaration( MemberAccessExpression("M.M"), - argumentList: ArgumentList()), + argumentList: EmptyParams), fileInformation: FileInformation( Path(@"C:\folder\m.cs"), @@ -261,7 +281,7 @@ public void TestFileInformation_PartialPath() @"M.M() in C:\folder\m.cs:line", methodDeclaration: MethodDeclaration( MemberAccessExpression("M.M"), - argumentList: ArgumentList()), + argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":", "line") ) @@ -273,7 +293,7 @@ public void TestFileInformation_PartialPath2() @"M.M() in C:\folder\m.cs:", methodDeclaration: MethodDeclaration( MemberAccessExpression("M.M"), - argumentList: ArgumentList()), + argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":") ) @@ -290,7 +310,7 @@ public void TestFilePaths(string path, int line) $"M.M() in {path}:line {line}", methodDeclaration: MethodDeclaration( MemberAccessExpression("M.M"), - argumentList: ArgumentList()), + argumentList: EmptyParams), fileInformation: FileInformation( Path(path), @@ -304,7 +324,7 @@ public void TestFileInformation_TrailingTrivia() @"M.M() in C:\folder\m.cs:line 1[trailingtrivia]", methodDeclaration: MethodDeclaration( MemberAccessExpression("M.M"), - argumentList: ArgumentList()), + argumentList: EmptyParams), fileInformation: FileInformation( Path(@"C:\folder\m.cs"), @@ -320,7 +340,7 @@ public void TestFileInformation_TrailingTrivia2() @"M.M() in C:\folder\m.cs:[trailingtrivia]", methodDeclaration: MethodDeclaration( MemberAccessExpression("M.M"), - argumentList: ArgumentList()), + argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":", "[trailingtrivia]")) ); @@ -331,7 +351,7 @@ public void TestFileInformation_InvalidDirectory() @"M.M() in C:\<\m.cs", methodDeclaration: MethodDeclaration( MemberAccessExpression("M.M"), - argumentList: ArgumentList()), + argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\", @"<\m.cs")) ); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index 75528a5a7df64..5228f384bd91f 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -2,22 +2,19 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; -using System.Collections.Generic; -using System.Text; - namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { internal interface IStackFrameNodeVisitor { - void Visit(StackFrameMethodDeclarationNode stackFrameMethodDeclarationNode); - void Visit(StackFrameMemberAccessExpressionNode stackFrameMemberAccessExpressionNode); - void Visit(StackFrameTypeArgumentList stackFrameTypeArguments); - void Visit(StackFrameParameterList stackFrameArgumentList); - void Visit(StackFrameIdentifierNode stackFrameIdentifierNode); - void Visit(StackFrameGenericTypeIdentifier stackFrameGenericTypeIdentifier); - void Visit(StackFrameTypeArgument stackFrameTypeArgument); - void Visit(StackFrameArrayExpressionNode stackFrameArrayExpressionNode); - void Visit(StackFrameFileInformationNode stackFrameFileInformationNode); + void Visit(StackFrameMethodDeclarationNode node); + void Visit(StackFrameMemberAccessExpressionNode node); + void Visit(StackFrameTypeArgumentList node); + void Visit(StackFrameParameterList node); + void Visit(StackFrameGenericTypeIdentifier node); + void Visit(StackFrameTypeArgumentNode node); + void Visit(StackFrameArrayRankSpecifier node); + void Visit(StackFrameFileInformationNode node); + void Visit(StackFrameArrayTypeExpression node); + void Visit(StackFrameParameterNode stackFrameParameterNode); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 3fe722829e975..06f322c3b981e 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -9,13 +9,16 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame internal enum StackFrameKind { None = 0, + + // Nodes EndOfLine, MethodDeclaration, MemberAccess, - Identifier, + ArrayTypeExpression, GenericTypeIdentifier, TypeArgument, TypeIdentifier, + Parameter, ParameterList, ArrayExpression, FileInformation, diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 68bf06ad1882b..de0d420420996 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -7,6 +7,7 @@ using System.IO; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; +using Microsoft.CodeAnalysis.PooledObjects; using Microsoft.CodeAnalysis.Shared.Utilities; using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; @@ -50,18 +51,20 @@ public VirtualCharSequence GetSubPattern(int start, int end) return CreateTrivia(StackFrameKind.TextTrivia, GetSubPatternToCurrentPos(start)); } - public StackFrameToken? ScanIdentifier() + public StackFrameToken? ScanIdentifier(bool scanAtTrivia = false, bool scanWhitespace = false) { - if (Position == Text.Length) - { - return null; - } + var originalPosition = Position; + var atTrivia = scanAtTrivia ? ScanAtTrivia() : null; + var whitespaceTrivia = scanWhitespace ? ScanWhiteSpace() : null; var startPosition = Position; - var ch = CurrentChar; if (!UnicodeCharacterUtilities.IsIdentifierStartCharacter((char)ch.Value)) { + // If we scan only trivia but don't get an identifier, we want to make sure + // to reset back to this original position to let the trivia be consumed + // in some other fashion if necessary + Position = originalPosition; return null; } @@ -71,8 +74,7 @@ public VirtualCharSequence GetSubPattern(int start, int end) ch = CurrentChar; } - var identifier = CreateToken(StackFrameKind.IdentifierToken, GetSubPatternToCurrentPos(startPosition)); - return identifier; + return CreateToken(StackFrameKind.IdentifierToken, CreateTrivia(atTrivia, whitespaceTrivia), GetSubPatternToCurrentPos(startPosition)); } internal StackFrameTrivia? ScanWhiteSpace() @@ -134,11 +136,27 @@ private bool TextAt(int position, string val) /// if the position was incremented /// internal bool ScanIfMatch(StackFrameKind kind, out StackFrameToken token) + => ScanIfMatch(kind, scanTrailingWhitespace: false, out token); + + /// + /// Progress the position by one if the current character + /// matches the kind. + /// + /// + /// if the position was incremented + /// + internal bool ScanIfMatch(StackFrameKind kind, bool scanTrailingWhitespace, out StackFrameToken token) { if (GetKind(CurrentChar) == kind) { token = CurrentCharAsToken(); Position++; + + if (scanTrailingWhitespace) + { + token = token.With(trailingTrivia: CreateTrivia(ScanWhiteSpace())); + } + return true; } @@ -342,7 +360,7 @@ public static bool IsBlank(VirtualChar ch) } public static StackFrameToken CreateToken(StackFrameKind kind, VirtualCharSequence virtualChars) - => new(kind, ImmutableArray.Empty, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null!); + => CreateToken(kind, ImmutableArray.Empty, virtualChars); public static StackFrameToken CreateToken(StackFrameKind kind, ImmutableArray leadingTrivia, VirtualCharSequence virtualChars) => new(kind, leadingTrivia, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null!); @@ -352,5 +370,19 @@ public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequ public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars, ImmutableArray diagnostics) => new(kind, virtualChars, diagnostics); + + public static ImmutableArray CreateTrivia(params StackFrameTrivia?[] triviaArray) + { + using var _ = ArrayBuilder.GetInstance(out var builder); + foreach (var trivia in triviaArray) + { + if (trivia.HasValue) + { + builder.Add(trivia.Value); + } + } + + return builder.ToImmutable(); + } } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index 1e2527aff33e1..c9ef7010959a3 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Text; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; +using Microsoft.CodeAnalysis.PooledObjects; using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame @@ -24,6 +25,64 @@ protected StackFrameNode(StackFrameKind kind) : base(kind) } public abstract void Accept(IStackFrameNodeVisitor visitor); + + public StackFrameNodeOrToken this[int index] => ChildAt(index); + public StackFrameNodeOrToken this[Index index] => this[index.GetOffset(this.ChildCount)]; + } + + internal sealed class SeparatedStackFrameNodeList where TNode : StackFrameNode + { + public SeparatedStackFrameNodeList(ImmutableArray nodesAndTokens) + { + Contract.ThrowIfTrue(nodesAndTokens.IsDefaultOrEmpty); + NodesAndTokens = nodesAndTokens; + +#if DEBUG + // Length should represent (nodes.Length) + (nodes.Length - 1), where the latter + // represents the number of separator tokens + Debug.Assert(nodesAndTokens.Length % 2 == 1); + for (var i = 0; i < nodesAndTokens.Length; i++) + { + if (i % 2 == 0) + { + // All even values should be TNode + Debug.Assert(nodesAndTokens[i].IsNode); + Debug.Assert(nodesAndTokens[i].Node is TNode); + } + else + { + // All odd values should be separator tokens + Debug.Assert(!nodesAndTokens[i].IsNode); + Debug.Assert(!nodesAndTokens[i].Token.IsMissing); + } + } +#endif + } + + private SeparatedStackFrameNodeList() + { + NodesAndTokens = ImmutableArray.Empty; + } + + public ImmutableArray NodesAndTokens { get; } + public int Length => NodesAndTokens.Length; + public StackFrameNodeOrToken this[int index] => NodesAndTokens[index]; + + public static SeparatedStackFrameNodeList Empty => new SeparatedStackFrameNodeList(); + + public ImmutableArray GetNodes() + { + using var _ = ArrayBuilder.GetInstance(out var builder); + + for (var i = 0; i < NodesAndTokens.Length; i = i + 2) + { + var node = NodesAndTokens[i].Node; + RoslynDebug.AssertNotNull(node); + builder.Add((TNode)node); + } + + return builder.ToImmutable(); + } } /// @@ -34,9 +93,6 @@ internal abstract class StackFrameExpressionNode : StackFrameNode protected StackFrameExpressionNode(StackFrameKind kind) : base(kind) { } - - internal abstract StackFrameExpressionNode WithLeadingTrivia(ImmutableArray trivia); - internal abstract StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trivia); } internal sealed class StackFrameMethodDeclarationNode : StackFrameNode @@ -69,22 +125,21 @@ internal override StackFrameNodeOrToken ChildAt(int index) 2 => ArgumentList, _ => throw new InvalidOperationException(), }; - - internal StackFrameMethodDeclarationNode WithLeadingTrivia(ImmutableArray trivia) - => new((StackFrameMemberAccessExpressionNode)MemberAccessExpression.WithLeadingTrivia(trivia), TypeArguments, ArgumentList); } internal sealed class StackFrameMemberAccessExpressionNode : StackFrameExpressionNode { - public readonly StackFrameExpressionNode Expression; + public readonly StackFrameNodeOrToken Left; public readonly StackFrameToken Operator; - public readonly StackFrameBaseIdentifierNode Identifier; + public readonly StackFrameNodeOrToken Right; - public StackFrameMemberAccessExpressionNode(StackFrameExpressionNode expression, StackFrameToken operatorToken, StackFrameBaseIdentifierNode identifier) : base(StackFrameKind.MemberAccess) + public StackFrameMemberAccessExpressionNode(StackFrameNodeOrToken left, StackFrameToken operatorToken, StackFrameNodeOrToken right) : base(StackFrameKind.MemberAccess) { - Expression = expression; + Debug.Assert(left.IsNode || left.Token.Kind == StackFrameKind.IdentifierToken); + Debug.Assert(right.IsNode || right.Token.Kind == StackFrameKind.IdentifierToken); + Left = left; Operator = operatorToken; - Identifier = identifier; + Right = right; } internal override int ChildCount => 3; @@ -95,23 +150,11 @@ public override void Accept(IStackFrameNodeVisitor visitor) internal override StackFrameNodeOrToken ChildAt(int index) => index switch { - 0 => Expression, + 0 => Left, 1 => Operator, - 2 => Identifier, + 2 => Right, _ => throw new InvalidOperationException() }; - - internal StackFrameExpressionNode WithLeadingTrivia(StackFrameTrivia trivia) - => WithLeadingTrivia(ImmutableArray.Create(trivia)); - - internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray trivia) - => new StackFrameMemberAccessExpressionNode(Expression.WithLeadingTrivia(trivia), Operator, Identifier); - - internal StackFrameExpressionNode WithTrailingTrivia(StackFrameTrivia trivia) - => WithTrailingTrivia(ImmutableArray.Create(trivia)); - - internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trivia) - => new StackFrameMemberAccessExpressionNode(Expression, Operator, (StackFrameBaseIdentifierNode)Identifier.WithTrailingTrivia(trivia)); } internal abstract class StackFrameBaseIdentifierNode : StackFrameExpressionNode @@ -119,44 +162,6 @@ internal abstract class StackFrameBaseIdentifierNode : StackFrameExpressionNode protected StackFrameBaseIdentifierNode(StackFrameKind kind) : base(kind) { } - - internal StackFrameExpressionNode WithLeadingTrivia(StackFrameTrivia leadingTrivia) - => WithLeadingTrivia(ImmutableArray.Create(leadingTrivia)); - - internal StackFrameExpressionNode WithTrailingTrivia(StackFrameTrivia trailingTrivia) - => WithTrailingTrivia(ImmutableArray.Create(trailingTrivia)); - } - - internal sealed class StackFrameIdentifierNode : StackFrameBaseIdentifierNode - { - public readonly StackFrameToken Identifier; - - internal override int ChildCount => 1; - - internal StackFrameIdentifierNode(StackFrameToken identifier) - : base(StackFrameKind.Identifier) - { - Identifier = identifier; - } - - public override void Accept(IStackFrameNodeVisitor visitor) - => visitor.Visit(this); - - internal override StackFrameNodeOrToken ChildAt(int index) - => index switch - { - 0 => Identifier, - _ => throw new InvalidOperationException() - }; - - public override string ToString() - => Identifier.VirtualChars.CreateString(); - - internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trailingTrivia) - => new StackFrameIdentifierNode(Identifier.With(trailingTrivia: trailingTrivia)); - - internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray leadingTrivia) - => new StackFrameIdentifierNode(Identifier.With(leadingTrivia: leadingTrivia)); } internal sealed class StackFrameGenericTypeIdentifier : StackFrameBaseIdentifierNode @@ -186,55 +191,86 @@ internal override StackFrameNodeOrToken ChildAt(int index) 2 => ArityNumericToken, _ => throw new InvalidOperationException() }; - - internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray leadingTrivia) - => new StackFrameGenericTypeIdentifier( - Identifier.With(leadingTrivia: leadingTrivia), - ArityToken, - ArityNumericToken); - - internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trailingTrivia) - => new StackFrameGenericTypeIdentifier( - Identifier, - ArityToken, - ArityNumericToken.With(trailingTrivia: trailingTrivia)); } - internal sealed class StackFrameArrayExpressionNode : StackFrameExpressionNode + /// + /// Represents an array type declaration, such as string[,][] + /// + internal sealed class StackFrameArrayTypeExpression : StackFrameExpressionNode { - private readonly StackFrameExpressionNode _identifier; - private readonly ImmutableArray _arrayBrackets; - - public StackFrameArrayExpressionNode(StackFrameExpressionNode identifier, ImmutableArray arrayBrackets) - : base(StackFrameKind.ArrayExpression) + /// + /// The type identifier without the array indicators. + /// string[][] + /// ^----^ + /// + public readonly StackFrameNodeOrToken TypeIdentifier; + + /// + /// Each unique array identifier for the type + /// string[,][] + /// ^--- First array expression = "[,]" + /// ^- Second array expression = "[]" + /// + public ImmutableArray ArrayExpressions; + + public StackFrameArrayTypeExpression(StackFrameNodeOrToken typeIdentifier, ImmutableArray arrayExpressions) : base(StackFrameKind.ArrayTypeExpression) { - _identifier = identifier; - _arrayBrackets = arrayBrackets; - - Debug.Assert(arrayBrackets.All(t => t.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.CloseBracketToken or StackFrameKind.CommaToken)); + Contract.ThrowIfTrue(arrayExpressions.IsDefaultOrEmpty); + TypeIdentifier = typeIdentifier; + ArrayExpressions = arrayExpressions; } - internal override int ChildCount => _arrayBrackets.Length + 1; + internal override int ChildCount => 1 + ArrayExpressions.Length; public override void Accept(IStackFrameNodeVisitor visitor) - => visitor.Visit(this); + { + visitor.Visit(this); + } internal override StackFrameNodeOrToken ChildAt(int index) => index switch { - 0 => _identifier, - _ => _arrayBrackets[index - 1] + 0 => TypeIdentifier, + _ => ArrayExpressions[index - 1] }; + } + internal sealed class StackFrameArrayRankSpecifier : StackFrameExpressionNode + { + public readonly StackFrameToken OpenBracket; + public readonly StackFrameToken CloseBracket; + public ImmutableArray CommaTokens; + + public StackFrameArrayRankSpecifier(StackFrameToken openBracket, StackFrameToken closeBracket, ImmutableArray commaTokens) + : base(StackFrameKind.ArrayExpression) + { + Contract.ThrowIfTrue(commaTokens.IsDefault); + Debug.Assert(openBracket.Kind == StackFrameKind.OpenBracketToken); + Debug.Assert(closeBracket.Kind == StackFrameKind.CloseBracketToken); + Debug.Assert(commaTokens.All(t => t.Kind == StackFrameKind.CommaToken)); + + OpenBracket = openBracket; + CloseBracket = closeBracket; + CommaTokens = commaTokens; + } + + internal override int ChildCount => 2 + CommaTokens.Length; - internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray trivia) - => new StackFrameArrayExpressionNode(_identifier.WithLeadingTrivia(trivia), _arrayBrackets); + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); - internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trivia) + internal override StackFrameNodeOrToken ChildAt(int index) { - var lastBracket = _arrayBrackets.Last().With(trailingTrivia: trivia); - var newBrackets = _arrayBrackets.RemoveAt(_arrayBrackets.Length - 1).Add(lastBracket); + if (index == 0) + { + return OpenBracket; + } + + if (index == ChildCount - 1) + { + return CloseBracket; + } - return new StackFrameArrayExpressionNode(_identifier, newBrackets); + return CommaTokens[index - 1]; } } @@ -242,24 +278,19 @@ internal sealed class StackFrameTypeArgumentList : StackFrameNode { public readonly StackFrameToken OpenToken; public readonly StackFrameToken CloseToken; + public readonly SeparatedStackFrameNodeList TypeArguments; - public StackFrameTypeArgumentList(StackFrameToken openToken, ImmutableArray childNodesOrTokens, StackFrameToken closeToken) : base(StackFrameKind.TypeArgument) + public StackFrameTypeArgumentList(StackFrameToken openToken, SeparatedStackFrameNodeList typeArguments, StackFrameToken closeToken) : base(StackFrameKind.TypeArgument) { Debug.Assert(openToken.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken); Debug.Assert(openToken.Kind == StackFrameKind.OpenBracketToken ? closeToken.Kind == StackFrameKind.CloseBracketToken : closeToken.Kind == StackFrameKind.GreaterThanToken); - Debug.Assert(childNodesOrTokens.All(nodeOrToken => nodeOrToken.IsNode - ? nodeOrToken.Node is StackFrameTypeArgument - : nodeOrToken.Token.Kind == StackFrameKind.CommaToken)); OpenToken = openToken; CloseToken = closeToken; - _childNodesOrTokens = childNodesOrTokens; - ChildCount = childNodesOrTokens.Length + 2; + TypeArguments = typeArguments; } - private readonly ImmutableArray _childNodesOrTokens; - - internal override int ChildCount { get; } + internal override int ChildCount => TypeArguments.Length + 2; public override void Accept(IStackFrameNodeVisitor visitor) => visitor.Visit(this); @@ -281,17 +312,17 @@ internal override StackFrameNodeOrToken ChildAt(int index) return CloseToken; } - return _childNodesOrTokens[index - 1]; + return TypeArguments[index - 1]; } } - internal sealed class StackFrameTypeArgument : StackFrameBaseIdentifierNode + internal sealed class StackFrameTypeArgumentNode : StackFrameBaseIdentifierNode { public readonly StackFrameToken Identifier; internal override int ChildCount => 1; - internal StackFrameTypeArgument(StackFrameToken identifier) + internal StackFrameTypeArgumentNode(StackFrameToken identifier) : base(StackFrameKind.TypeIdentifier) { Identifier = identifier; @@ -306,46 +337,61 @@ internal override StackFrameNodeOrToken ChildAt(int index) 0 => Identifier, _ => throw new InvalidOperationException() }; + } - internal override StackFrameExpressionNode WithTrailingTrivia(ImmutableArray trailingTrivia) - => new StackFrameTypeArgument(Identifier.With(trailingTrivia: trailingTrivia)); + internal sealed class StackFrameParameterNode : StackFrameExpressionNode + { + public readonly StackFrameNodeOrToken Type; + public readonly StackFrameToken Identifier; - internal override StackFrameExpressionNode WithLeadingTrivia(ImmutableArray leadingTrivia) - => new StackFrameTypeArgument(Identifier.With(leadingTrivia: leadingTrivia)); + internal override int ChildCount => 2; + + public StackFrameParameterNode(StackFrameNodeOrToken type, StackFrameToken identifier) + : base(StackFrameKind.Parameter) + { + Debug.Assert(type.IsNode || !type.Token.IsMissing); + Type = type; + Identifier = identifier; + } + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => Type, + 1 => Identifier, + _ => throw new InvalidOperationException() + }; } - internal sealed class StackFrameParameterList : StackFrameNode + internal sealed class StackFrameParameterList : StackFrameExpressionNode { public readonly StackFrameToken OpenParen; public readonly StackFrameToken CloseParen; - private readonly ImmutableArray _childNodesOrTokens; + public readonly SeparatedStackFrameNodeList Parameters; - public StackFrameParameterList(StackFrameToken openParen, ImmutableArray childNodesOrTokens, StackFrameToken closeParen) : base(StackFrameKind.ParameterList) + public StackFrameParameterList(StackFrameToken openToken, StackFrameToken closeToken, SeparatedStackFrameNodeList parameters) + : base(StackFrameKind.ParameterList) { - Debug.Assert(openParen.Kind == StackFrameKind.OpenParenToken); - Debug.Assert(closeParen.Kind == StackFrameKind.CloseParenToken); - Debug.Assert(childNodesOrTokens.IsDefaultOrEmpty || childNodesOrTokens.All(nodeOrToken => nodeOrToken.IsNode - ? nodeOrToken.Node is StackFrameIdentifierNode or StackFrameMemberAccessExpressionNode or StackFrameArrayExpressionNode - : nodeOrToken.Token.Kind is StackFrameKind.CommaToken)); - - OpenParen = openParen; - CloseParen = closeParen; - _childNodesOrTokens = childNodesOrTokens; - ChildCount = _childNodesOrTokens.IsDefault ? 2 : _childNodesOrTokens.Length + 2; + Debug.Assert(openToken.Kind == StackFrameKind.OpenParenToken); + Debug.Assert(closeToken.Kind == StackFrameKind.CloseParenToken); + + OpenParen = openToken; + CloseParen = closeToken; + Parameters = parameters; } - internal override int ChildCount { get; } + internal override int ChildCount => 2 + Parameters.Length; public override void Accept(IStackFrameNodeVisitor visitor) - => visitor.Visit(this); + { + visitor.Visit(this); + } internal override StackFrameNodeOrToken ChildAt(int index) { - if (index >= ChildCount) - { - throw new InvalidOperationException(); - } - if (index == 0) { return OpenParen; @@ -356,7 +402,7 @@ internal override StackFrameNodeOrToken ChildAt(int index) return CloseParen; } - return _childNodesOrTokens[index - 1]; + return Parameters[index - 1]; } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 14d2fba521c4a..bb314b3c21efa 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -6,12 +6,9 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; -using System.IO; -using System.Linq; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.PooledObjects; -using Microsoft.CodeAnalysis.Shared.Utilities; using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; @@ -22,14 +19,6 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame using StackFrameToken = EmbeddedSyntaxToken; using StackFrameTrivia = EmbeddedSyntaxTrivia; - internal class StackFrameParseException : Exception - { - public StackFrameParseException(StackFrameKind expectedKind, StackFrameToken actual) - : base($"Expected {expectedKind} instead of '{actual.VirtualChars.CreateString()}' at {actual.GetSpan().Start}") - { - } - } - /// /// Attempts to parse a stack frame line from given input. StackFrame is generally /// defined as a string line in a StackTrace. See https://docs.microsoft.com/en-us/dotnet/api/system.environment.stacktrace for @@ -37,6 +26,33 @@ public StackFrameParseException(StackFrameKind expectedKind, StackFrameToken act /// internal struct StackFrameParser { + private class StackFrameParseException : Exception + { + public StackFrameParseException(StackFrameKind expectedKind, StackFrameNodeOrToken actual) + : this($"Expected {expectedKind} instead of {GetDetails(actual)}") + { + } + + private static string GetDetails(StackFrameNodeOrToken actual) + { + if (actual.IsNode) + { + var node = actual.Node; + return $"'{node.Kind}' at {node.GetSpan().Start}"; + } + else + { + var token = actual.Token; + return $"'{token.VirtualChars.CreateString()}' at {token.GetSpan().Start}"; + } + } + + public StackFrameParseException(string message) + : base(message) + { + } + } + private StackFrameLexer _lexer; private StackFrameToken CurrentToken => _lexer.CurrentCharAsToken(); @@ -83,24 +99,12 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameTree? TryParseTree() { - var atTrivia = _lexer.ScanAtTrivia(); - var methodDeclaration = TryParseMethodDeclaration(); if (methodDeclaration is null) { return null; } - if (atTrivia.HasValue) - { - var currentTrivia = methodDeclaration.ChildAt(0).Token.LeadingTrivia; - var newList = currentTrivia.IsDefaultOrEmpty - ? ImmutableArray.Create(atTrivia.Value) - : currentTrivia.Prepend(atTrivia.Value).ToImmutableArray(); - - methodDeclaration = methodDeclaration.WithLeadingTrivia(newList); - } - var inTrivia = _lexer.ScanInTrivia(); var fileInformationNodeOrToken = inTrivia.HasValue ? TryParseFileInformation() @@ -159,8 +163,13 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameMethodDeclarationNode? TryParseMethodDeclaration() { - var identifierExpression = TryParseIdentifierExpression(); - if (identifierExpression is not StackFrameMemberAccessExpressionNode memberAccessExpression) + var identifierExpression = TryParseIdentifierExpression(scanAtTrivia: true); + if (!(identifierExpression.HasValue && identifierExpression.Value.IsNode)) + { + return null; + } + + if (identifierExpression.Value.Node is not StackFrameMemberAccessExpressionNode memberAccessExpression) { return null; } @@ -189,12 +198,11 @@ private StackFrameParser(VirtualCharSequence text) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) /// - private StackFrameExpressionNode? TryParseIdentifierExpression() + private StackFrameNodeOrToken? TryParseIdentifierExpression(bool scanAtTrivia = false) { - Queue<(StackFrameBaseIdentifierNode identifier, StackFrameToken separator)> typeIdentifierNodes = new(); + Queue<(StackFrameNodeOrToken identifier, StackFrameToken separator)> typeIdentifierNodes = new(); - var leadingTrivia = _lexer.ScanWhiteSpace(); - var currentIdentifier = _lexer.ScanIdentifier(); + var currentIdentifier = _lexer.ScanIdentifier(scanAtTrivia: scanAtTrivia, scanWhitespace: true); while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { @@ -208,15 +216,9 @@ private StackFrameParser(VirtualCharSequence text) } } - if (leadingTrivia.HasValue) - { - currentIdentifier = currentIdentifier.Value.With(leadingTrivia: ImmutableArray.Create(leadingTrivia.Value)); - leadingTrivia = null; - } - - StackFrameBaseIdentifierNode identifierNode = arity.HasValue + StackFrameNodeOrToken identifierNode = arity.HasValue ? new StackFrameGenericTypeIdentifier(currentIdentifier.Value, graveAccentToken, arity.Value) - : new StackFrameIdentifierNode(currentIdentifier.Value); + : currentIdentifier.Value; typeIdentifierNodes.Enqueue((identifierNode, CurrentToken)); @@ -235,18 +237,13 @@ private StackFrameParser(VirtualCharSequence text) return null; } - var trailingTrivia = _lexer.ScanWhiteSpace(); - - if (typeIdentifierNodes.Count == 1) + var (firstIdentifierNode, firstSeparator) = typeIdentifierNodes.Dequeue(); + if (typeIdentifierNodes.Count == 0) { - var identifierNode = typeIdentifierNodes.Dequeue().identifier; - return trailingTrivia.HasValue - ? identifierNode.WithTrailingTrivia(trailingTrivia.Value) - : identifierNode; + return firstIdentifierNode; } - // Construct the member access expression from the identifiers - var (firstIdentifierNode, firstSeparator) = typeIdentifierNodes.Dequeue(); + // Construct the member access expression from the identifiers in the list var currentSeparator = firstSeparator; StackFrameMemberAccessExpressionNode? memberAccessExpression = null; @@ -256,7 +253,7 @@ private StackFrameParser(VirtualCharSequence text) var previousSeparator = currentSeparator; (var currentIdentifierNode, currentSeparator) = typeIdentifierNodes.Dequeue(); - StackFrameExpressionNode leftHandNode = memberAccessExpression is null + var leftHandNode = memberAccessExpression is null ? firstIdentifierNode : memberAccessExpression; @@ -265,9 +262,7 @@ private StackFrameParser(VirtualCharSequence text) RoslynDebug.AssertNotNull(memberAccessExpression); - return trailingTrivia.HasValue - ? memberAccessExpression.WithTrailingTrivia(trailingTrivia.Value) - : memberAccessExpression; + return memberAccessExpression; } /// @@ -296,7 +291,7 @@ private StackFrameParser(VirtualCharSequence text) while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { - builder.Add(new StackFrameTypeArgument(currentIdentifier.Value)); + builder.Add(new StackFrameTypeArgumentNode(currentIdentifier.Value)); if (_lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, out var closeBracket)) { @@ -329,7 +324,7 @@ private StackFrameParser(VirtualCharSequence text) return null; } - return new StackFrameTypeArgumentList(openToken, builder.ToImmutable(), closeToken); + return new StackFrameTypeArgumentList(openToken, new SeparatedStackFrameNodeList(builder.ToImmutable()), closeToken); } /// @@ -340,95 +335,99 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameParameterList? TryParseMethodParameters() { - if (!_lexer.ScanIfMatch(StackFrameKind.OpenParenToken, out var openParen)) + if (!_lexer.ScanIfMatch(StackFrameKind.OpenParenToken, scanTrailingWhitespace: true, out var openParen)) { return null; } - var spaceTrivia = _lexer.ScanWhiteSpace(); - if (spaceTrivia.HasValue) + if (_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) { - openParen = openParen.With(trailingTrivia: ImmutableArray.Create(spaceTrivia.Value)); + return new StackFrameParameterList(openParen, closeParen, SeparatedStackFrameNodeList.Empty); } - StackFrameToken closeParen; - using var _ = ArrayBuilder.GetInstance(out var builder); - var identifier = TryParseIdentifierExpression(); + builder.Add(ParseParameterNode()); + while (_lexer.ScanIfMatch(StackFrameKind.CommaToken, out var commaToken)) + { + builder.Add(commaToken); + builder.Add(ParseParameterNode()); + } - while (identifier is not null) + if (!_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out closeParen)) { - // Check if there's an array type for the identifier - if (CurrentToken.Kind == StackFrameKind.OpenBracketToken) - { - if (TryParseArrayIdentifier(identifier, out var parsedTokens)) - { - builder.Add(new StackFrameArrayExpressionNode(identifier, parsedTokens)); - } - else - { - // Invalid array identifiers, bail parsing parameters - return null; - } - } - else - { - builder.Add(identifier); - } + throw new StackFrameParseException(StackFrameKind.CloseParenToken, CurrentToken); + } - if (_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out closeParen)) - { - return new StackFrameParameterList(openParen, builder.ToImmutable(), closeParen); - } + var parameters = new SeparatedStackFrameNodeList(builder.ToImmutable()); + return new StackFrameParameterList(openParen, closeParen, parameters); + } - if (_lexer.ScanIfMatch(StackFrameKind.CommaToken, out var commaToken)) - { - builder.Add(commaToken); - } + /// + /// Parses a by parsing identifiers first representing the type and then the parameter identifier. + /// Ex: System.String[] s + /// ^--------------^ -- Type = "System.String[]" + /// ^-- Identifier = "s" + /// + private StackFrameParameterNode ParseParameterNode() + { + var typeIdentifier = TryParseIdentifierExpression(); + if (!typeIdentifier.HasValue) + { + throw new StackFrameParseException("Expected type identifier when parsing parameters"); + } + + if (CurrentToken.Kind == StackFrameKind.OpenBracketToken) + { + var arrayIdentifiers = ParseArrayIdentifiers(); + typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier.Value, arrayIdentifiers); + } - var addLeadingWhitespaceAsTrivia = identifier.ChildAt(identifier.ChildCount - 1).Token.TrailingTrivia.IsDefaultOrEmpty; - identifier = TryParseIdentifierExpression(); + var identifier = TryParseIdentifierExpression(); + if (!identifier.HasValue) + { + throw new StackFrameParseException("Expected a parameter identifier"); } - if (_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out closeParen)) + // Parameter identifiers should only be tokens + if (identifier.Value.IsNode) { - return new StackFrameParameterList(openParen, builder.ToImmutable(), closeParen); + throw new StackFrameParseException(StackFrameKind.IdentifierToken, identifier.Value); } - return null; + return new StackFrameParameterNode(typeIdentifier.Value, identifier.Value.Token); } /// - /// Given an input like "string[]" where "string" is the existing - /// passed in, converts it into by parsing the array portion - /// of the identifier. + /// Parses the array rank specifiers for an identifier. + /// Ex: string[,][] + /// ^----^ both are array rank specifiers + /// 0: "[,] + /// 1: "[]" /// - private bool TryParseArrayIdentifier(StackFrameExpressionNode identifier, out ImmutableArray arrayTokens) + private ImmutableArray ParseArrayIdentifiers() { - using var _ = ArrayBuilder.GetInstance(out var builder); + using var _ = ArrayBuilder.GetInstance(out var builder); + using var _1 = ArrayBuilder.GetInstance(out var commaBuilder); while (true) { - if (!_lexer.ScanIfMatch(StackFrameKind.OpenBracketToken, out var openBracket)) + if (!_lexer.ScanIfMatch(StackFrameKind.OpenBracketToken, scanTrailingWhitespace: true, out var openBracket)) { - arrayTokens = builder.ToImmutable(); - return arrayTokens.Length > 0 && arrayTokens[^1].Kind == StackFrameKind.CloseBracketToken; + return builder.ToImmutable(); } - builder.Add(AddTrailingWhitespace(openBracket)); - - if (_lexer.ScanIfMatch(StackFrameKind.CommaToken, out var commaToken)) + commaBuilder.Clear(); + while (_lexer.ScanIfMatch(StackFrameKind.CommaToken, scanTrailingWhitespace: true, out var commaToken)) { - builder.Add(AddTrailingWhitespace(commaToken)); + commaBuilder.Add(commaToken); } - if (!_lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, out var closeBracket)) + if (!_lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, scanTrailingWhitespace: true, out var closeBracket)) { - arrayTokens = builder.ToImmutable(); - return arrayTokens.Length > 0 && arrayTokens.Last().Kind == StackFrameKind.CloseBracketToken; + throw new StackFrameParseException(StackFrameKind.CloseBracketToken, CurrentToken); } - builder.Add(AddTrailingWhitespace(closeBracket)); + builder.Add(new StackFrameArrayRankSpecifier(openBracket, closeBracket, commaBuilder.ToImmutable())); } throw ExceptionUtilities.Unreachable; @@ -475,16 +474,5 @@ private StackFrameNodeOrToken TryParseFileInformation() colonToken, numbers.Value.With(leadingTrivia: ImmutableArray.Create(lineIdentifier.Value))); } - - private StackFrameToken AddTrailingWhitespace(StackFrameToken token) - { - var whitespace = _lexer.ScanWhiteSpace(); - if (whitespace.HasValue) - { - return token.With(trailingTrivia: ImmutableArray.Create(whitespace.Value)); - } - - return token; - } } } From 3b4cbcb002eb4d4639244e2698b10a049ff15562 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Mon, 25 Oct 2021 19:48:52 -0700 Subject: [PATCH 18/59] PR feedback --- .../StackFrameParserTests.Generators.cs | 2 +- .../StackFrame/IStackFrameNodeVisitor.cs | 1 + .../StackFrame/StackFrameCompilationUnit.cs | 18 ++- .../StackFrame/StackFrameKind.cs | 5 +- .../StackFrame/StackFrameLexer.cs | 117 ++++++++---------- .../StackFrame/StackFrameNodeDefinitions.cs | 14 ++- .../StackFrame/StackFrameParser.cs | 86 +++++++------ .../StackFrame/StackFrameTree.cs | 4 +- 8 files changed, 128 insertions(+), 119 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs index 96d4b0c17bbb4..5fb30f603b73e 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs @@ -37,7 +37,7 @@ private static ImmutableArray CreateTriviaArray(StackFrameTriv => trivia.HasValue ? ImmutableArray.Create(trivia.Value) : ImmutableArray.Empty; private static ImmutableArray CreateTriviaArray(params string[] strings) - => strings.Select(s => CreateTrivia(StackFrameKind.TextTrivia, s)).ToImmutableArray(); + => strings.Select(s => CreateTrivia(StackFrameKind.SkippedTextTrivia, s)).ToImmutableArray(); private static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); private static readonly StackFrameToken CommaToken = CreateToken(StackFrameKind.CommaToken, ","); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index 5228f384bd91f..7ad72234d7d78 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -16,5 +16,6 @@ internal interface IStackFrameNodeVisitor void Visit(StackFrameFileInformationNode node); void Visit(StackFrameArrayTypeExpression node); void Visit(StackFrameParameterNode stackFrameParameterNode); + void Visit(StackFrameCompilationUnit stackFrameCompilationUnit); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs index ca8424363d4e1..ee4b529cb31e7 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameCompilationUnit.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame @@ -15,7 +16,7 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame /// Any leading "at " is considered trivia of , and " in " is put as trivia for the . /// Remaining unparsable text is put as leading trivia on the /// - internal class StackFrameCompilationUnit + internal class StackFrameCompilationUnit : StackFrameNode { /// /// Represents the method declaration for a stack frame. Requires at least a member @@ -36,10 +37,25 @@ internal class StackFrameCompilationUnit public readonly StackFrameToken EndOfLineToken; public StackFrameCompilationUnit(StackFrameMethodDeclarationNode methodDeclaration, StackFrameFileInformationNode? fileInformationExpression, StackFrameToken endOfLineToken) + : base(StackFrameKind.CompilationUnit) { MethodDeclaration = methodDeclaration; FileInformationExpression = fileInformationExpression; EndOfLineToken = endOfLineToken; } + + internal override int ChildCount => 3; + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => MethodDeclaration, + 1 => FileInformationExpression, + 2 => EndOfLineToken, + _ => throw new InvalidOperationException() + }; } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 06f322c3b981e..f56b544394cac 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -11,7 +11,6 @@ internal enum StackFrameKind None = 0, // Nodes - EndOfLine, MethodDeclaration, MemberAccess, ArrayTypeExpression, @@ -22,8 +21,10 @@ internal enum StackFrameKind ParameterList, ArrayExpression, FileInformation, + CompilationUnit, // Tokens + EndOfLine, AmpersandToken, OpenBracketToken, CloseBracketToken, @@ -50,6 +51,6 @@ internal enum StackFrameKind AtTrivia, // "at " portion of the stack frame InTrivia, // optional " in " portion of the stack frame LineTrivia, // optional "line " string indicating the line number of a file - TextTrivia, + SkippedTextTrivia, // any skipped text that isn't a node, token, or special kind of trivia already presented } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index de0d420420996..fb263461ac77f 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Immutable; +using System.Diagnostics; using System.IO; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; @@ -32,13 +33,13 @@ public StackFrameLexer(VirtualCharSequence text) : this() public VirtualChar CurrentChar => Position < Text.Length ? Text[Position] : default; - public VirtualCharSequence GetSubPatternToCurrentPos(int start) - => GetSubPattern(start, Position); + public VirtualCharSequence GetSubSequenceToCurrentPos(int start) + => GetSubSequence(start, Position); - public VirtualCharSequence GetSubPattern(int start, int end) + public VirtualCharSequence GetSubSequence(int start, int end) => Text.GetSubSequence(TextSpan.FromBounds(start, end)); - public StackFrameTrivia? ScanRemainingTrivia() + public StackFrameTrivia? TryScanRemainingTrivia() { if (Position == Text.Length) { @@ -48,14 +49,14 @@ public VirtualCharSequence GetSubPattern(int start, int end) var start = Position; Position = Text.Length; - return CreateTrivia(StackFrameKind.TextTrivia, GetSubPatternToCurrentPos(start)); + return CreateTrivia(StackFrameKind.SkippedTextTrivia, GetSubSequenceToCurrentPos(start)); } - public StackFrameToken? ScanIdentifier(bool scanAtTrivia = false, bool scanWhitespace = false) + public StackFrameToken? TryScanIdentifier(bool scanAtTrivia = false, bool scanWhitespace = false) { var originalPosition = Position; - var atTrivia = scanAtTrivia ? ScanAtTrivia() : null; - var whitespaceTrivia = scanWhitespace ? ScanWhiteSpace() : null; + var atTrivia = scanAtTrivia ? TryScanAtTrivia() : null; + var whitespaceTrivia = scanWhitespace ? TryScanWhiteSpace() : null; var startPosition = Position; var ch = CurrentChar; @@ -74,16 +75,11 @@ public VirtualCharSequence GetSubPattern(int start, int end) ch = CurrentChar; } - return CreateToken(StackFrameKind.IdentifierToken, CreateTrivia(atTrivia, whitespaceTrivia), GetSubPatternToCurrentPos(startPosition)); + return CreateToken(StackFrameKind.IdentifierToken, CreateTrivia(atTrivia, whitespaceTrivia), GetSubSequenceToCurrentPos(startPosition)); } - internal StackFrameTrivia? ScanWhiteSpace() + internal StackFrameTrivia? TryScanWhiteSpace() { - if (Position == Text.Length) - { - return null; - } - var startPosition = Position; while (IsBlank(CurrentChar)) @@ -96,8 +92,7 @@ public VirtualCharSequence GetSubPattern(int start, int end) return null; } - var whitespaceSpan = new TextSpan(startPosition, Position - startPosition); - return CreateTrivia(StackFrameKind.WhitespaceTrivia, Text.GetSubSequence(whitespaceSpan)); + return CreateTrivia(StackFrameKind.WhitespaceTrivia, GetSubSequenceToCurrentPos(startPosition)); } public StackFrameToken CurrentCharAsToken() @@ -107,8 +102,8 @@ public StackFrameToken CurrentCharAsToken() return CreateToken(StackFrameKind.EndOfLine, VirtualCharSequence.Empty); } - var previousChar = Text[Position]; - return CreateToken(GetKind(previousChar), Text.GetSubSequence(new TextSpan(Position, 1))); + var ch = Text[Position]; + return CreateToken(GetKind(ch), Text.GetSubSequence(new TextSpan(Position, 1))); } public bool IsStringAtPosition(string val) @@ -135,8 +130,8 @@ private bool TextAt(int position, string val) /// /// if the position was incremented /// - internal bool ScanIfMatch(StackFrameKind kind, out StackFrameToken token) - => ScanIfMatch(kind, scanTrailingWhitespace: false, out token); + internal bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, out StackFrameToken token) + => ScanCurrentCharAsTokenIfMatch(kind, scanTrailingWhitespace: false, out token); /// /// Progress the position by one if the current character @@ -145,7 +140,7 @@ internal bool ScanIfMatch(StackFrameKind kind, out StackFrameToken token) /// /// if the position was incremented /// - internal bool ScanIfMatch(StackFrameKind kind, bool scanTrailingWhitespace, out StackFrameToken token) + internal bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, bool scanTrailingWhitespace, out StackFrameToken token) { if (GetKind(CurrentChar) == kind) { @@ -154,7 +149,7 @@ internal bool ScanIfMatch(StackFrameKind kind, bool scanTrailingWhitespace, out if (scanTrailingWhitespace) { - token = token.With(trailingTrivia: CreateTrivia(ScanWhiteSpace())); + token = token.With(trailingTrivia: CreateTrivia(TryScanWhiteSpace())); } return true; @@ -171,7 +166,7 @@ internal bool ScanIfMatch(StackFrameKind kind, bool scanTrailingWhitespace, out /// /// if the position was incremented /// - internal bool ScanIfMatch(Func matchFn, out StackFrameToken token) + internal bool ScanCurrentCharAsTokenIfMatch(Func matchFn, out StackFrameToken token) { if (matchFn(GetKind(CurrentChar))) { @@ -209,23 +204,14 @@ private static StackFrameKind GetKind(VirtualChar ch) ? StackFrameKind.WhitespaceTrivia : IsNumber(ch) ? StackFrameKind.NumberToken - : StackFrameKind.TextTrivia + : StackFrameKind.SkippedTextTrivia }; private static bool IsNumber(VirtualChar ch) - => ch.Value switch - { - >= '0' and <= '9' => true, - _ => false - }; + => ch.Value is >= '0' and <= '9'; - public StackFrameTrivia? ScanAtTrivia() + public StackFrameTrivia? TryScanAtTrivia() { - if (Position == Text.Length) - { - return null; - } - // TODO: Handle multiple languages? Right now we're going to only parse english const string AtString = "at "; @@ -234,19 +220,14 @@ private static bool IsNumber(VirtualChar ch) var start = Position; Position += AtString.Length; - return CreateTrivia(StackFrameKind.AtTrivia, GetSubPatternToCurrentPos(start)); + return CreateTrivia(StackFrameKind.AtTrivia, GetSubSequenceToCurrentPos(start)); } return null; } - public StackFrameTrivia? ScanInTrivia() + public StackFrameTrivia? TryScanInTrivia() { - if (Position == Text.Length) - { - return null; - } - // TODO: Handle multiple languages? Right now we're going to only parse english const string InString = " in "; @@ -255,7 +236,7 @@ private static bool IsNumber(VirtualChar ch) var start = Position; Position += InString.Length; - return CreateTrivia(StackFrameKind.InTrivia, GetSubPatternToCurrentPos(start)); + return CreateTrivia(StackFrameKind.InTrivia, GetSubSequenceToCurrentPos(start)); } return null; @@ -265,20 +246,26 @@ private static bool IsNumber(VirtualChar ch) /// Attempts to parse a path following https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names /// Uses as a tool to determine if the path is correct for returning. /// - internal StackFrameToken? ScanPath() + internal StackFrameToken? TryScanPath() { - if (Position == Text.Length) - { - return null; - } - var startPosition = Position; while (Position < Text.Length) { // Path needs to do a look ahead to determine if adding the next character - // invalidates the path. Break if it does - var str = GetSubPattern(startPosition, Position + 1).CreateString(); + // invalidates the path. Break if it does. + // + // This helps to keep the complex rules for what FileInfo does to validate path by calling to it directly. + // We can't simply check all invalid characters for a path because location in the path is important, and we're not + // in the business of validating if something is correctly a file. It's cheap enough to let the FileInfo constructor do that. The downside + // is that we are constructing a new object with every pass. If this becomes problematic we can revisit and fine a more + // optimized pattern to handle all of the edge cases. + // + // Example: C:\my\path \:other + // ^-- the ":" breaks the path, but we can't simply break on all ":" (which is included in the invalid characters for Path) + // + + var str = GetSubSequence(startPosition, Position + 1).CreateString(); var isValidPath = IOUtilities.PerformIO(() => { @@ -299,16 +286,11 @@ private static bool IsNumber(VirtualChar ch) return null; } - return CreateToken(StackFrameKind.PathToken, GetSubPatternToCurrentPos(startPosition)); + return CreateToken(StackFrameKind.PathToken, GetSubSequenceToCurrentPos(startPosition)); } - internal StackFrameTrivia? ScanLineTrivia() + internal StackFrameTrivia? TryScanLineTrivia() { - if (Position == Text.Length) - { - return null; - } - // TODO: Handle multiple languages? Right now we're going to only parse english const string LineString = "line "; if (IsStringAtPosition(LineString)) @@ -316,19 +298,14 @@ private static bool IsNumber(VirtualChar ch) var start = Position; Position += LineString.Length; - return CreateTrivia(StackFrameKind.LineTrivia, GetSubPatternToCurrentPos(start)); + return CreateTrivia(StackFrameKind.LineTrivia, GetSubSequenceToCurrentPos(start)); } return null; } - internal StackFrameToken? ScanNumbers() + internal StackFrameToken? TryScanNumbers() { - if (Position == Text.Length) - { - return null; - } - var start = Position; while (IsNumber(CurrentChar)) { @@ -340,7 +317,7 @@ private static bool IsNumber(VirtualChar ch) return null; } - return CreateToken(StackFrameKind.NumberToken, GetSubPatternToCurrentPos(start)); + return CreateToken(StackFrameKind.NumberToken, GetSubSequenceToCurrentPos(start)); } public static bool IsBlank(VirtualChar ch) @@ -369,7 +346,11 @@ public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequ => CreateTrivia(kind, virtualChars, ImmutableArray.Empty); public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars, ImmutableArray diagnostics) - => new(kind, virtualChars, diagnostics); + { + // Empty trivia is not supported in StackFrames + Debug.Assert(virtualChars.Length > 0); + return new(kind, virtualChars, diagnostics); + } public static ImmutableArray CreateTrivia(params StackFrameTrivia?[] triviaArray) { diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index c9ef7010959a3..886c1ae2f5f94 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -101,7 +101,7 @@ internal sealed class StackFrameMethodDeclarationNode : StackFrameNode public readonly StackFrameTypeArgumentList? TypeArguments; public readonly StackFrameParameterList ArgumentList; - internal StackFrameMethodDeclarationNode( + public StackFrameMethodDeclarationNode( StackFrameMemberAccessExpressionNode memberAccessExpression, StackFrameTypeArgumentList? typeArguments, StackFrameParameterList argumentList) @@ -172,7 +172,7 @@ internal sealed class StackFrameGenericTypeIdentifier : StackFrameBaseIdentifier internal override int ChildCount => 3; - internal StackFrameGenericTypeIdentifier(StackFrameToken identifier, StackFrameToken arityToken, StackFrameToken arityNumericToken) + public StackFrameGenericTypeIdentifier(StackFrameToken identifier, StackFrameToken arityToken, StackFrameToken arityNumericToken) : base(StackFrameKind.GenericTypeIdentifier) { Identifier = identifier; @@ -274,6 +274,13 @@ internal override StackFrameNodeOrToken ChildAt(int index) } } + /// + /// The type argument list for a method declaration. + /// Ex: MyType.MyMethod[T, U, V](T t, U u, V v) + /// ^----------------------- "[" = Open Token + /// ^------^ ------------ "T, U, V" = SeparatedStackFrameNodeList<StackFrameTypeArgumentNode> + /// ^-------------- "]" = Close Token + /// internal sealed class StackFrameTypeArgumentList : StackFrameNode { public readonly StackFrameToken OpenToken; @@ -284,6 +291,7 @@ public StackFrameTypeArgumentList(StackFrameToken openToken, SeparatedStackFrame { Debug.Assert(openToken.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken); Debug.Assert(openToken.Kind == StackFrameKind.OpenBracketToken ? closeToken.Kind == StackFrameKind.CloseBracketToken : closeToken.Kind == StackFrameKind.GreaterThanToken); + Debug.Assert(typeArguments.Length > 0); OpenToken = openToken; CloseToken = closeToken; @@ -322,7 +330,7 @@ internal sealed class StackFrameTypeArgumentNode : StackFrameBaseIdentifierNode internal override int ChildCount => 1; - internal StackFrameTypeArgumentNode(StackFrameToken identifier) + public StackFrameTypeArgumentNode(StackFrameToken identifier) : base(StackFrameKind.TypeIdentifier) { Identifier = identifier; diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index bb314b3c21efa..8d90609dc7497 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -54,7 +54,7 @@ public StackFrameParseException(string message) } private StackFrameLexer _lexer; - private StackFrameToken CurrentToken => _lexer.CurrentCharAsToken(); + private StackFrameToken CurrentCharAsToken() => _lexer.CurrentCharAsToken(); private StackFrameParser(VirtualCharSequence text) { @@ -105,7 +105,7 @@ private StackFrameParser(VirtualCharSequence text) return null; } - var inTrivia = _lexer.ScanInTrivia(); + var inTrivia = _lexer.TryScanInTrivia(); var fileInformationNodeOrToken = inTrivia.HasValue ? TryParseFileInformation() : null; @@ -122,7 +122,7 @@ private StackFrameParser(VirtualCharSequence text) else if (inTrivia.HasValue) { // If the file path wasn't valid make sure to add the consumed tokens to the trailing trivia - trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, inTrivia.Value.VirtualChars)); + trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.SkippedTextTrivia, inTrivia.Value.VirtualChars)); var fileToken = fileInformationNodeOrToken.Token; if (!fileToken.LeadingTrivia.IsDefaultOrEmpty) @@ -130,7 +130,7 @@ private StackFrameParser(VirtualCharSequence text) trailingTriviaBuilder.AddRange(fileToken.LeadingTrivia); } - trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, fileToken.VirtualChars)); + trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.SkippedTextTrivia, fileToken.VirtualChars)); if (!fileToken.TrailingTrivia.IsDefaultOrEmpty) { @@ -138,21 +138,20 @@ private StackFrameParser(VirtualCharSequence text) } } - var remainingTrivia = _lexer.ScanRemainingTrivia(); + var remainingTrivia = _lexer.TryScanRemainingTrivia(); if (remainingTrivia.HasValue) { trailingTriviaBuilder.Add(remainingTrivia.Value); } - var eolToken = CurrentToken.With(leadingTrivia: trailingTriviaBuilder.ToImmutable()); + var eolToken = CurrentCharAsToken().With(leadingTrivia: trailingTriviaBuilder.ToImmutable()); Debug.Assert(_lexer.Position == _lexer.Text.Length); Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); var root = new StackFrameCompilationUnit(methodDeclaration, fileInformation, eolToken); - return new StackFrameTree( - _lexer.Text, root, ImmutableArray.Empty); + return new(_lexer.Text, root); } /// @@ -202,17 +201,17 @@ private StackFrameParser(VirtualCharSequence text) { Queue<(StackFrameNodeOrToken identifier, StackFrameToken separator)> typeIdentifierNodes = new(); - var currentIdentifier = _lexer.ScanIdentifier(scanAtTrivia: scanAtTrivia, scanWhitespace: true); + var currentIdentifier = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanWhitespace: true); while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { StackFrameToken? arity = null; - if (_lexer.ScanIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) + if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { - arity = _lexer.ScanNumbers(); + arity = _lexer.TryScanNumbers(); if (!arity.HasValue) { - throw new StackFrameParseException(StackFrameKind.NumberToken, CurrentToken); + throw new StackFrameParseException(StackFrameKind.NumberToken, CurrentCharAsToken()); } } @@ -220,16 +219,16 @@ private StackFrameParser(VirtualCharSequence text) ? new StackFrameGenericTypeIdentifier(currentIdentifier.Value, graveAccentToken, arity.Value) : currentIdentifier.Value; - typeIdentifierNodes.Enqueue((identifierNode, CurrentToken)); + typeIdentifierNodes.Enqueue((identifierNode, CurrentCharAsToken())); // Progress the lexer if the current token is a dot token, which // was already added to the list. - if (!_lexer.ScanIfMatch(StackFrameKind.DotToken, out var _)) + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var _)) { break; } - currentIdentifier = _lexer.ScanIdentifier(); + currentIdentifier = _lexer.TryScanIdentifier(); } if (typeIdentifierNodes.Count == 0) @@ -270,15 +269,16 @@ private StackFrameParser(VirtualCharSequence text) /// starting character depending on output source. /// /// ex: MyNamespace.MyClass.MyMethod[T](T t) + /// ex: MyNamespace.MyClass.MyMethod<T<(T t) /// /// Assumes the identifier "MyMethod" has already been parsed, and the type arguments will need to be parsed. - /// Returns null if no type arguments are found or if they are malformed. + /// Returns null if no type arguments are found, and throw a if they are malformed /// private StackFrameTypeArgumentList? TryParseTypeArguments() { - if (!_lexer.ScanIfMatch( - kind => kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken, - out var openToken)) + if (!_lexer.ScanCurrentCharAsTokenIfMatch( + kind => kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken, + out var openToken)) { return null; } @@ -286,14 +286,14 @@ private StackFrameParser(VirtualCharSequence text) var useCloseBracket = openToken.Kind is StackFrameKind.OpenBracketToken; using var _ = ArrayBuilder.GetInstance(out var builder); - var currentIdentifier = _lexer.ScanIdentifier(); + var currentIdentifier = _lexer.TryScanIdentifier(); StackFrameToken closeToken = default; while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { builder.Add(new StackFrameTypeArgumentNode(currentIdentifier.Value)); - if (_lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, out var closeBracket)) + if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseBracketToken, out var closeBracket)) { if (useCloseBracket) { @@ -304,7 +304,7 @@ private StackFrameParser(VirtualCharSequence text) throw new StackFrameParseException(StackFrameKind.GreaterThanToken, closeBracket); } - if (_lexer.ScanIfMatch(StackFrameKind.GreaterThanToken, out var greaterThanToken)) + if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GreaterThanToken, out var greaterThanToken)) { if (useCloseBracket) { @@ -315,8 +315,8 @@ private StackFrameParser(VirtualCharSequence text) break; } - builder.Add(CurrentToken); - currentIdentifier = _lexer.ScanIdentifier(); + builder.Add(CurrentCharAsToken()); + currentIdentifier = _lexer.TryScanIdentifier(); } if (closeToken.IsMissing) @@ -331,31 +331,32 @@ private StackFrameParser(VirtualCharSequence text) /// MyNamespace.MyClass.MyMethod[|(string s1, string s2, int i1)|] /// Takes parameter declarations from method text and parses them into a . /// - /// Returns null in cases where the input is malformed. + /// Returns null in cases where the opening paren is not found. Throws if the + /// opening paren exists but the remaining parameter definitions are malformed. /// private StackFrameParameterList? TryParseMethodParameters() { - if (!_lexer.ScanIfMatch(StackFrameKind.OpenParenToken, scanTrailingWhitespace: true, out var openParen)) + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.OpenParenToken, scanTrailingWhitespace: true, out var openParen)) { return null; } - if (_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) + if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) { return new StackFrameParameterList(openParen, closeParen, SeparatedStackFrameNodeList.Empty); } using var _ = ArrayBuilder.GetInstance(out var builder); builder.Add(ParseParameterNode()); - while (_lexer.ScanIfMatch(StackFrameKind.CommaToken, out var commaToken)) + while (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, out var commaToken)) { builder.Add(commaToken); builder.Add(ParseParameterNode()); } - if (!_lexer.ScanIfMatch(StackFrameKind.CloseParenToken, out closeParen)) + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out closeParen)) { - throw new StackFrameParseException(StackFrameKind.CloseParenToken, CurrentToken); + throw new StackFrameParseException(StackFrameKind.CloseParenToken, CurrentCharAsToken()); } var parameters = new SeparatedStackFrameNodeList(builder.ToImmutable()); @@ -376,7 +377,7 @@ private StackFrameParameterNode ParseParameterNode() throw new StackFrameParseException("Expected type identifier when parsing parameters"); } - if (CurrentToken.Kind == StackFrameKind.OpenBracketToken) + if (CurrentCharAsToken().Kind == StackFrameKind.OpenBracketToken) { var arrayIdentifiers = ParseArrayIdentifiers(); typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier.Value, arrayIdentifiers); @@ -411,20 +412,20 @@ private ImmutableArray ParseArrayIdentifiers() while (true) { - if (!_lexer.ScanIfMatch(StackFrameKind.OpenBracketToken, scanTrailingWhitespace: true, out var openBracket)) + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.OpenBracketToken, scanTrailingWhitespace: true, out var openBracket)) { return builder.ToImmutable(); } commaBuilder.Clear(); - while (_lexer.ScanIfMatch(StackFrameKind.CommaToken, scanTrailingWhitespace: true, out var commaToken)) + while (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, scanTrailingWhitespace: true, out var commaToken)) { commaBuilder.Add(commaToken); } - if (!_lexer.ScanIfMatch(StackFrameKind.CloseBracketToken, scanTrailingWhitespace: true, out var closeBracket)) + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseBracketToken, scanTrailingWhitespace: true, out var closeBracket)) { - throw new StackFrameParseException(StackFrameKind.CloseBracketToken, CurrentToken); + throw new StackFrameParseException(StackFrameKind.CloseBracketToken, CurrentCharAsToken()); } builder.Add(new StackFrameArrayRankSpecifier(openBracket, closeBracket, commaBuilder.ToImmutable())); @@ -436,36 +437,39 @@ private ImmutableArray ParseArrayIdentifiers() /// /// Parses text for a valid file path using valid file characters. It's very possible this includes a path that doesn't exist but /// forms a valid path identifier. + /// + /// Returns tokens in cases where the path is partially parsed but not completed. Returns a + /// if the file information was fully parsed. /// private StackFrameNodeOrToken TryParseFileInformation() { - var path = _lexer.ScanPath(); + var path = _lexer.TryScanPath(); if (!path.HasValue) { return null; } - if (!_lexer.ScanIfMatch(StackFrameKind.ColonToken, out var colonToken)) + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.ColonToken, out var colonToken)) { return path.Value; } - var lineIdentifier = _lexer.ScanLineTrivia(); + var lineIdentifier = _lexer.TryScanLineTrivia(); if (!lineIdentifier.HasValue) { // malformed, we have a ": " with no "line " trivia // add the colonToken as trivia to the valid path and return it - var colonTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, colonToken.VirtualChars); + var colonTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.SkippedTextTrivia, colonToken.VirtualChars); return path.Value.With(trailingTrivia: ImmutableArray.Create(colonTrivia)); } - var numbers = _lexer.ScanNumbers(); + var numbers = _lexer.TryScanNumbers(); if (!numbers.HasValue) { // malformed, we have a ":line " but no following number. // Add the colon and line trivia as trailing trivia var jointTriviaSpan = new TextSpan(colonToken.GetSpan().Start, colonToken.VirtualChars.Length + lineIdentifier.Value.VirtualChars.Length); - var trailingTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.TextTrivia, _lexer.Text.GetSubSequence(jointTriviaSpan)); + var trailingTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.SkippedTextTrivia, _lexer.Text.GetSubSequence(jointTriviaSpan)); return path.Value.With(trailingTrivia: ImmutableArray.Create(trailingTrivia)); } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs index eaf73468d17fa..d2373c94630a9 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs @@ -10,15 +10,13 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { internal class StackFrameTree { - public StackFrameTree(VirtualCharSequence text, StackFrameCompilationUnit root, ImmutableArray embeddedDiagnostics) + public StackFrameTree(VirtualCharSequence text, StackFrameCompilationUnit root) { Text = text; Root = root; - EmbeddedDiagnostics = embeddedDiagnostics; } public VirtualCharSequence Text { get; } public StackFrameCompilationUnit Root { get; } - public ImmutableArray EmbeddedDiagnostics { get; } } } From 9b9541860044f08c49b2fc03ffa26ac61552d7ca Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Mon, 25 Oct 2021 20:27:25 -0700 Subject: [PATCH 19/59] Break up the TryParseIdentifierExpression function. Add tests for invalid parameter names --- .../StackFrame/StackFrameParserTests.cs | 8 ++ .../StackFrame/StackFrameParser.cs | 104 +++++++++--------- 2 files changed, 62 insertions(+), 50 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 6ac8892e692a4..d2c1c6cc7829b 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -188,6 +188,14 @@ public void TestCommaArrayParam() ) ); + [Fact] + public void TestInvalidParameterIdentifier_MemberAccess() + => Verify("at ConsoleApp4.MyClass(string my.string.name)", expectFailure: true); + + [Fact] + public void TestInvalidParameterIdentifier_TypeArity() + => Verify("at ConsoleApp4.MyClass(string s`1)", expectFailure: true); + [Fact] public void TestGenericMethod_Brackets() => Verify( diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 8d90609dc7497..8b5a0e9197827 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -193,75 +193,85 @@ private StackFrameParser(VirtualCharSequence text) /// All of the following are valid identifiers, where "$$" marks the parsing starting point, and "[|" + "|]" mark the endpoints of the parsed identifier including trivia /// * [|$$MyNamespace.MyClass.MyMethod|](string s) /// * MyClass.MyMethod([|$$string |]s) - /// * MyClass.MyMethod(string[| $$s|]) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) /// private StackFrameNodeOrToken? TryParseIdentifierExpression(bool scanAtTrivia = false) { - Queue<(StackFrameNodeOrToken identifier, StackFrameToken separator)> typeIdentifierNodes = new(); - - var currentIdentifier = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanWhitespace: true); - - while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) + var currentIdentifer = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanWhitespace: true); + if (!currentIdentifer.HasValue) { - StackFrameToken? arity = null; - if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) - { - arity = _lexer.TryScanNumbers(); - if (!arity.HasValue) - { - throw new StackFrameParseException(StackFrameKind.NumberToken, CurrentCharAsToken()); - } - } + return null; + } - StackFrameNodeOrToken identifierNode = arity.HasValue - ? new StackFrameGenericTypeIdentifier(currentIdentifier.Value, graveAccentToken, arity.Value) - : currentIdentifier.Value; + var lhs = TryScanGenericTypeIdentifier(currentIdentifer.Value) + ?? (StackFrameNodeOrToken)currentIdentifer.Value; - typeIdentifierNodes.Enqueue((identifierNode, CurrentCharAsToken())); + var memberAccess = TryScanMemberAccessExpression(lhs); + if (memberAccess is null) + { + return lhs; + } - // Progress the lexer if the current token is a dot token, which - // was already added to the list. - if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var _)) + while (true) + { + var newMemberAccess = TryScanMemberAccessExpression(memberAccess); + if (newMemberAccess is null) { - break; + return memberAccess; } - currentIdentifier = _lexer.TryScanIdentifier(); + memberAccess = newMemberAccess; } + } - if (typeIdentifierNodes.Count == 0) + /// + /// Given an existing left hand side node or token, which can either be + /// an or + /// + private StackFrameMemberAccessExpressionNode? TryScanMemberAccessExpression(StackFrameNodeOrToken lhs) + { + Debug.Assert((lhs.IsNode && lhs.Node is StackFrameMemberAccessExpressionNode) || + lhs.Token.Kind == StackFrameKind.IdentifierToken); + + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) { return null; } - var (firstIdentifierNode, firstSeparator) = typeIdentifierNodes.Dequeue(); - if (typeIdentifierNodes.Count == 0) + var identifier = _lexer.TryScanIdentifier(); + if (!identifier.HasValue) { - return firstIdentifierNode; + throw new StackFrameParseException(StackFrameKind.IdentifierToken, CurrentCharAsToken()); } - // Construct the member access expression from the identifiers in the list - var currentSeparator = firstSeparator; + var rhs = TryScanGenericTypeIdentifier(identifier.Value) + ?? (StackFrameNodeOrToken)identifier.Value; - StackFrameMemberAccessExpressionNode? memberAccessExpression = null; + return new StackFrameMemberAccessExpressionNode(lhs, dotToken, rhs); + } - while (typeIdentifierNodes.Count != 0) + /// + /// Given an identifier, attempts to parse the type identifier arity for it. + /// ex: MyNamespace.MyClass`1.MyMethod() + /// ^--------------------- MyClass would be the identifier passed in + /// ^-------------- Grave token + /// ^------------- Arity token of "1" + /// + private StackFrameGenericTypeIdentifier? TryScanGenericTypeIdentifier(StackFrameToken identifierToken) + { + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { - var previousSeparator = currentSeparator; - (var currentIdentifierNode, currentSeparator) = typeIdentifierNodes.Dequeue(); - - var leftHandNode = memberAccessExpression is null - ? firstIdentifierNode - : memberAccessExpression; - - memberAccessExpression = new StackFrameMemberAccessExpressionNode(leftHandNode, previousSeparator, currentIdentifierNode); + return null; } - RoslynDebug.AssertNotNull(memberAccessExpression); + var arity = _lexer.TryScanNumbers(); + if (!arity.HasValue) + { + throw new StackFrameParseException(StackFrameKind.NumberToken, CurrentCharAsToken()); + } - return memberAccessExpression; + return new StackFrameGenericTypeIdentifier(identifierToken, graveAccentToken, arity.Value); } /// @@ -383,19 +393,13 @@ private StackFrameParameterNode ParseParameterNode() typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier.Value, arrayIdentifiers); } - var identifier = TryParseIdentifierExpression(); + var identifier = _lexer.TryScanIdentifier(scanWhitespace: true); if (!identifier.HasValue) { throw new StackFrameParseException("Expected a parameter identifier"); } - // Parameter identifiers should only be tokens - if (identifier.Value.IsNode) - { - throw new StackFrameParseException(StackFrameKind.IdentifierToken, identifier.Value); - } - - return new StackFrameParameterNode(typeIdentifier.Value, identifier.Value.Token); + return new StackFrameParameterNode(typeIdentifier.Value, identifier.Value); } /// From 9f28a297ac93a1b92fff9e2cb6eb60111c29dd4b Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 26 Oct 2021 11:32:16 -0700 Subject: [PATCH 20/59] Update the lexer to handle line numbers and paths instead of the parser doing it. --- .../StackFrame/StackFrameParserTests.cs | 46 ++--------- .../StackFrame/StackFrameLexer.cs | 38 +++++++++- .../StackFrame/StackFrameNodeDefinitions.cs | 13 ++-- .../StackFrame/StackFrameParser.cs | 76 ++++--------------- 4 files changed, 67 insertions(+), 106 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index d2c1c6cc7829b..b983cb09373e5 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -285,27 +285,15 @@ public void TestFileInformation() [Fact] public void TestFileInformation_PartialPath() - => Verify( - @"M.M() in C:\folder\m.cs:line", - methodDeclaration: MethodDeclaration( - MemberAccessExpression("M.M"), - argumentList: EmptyParams), - - eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":", "line") - ) - ); + => Verify(@"M.M() in C:\folder\m.cs:line", expectFailure: true); [Fact] public void TestFileInformation_PartialPath2() - => Verify( - @"M.M() in C:\folder\m.cs:", - methodDeclaration: MethodDeclaration( - MemberAccessExpression("M.M"), - argumentList: EmptyParams), + => Verify(@"M.M() in C:\folder\m.cs:", expectFailure: true); - eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":") - ) - ); + [Fact] + public void TestFileInformation_PartialPath3() + => Verify(@"M.M() in C:\folder\m.cs:[trailingtrivia]", expectFailure: true); [Theory] [InlineData(@"C:\folder\m.cs", 1)] @@ -337,32 +325,14 @@ public void TestFileInformation_TrailingTrivia() fileInformation: FileInformation( Path(@"C:\folder\m.cs"), ColonToken, - Line(1)), + Line(1).With(trailingTrivia: CreateTriviaArray("[trailingtrivia]"))), - eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray("[trailingtrivia]")) - ); - - [Fact] - public void TestFileInformation_TrailingTrivia2() - => Verify( - @"M.M() in C:\folder\m.cs:[trailingtrivia]", - methodDeclaration: MethodDeclaration( - MemberAccessExpression("M.M"), - argumentList: EmptyParams), - - eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\folder\m.cs", ":", "[trailingtrivia]")) + eolTokenOpt: EOLToken ); [Fact] public void TestFileInformation_InvalidDirectory() - => Verify( - @"M.M() in C:\<\m.cs", - methodDeclaration: MethodDeclaration( - MemberAccessExpression("M.M"), - argumentList: EmptyParams), - - eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" in ", @"C:\", @"<\m.cs")) - ); + => Verify(@"M.M() in <\m.cs", expectFailure: true); [Theory] [InlineData("")] diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index fb263461ac77f..09d862ec2cf67 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -78,7 +78,7 @@ public VirtualCharSequence GetSubSequence(int start, int end) return CreateToken(StackFrameKind.IdentifierToken, CreateTrivia(atTrivia, whitespaceTrivia), GetSubSequenceToCurrentPos(startPosition)); } - internal StackFrameTrivia? TryScanWhiteSpace() + private StackFrameTrivia? TryScanWhiteSpace() { var startPosition = Position; @@ -243,11 +243,17 @@ private static bool IsNumber(VirtualChar ch) } /// - /// Attempts to parse a path following https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names + /// Attempts to parse and a path following https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names /// Uses as a tool to determine if the path is correct for returning. /// internal StackFrameToken? TryScanPath() { + var inTrivia = TryScanInTrivia(); + if (!inTrivia.HasValue) + { + return null; + } + var startPosition = Position; while (Position < Text.Length) @@ -282,11 +288,37 @@ private static bool IsNumber(VirtualChar ch) } if (startPosition == Position) + { + return CreateToken(StackFrameKind.None, ImmutableArray.Create(inTrivia.Value), VirtualCharSequence.Empty); + } + + return CreateToken(StackFrameKind.PathToken, ImmutableArray.Create(inTrivia.Value), GetSubSequenceToCurrentPos(startPosition)); + } + + /// + /// Returns a number token with the and remainging + /// attached to it. If no numbers are found, returns a token with the trivia that was scanned. + /// + /// + internal StackFrameToken? TryScanLineNumber() + { + var lineTrivia = TryScanLineTrivia(); + if (!lineTrivia.HasValue) { return null; } - return CreateToken(StackFrameKind.PathToken, GetSubSequenceToCurrentPos(startPosition)); + var numberToken = TryScanNumbers(); + if (!numberToken.HasValue) + { + return CreateToken(StackFrameKind.None, ImmutableArray.Create(lineTrivia.Value), VirtualCharSequence.Empty); + } + + var remainingTrivia = TryScanRemainingTrivia(); + + return numberToken.Value.With( + leadingTrivia: ImmutableArray.Create(lineTrivia.Value), + trailingTrivia: remainingTrivia.HasValue ? ImmutableArray.Create(remainingTrivia.Value) : ImmutableArray.Empty); } internal StackFrameTrivia? TryScanLineTrivia() diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index 886c1ae2f5f94..ba38f154586a0 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -417,11 +417,14 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameFileInformationNode : StackFrameNode { public readonly StackFrameToken Path; - public readonly StackFrameToken Colon; - public readonly StackFrameToken Line; + public readonly StackFrameToken? Colon; + public readonly StackFrameToken? Line; - public StackFrameFileInformationNode(StackFrameToken path, StackFrameToken colon, StackFrameToken line) : base(StackFrameKind.FileInformation) + public StackFrameFileInformationNode(StackFrameToken path, StackFrameToken? colon, StackFrameToken? line) : base(StackFrameKind.FileInformation) { + Debug.Assert(colon.HasValue == line.HasValue); + Debug.Assert(!line.HasValue || line.Value.Kind == StackFrameKind.NumberToken); + Path = path; Colon = colon; Line = line; @@ -436,8 +439,8 @@ internal override StackFrameNodeOrToken ChildAt(int index) => index switch { 0 => Path, - 1 => Colon, - 2 => Line, + 1 => Colon.HasValue ? Colon.Value : null, + 2 => Line.HasValue ? Line.Value : null, _ => throw new InvalidOperationException() }; diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 8b5a0e9197827..c6879379f9003 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -105,46 +105,10 @@ private StackFrameParser(VirtualCharSequence text) return null; } - var inTrivia = _lexer.TryScanInTrivia(); - var fileInformationNodeOrToken = inTrivia.HasValue - ? TryParseFileInformation() - : null; - - using var _ = ArrayBuilder.GetInstance(out var trailingTriviaBuilder); - - var fileInformation = fileInformationNodeOrToken.Node as StackFrameFileInformationNode; - - if (fileInformation is not null) - { - Debug.Assert(inTrivia.HasValue); - fileInformation = fileInformation.WithLeadingTrivia(inTrivia.Value); - } - else if (inTrivia.HasValue) - { - // If the file path wasn't valid make sure to add the consumed tokens to the trailing trivia - trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.SkippedTextTrivia, inTrivia.Value.VirtualChars)); - - var fileToken = fileInformationNodeOrToken.Token; - if (!fileToken.LeadingTrivia.IsDefaultOrEmpty) - { - trailingTriviaBuilder.AddRange(fileToken.LeadingTrivia); - } - - trailingTriviaBuilder.Add(StackFrameLexer.CreateTrivia(StackFrameKind.SkippedTextTrivia, fileToken.VirtualChars)); - - if (!fileToken.TrailingTrivia.IsDefaultOrEmpty) - { - trailingTriviaBuilder.AddRange(fileToken.TrailingTrivia); - } - } - + var fileInformation = TryParseFileInformation(); var remainingTrivia = _lexer.TryScanRemainingTrivia(); - if (remainingTrivia.HasValue) - { - trailingTriviaBuilder.Add(remainingTrivia.Value); - } - var eolToken = CurrentCharAsToken().With(leadingTrivia: trailingTriviaBuilder.ToImmutable()); + var eolToken = CurrentCharAsToken().With(leadingTrivia: remainingTrivia.HasValue ? ImmutableArray.Create(remainingTrivia.Value) : ImmutableArray.Empty); Debug.Assert(_lexer.Position == _lexer.Text.Length); Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); @@ -442,10 +406,10 @@ private ImmutableArray ParseArrayIdentifiers() /// Parses text for a valid file path using valid file characters. It's very possible this includes a path that doesn't exist but /// forms a valid path identifier. /// - /// Returns tokens in cases where the path is partially parsed but not completed. Returns a - /// if the file information was fully parsed. + /// Can return if only a path is available but not line numbers, but throws if the value after the path is a colon as the expectation + /// is that line number should follow. /// - private StackFrameNodeOrToken TryParseFileInformation() + private StackFrameFileInformationNode? TryParseFileInformation() { var path = _lexer.TryScanPath(); if (!path.HasValue) @@ -453,34 +417,26 @@ private StackFrameNodeOrToken TryParseFileInformation() return null; } - if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.ColonToken, out var colonToken)) + if (path.Value.Kind != StackFrameKind.PathToken) { - return path.Value; + throw new StackFrameParseException("'In ' trivia was present "); } - var lineIdentifier = _lexer.TryScanLineTrivia(); - if (!lineIdentifier.HasValue) + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.ColonToken, out var colonToken)) { - // malformed, we have a ": " with no "line " trivia - // add the colonToken as trivia to the valid path and return it - var colonTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.SkippedTextTrivia, colonToken.VirtualChars); - return path.Value.With(trailingTrivia: ImmutableArray.Create(colonTrivia)); + return new(path.Value, null, null); } - var numbers = _lexer.TryScanNumbers(); - if (!numbers.HasValue) + var lineNumber = _lexer.TryScanLineNumber(); + + // TryScanLineNumber can return a token that isn't a number, in which case we want + // to bail in error and consider this malformed. + if (!lineNumber.HasValue || lineNumber.Value.Kind != StackFrameKind.NumberToken) { - // malformed, we have a ":line " but no following number. - // Add the colon and line trivia as trailing trivia - var jointTriviaSpan = new TextSpan(colonToken.GetSpan().Start, colonToken.VirtualChars.Length + lineIdentifier.Value.VirtualChars.Length); - var trailingTrivia = StackFrameLexer.CreateTrivia(StackFrameKind.SkippedTextTrivia, _lexer.Text.GetSubSequence(jointTriviaSpan)); - return path.Value.With(trailingTrivia: ImmutableArray.Create(trailingTrivia)); + throw new StackFrameParseException("Expected line number to exist after colon token"); } - return new StackFrameFileInformationNode( - path.Value, - colonToken, - numbers.Value.With(leadingTrivia: ImmutableArray.Create(lineIdentifier.Value))); + return new(path.Value, colonToken, lineNumber.Value); } } } From b3bdfa1a089d0a7e7ae699ae2b14d058b2cd5870 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 26 Oct 2021 14:51:38 -0700 Subject: [PATCH 21/59] PR feedback changes --- .../StackFrameParserTests.Utilities.cs | 53 ++--- .../StackFrame/StackFrameParserTests.cs | 15 ++ ...nerators.cs => StackFrameSyntaxFactory.cs} | 88 +++---- .../StackFrame/IStackFrameNodeVisitor.cs | 4 +- .../StackFrame/StackFrameKind.cs | 2 +- .../StackFrame/StackFrameLexer.cs | 217 +++++++++--------- .../StackFrame/StackFrameNodeDefinitions.cs | 14 -- .../StackFrame/StackFrameParser.cs | 4 +- 8 files changed, 182 insertions(+), 215 deletions(-) rename src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/{StackFrameParserTests.Generators.cs => StackFrameSyntaxFactory.cs} (56%) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index be2aabe4ad804..6f00ce3f5f037 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -12,6 +12,7 @@ using Roslyn.Test.Utilities; using Xunit; using System; +using static Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame.StackFrameSyntaxFactory; namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame { @@ -21,7 +22,12 @@ namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame public partial class StackFrameParserTests { - private static void Verify(string input, StackFrameMethodDeclarationNode? methodDeclaration = null, bool expectFailure = false, StackFrameFileInformationNode? fileInformation = null, StackFrameToken? eolTokenOpt = null) + private static void Verify( + string input, + StackFrameMethodDeclarationNode? methodDeclaration = null, + bool expectFailure = false, + StackFrameFileInformationNode? fileInformation = null, + StackFrameToken? eolTokenOpt = null) { var tree = StackFrameParser.TryParse(input); if (expectFailure) @@ -57,6 +63,7 @@ private static void Verify(string input, StackFrameMethodDeclarationNode? method AssertEqual(eolToken, tree.Root.EndOfLineToken); } + private static void AssertEqual(StackFrameNodeOrToken expected, StackFrameNodeOrToken actual) { Assert.Equal(expected.IsNode, actual.IsNode); @@ -163,7 +170,7 @@ private static void VerifyCharacterSpans(string originalText, StackFrameTree tre var index = 0; List enumeratedParsedCharacters = new(); - foreach (var charSeq in EnumerateTree(tree)) + foreach (var charSeq in Enumerate(tree.Root)) { foreach (var ch in charSeq) { @@ -214,42 +221,18 @@ static void PrintString(string s, int start, int end, StringBuilder sb) } } - private static IEnumerable EnumerateTree(StackFrameTree tree) - { - var root = tree.Root; - - var methodDeclaration = root.MethodDeclaration; - foreach (var seq in Enumerate(methodDeclaration)) - { - yield return seq; - } - - if (root.FileInformationExpression is not null) - { - foreach (var seq in Enumerate(root.FileInformationExpression)) - { - yield return seq; - } - } - - foreach (var charSequence in Enumerate(root.EndOfLineToken)) - { - yield return charSequence; - } - } - private static IEnumerable Enumerate(StackFrameNode node) { foreach (var nodeOrToken in node) { if (nodeOrToken.IsNode) { - foreach (var charSequence in Enumerate(nodeOrToken.Node).ToArray()) + foreach (var charSequence in Enumerate(nodeOrToken.Node)) { yield return charSequence; } } - else if (!nodeOrToken.Token.VirtualChars.IsDefaultOrEmpty) + else { foreach (var charSequence in Enumerate(nodeOrToken.Token)) { @@ -261,22 +244,16 @@ private static IEnumerable Enumerate(StackFrameNode node) private static IEnumerable Enumerate(StackFrameToken token) { - if (!token.LeadingTrivia.IsDefault) + foreach (var trivia in token.LeadingTrivia) { - foreach (var trivia in token.LeadingTrivia) - { - yield return trivia.VirtualChars; - } + yield return trivia.VirtualChars; } yield return token.VirtualChars; - if (!token.TrailingTrivia.IsDefault) + foreach (var trivia in token.TrailingTrivia) { - foreach (var trivia in token.TrailingTrivia) - { - yield return trivia.VirtualChars; - } + yield return trivia.VirtualChars; } } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index b983cb09373e5..9e48864a11b5e 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using Xunit; +using static Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame.StackFrameSyntaxFactory; namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame { @@ -356,5 +357,19 @@ public void TestFileInformation_InvalidDirectory() [InlineData(@"at M.N`9N.P()")] // Invalid character after arity public void TestInvalidInputs(string input) => Verify(input, expectFailure: true); + + [Theory] + [InlineData("at ")] + [InlineData(" in ")] + public void TestKeywordsAsIdentifiers(string keyword) + => Verify(@$"MyNamespace.MyType.MyMethod[{keyword}]({keyword} {keyword})", + methodDeclaration: MethodDeclaration( + MemberAccessExpression("MyNamespace.MyType.MyMethod"), + typeArguments: TypeArgumentList(TypeArgument(keyword.Trim())), + argumentList: ParameterList( + OpenParenToken, + CloseParenToken, + Parameter(Identifier(keyword), Identifier(keyword)))) + ); } } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs similarity index 56% rename from src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs rename to src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 5fb30f603b73e..4d13e639d4e5b 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Generators.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -16,9 +16,9 @@ namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame using StackFrameTrivia = EmbeddedSyntaxTrivia; using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; - public partial class StackFrameParserTests + internal static class StackFrameSyntaxFactory { - private static StackFrameToken CreateToken(StackFrameKind kind, string s, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) + public static StackFrameToken CreateToken(StackFrameKind kind, string s, ImmutableArray leadingTrivia = default, ImmutableArray trailingTrivia = default) => new( kind, leadingTrivia.IsDefaultOrEmpty ? ImmutableArray.Empty : leadingTrivia, @@ -27,40 +27,40 @@ private static StackFrameToken CreateToken(StackFrameKind kind, string s, Immuta ImmutableArray.Empty, value: null!); - private static StackFrameTrivia CreateTrivia(StackFrameKind kind, string s) - => new(kind, CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, s), ImmutableArray.Empty); + public static StackFrameTrivia CreateTrivia(StackFrameKind kind, string text) + => new(kind, CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, text), ImmutableArray.Empty); - private static ImmutableArray CreateTriviaArray(params StackFrameTrivia[] trivia) + public static ImmutableArray CreateTriviaArray(params StackFrameTrivia[] trivia) => ImmutableArray.Create(trivia); - private static ImmutableArray CreateTriviaArray(StackFrameTrivia? trivia) + public static ImmutableArray CreateTriviaArray(StackFrameTrivia? trivia) => trivia.HasValue ? ImmutableArray.Create(trivia.Value) : ImmutableArray.Empty; - private static ImmutableArray CreateTriviaArray(params string[] strings) + public static ImmutableArray CreateTriviaArray(params string[] strings) => strings.Select(s => CreateTrivia(StackFrameKind.SkippedTextTrivia, s)).ToImmutableArray(); - private static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); - private static readonly StackFrameToken CommaToken = CreateToken(StackFrameKind.CommaToken, ","); - private static readonly StackFrameToken OpenParenToken = CreateToken(StackFrameKind.OpenParenToken, "("); - private static readonly StackFrameToken CloseParenToken = CreateToken(StackFrameKind.CloseParenToken, ")"); - private static readonly StackFrameToken OpenBracketToken = CreateToken(StackFrameKind.OpenBracketToken, "["); - private static readonly StackFrameToken CloseBracketToken = CreateToken(StackFrameKind.CloseBracketToken, "]"); - private static readonly StackFrameToken LessThanToken = CreateToken(StackFrameKind.LessThanToken, "<"); - private static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); - private static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); - private static readonly StackFrameToken EOLToken = CreateToken(StackFrameKind.EndOfLine, ""); - private static readonly StackFrameToken ColonToken = CreateToken(StackFrameKind.ColonToken, ":"); - - private static readonly StackFrameTrivia AtTrivia = CreateTrivia(StackFrameKind.AtTrivia, "at "); - private static readonly StackFrameTrivia LineTrivia = CreateTrivia(StackFrameKind.LineTrivia, "line "); - private static readonly StackFrameTrivia InTrivia = CreateTrivia(StackFrameKind.InTrivia, " in "); - - private static readonly StackFrameParameterList EmptyParams = ParameterList(OpenParenToken, CloseParenToken); - - private static StackFrameParameterNode Parameter(StackFrameNodeOrToken type, StackFrameToken identifier) + public static readonly StackFrameToken DotToken = CreateToken(StackFrameKind.DotToken, "."); + public static readonly StackFrameToken CommaToken = CreateToken(StackFrameKind.CommaToken, ","); + public static readonly StackFrameToken OpenParenToken = CreateToken(StackFrameKind.OpenParenToken, "("); + public static readonly StackFrameToken CloseParenToken = CreateToken(StackFrameKind.CloseParenToken, ")"); + public static readonly StackFrameToken OpenBracketToken = CreateToken(StackFrameKind.OpenBracketToken, "["); + public static readonly StackFrameToken CloseBracketToken = CreateToken(StackFrameKind.CloseBracketToken, "]"); + public static readonly StackFrameToken LessThanToken = CreateToken(StackFrameKind.LessThanToken, "<"); + public static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); + public static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); + public static readonly StackFrameToken EOLToken = CreateToken(StackFrameKind.EndOfLine, ""); + public static readonly StackFrameToken ColonToken = CreateToken(StackFrameKind.ColonToken, ":"); + + public static readonly StackFrameTrivia AtTrivia = CreateTrivia(StackFrameKind.AtTrivia, "at "); + public static readonly StackFrameTrivia LineTrivia = CreateTrivia(StackFrameKind.LineTrivia, "line "); + public static readonly StackFrameTrivia InTrivia = CreateTrivia(StackFrameKind.InTrivia, " in "); + + public static readonly StackFrameParameterList EmptyParams = ParameterList(OpenParenToken, CloseParenToken); + + public static StackFrameParameterNode Parameter(StackFrameNodeOrToken type, StackFrameToken identifier) => new(type, identifier); - private static StackFrameParameterList ParameterList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameParameterNode[] parameters) + public static StackFrameParameterList ParameterList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameParameterNode[] parameters) { var separatedList = parameters.Length == 0 ? SeparatedStackFrameNodeList.Empty @@ -83,7 +83,7 @@ static ImmutableArray CommaSeparateList(StackFrameParamet } } - private static StackFrameMethodDeclarationNode MethodDeclaration( + public static StackFrameMethodDeclarationNode MethodDeclaration( StackFrameMemberAccessExpressionNode memberAccessExpression, StackFrameTypeArgumentList? typeArguments = null, StackFrameParameterList? argumentList = null) @@ -91,10 +91,10 @@ private static StackFrameMethodDeclarationNode MethodDeclaration( return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, argumentList ?? ParameterList(OpenParenToken, CloseParenToken)); } - private static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + public static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) => MemberAccessExpression(s, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); - private static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) + public static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) { StackFrameNodeOrToken? current = null; var identifiers = s.Split('.'); @@ -124,34 +124,34 @@ private static StackFrameMemberAccessExpressionNode MemberAccessExpression(strin return (StackFrameMemberAccessExpressionNode)node; } - private static StackFrameTrivia SpaceTrivia(int count = 1) + public static StackFrameTrivia SpaceTrivia(int count = 1) => CreateTrivia(StackFrameKind.WhitespaceTrivia, new string(' ', count)); - private static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameNodeOrToken left, StackFrameNodeOrToken right) + public static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameNodeOrToken left, StackFrameNodeOrToken right) => new(left, DotToken, right); - private static StackFrameToken Identifier(string identifierName) + public static StackFrameToken Identifier(string identifierName) => Identifier(identifierName, leadingTrivia: null, trailingTrivia: null); - private static StackFrameToken Identifier(string identifierName, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + public static StackFrameToken Identifier(string identifierName, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) => Identifier(identifierName, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); - private static StackFrameToken Identifier(string identifierName, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) + public static StackFrameToken Identifier(string identifierName, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) => CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia); - private static StackFrameArrayRankSpecifier ArrayRankSpecifier(int commaCount = 0, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + public static StackFrameArrayRankSpecifier ArrayRankSpecifier(int commaCount = 0, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) => new(OpenBracketToken.With(leadingTrivia: CreateTriviaArray(leadingTrivia)), CloseBracketToken.With(trailingTrivia: CreateTriviaArray(trailingTrivia)), Enumerable.Repeat(CommaToken, commaCount).ToImmutableArray()); - private static StackFrameArrayTypeExpression ArrayExpression(StackFrameNodeOrToken identifier, params StackFrameArrayRankSpecifier[] arrayTokens) + public static StackFrameArrayTypeExpression ArrayExpression(StackFrameNodeOrToken identifier, params StackFrameArrayRankSpecifier[] arrayTokens) => new(identifier, arrayTokens.ToImmutableArray()); - private static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) + public static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.NumberToken, arity.ToString())); - private static StackFrameTypeArgumentList TypeArgumentList(params StackFrameTypeArgumentNode[] typeArguments) + public static StackFrameTypeArgumentList TypeArgumentList(params StackFrameTypeArgumentNode[] typeArguments) => TypeArgumentList(useBrackets: true, typeArguments); - private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgumentNode[] typeArguments) + public static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgumentNode[] typeArguments) { using var _ = PooledObjects.ArrayBuilder.GetInstance(out var builder); var openToken = useBrackets ? OpenBracketToken : LessThanToken; @@ -177,16 +177,16 @@ private static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, par return new(openToken, typeArgumentsList, closeToken); } - private static StackFrameTypeArgumentNode TypeArgument(string identifier) + public static StackFrameTypeArgumentNode TypeArgument(string identifier) => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); - private static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken colon, StackFrameToken line) + public static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken colon, StackFrameToken line) => new(path.With(leadingTrivia: CreateTriviaArray(InTrivia)), colon, line); - private static StackFrameToken Path(string path) + public static StackFrameToken Path(string path) => CreateToken(StackFrameKind.PathToken, path); - private static StackFrameToken Line(int lineNumber) + public static StackFrameToken Line(int lineNumber) => CreateToken(StackFrameKind.NumberToken, lineNumber.ToString(), leadingTrivia: ImmutableArray.Create(LineTrivia)); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index 7ad72234d7d78..e23d9da97a459 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -6,6 +6,7 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { internal interface IStackFrameNodeVisitor { + void Visit(StackFrameCompilationUnit node); void Visit(StackFrameMethodDeclarationNode node); void Visit(StackFrameMemberAccessExpressionNode node); void Visit(StackFrameTypeArgumentList node); @@ -15,7 +16,6 @@ internal interface IStackFrameNodeVisitor void Visit(StackFrameArrayRankSpecifier node); void Visit(StackFrameFileInformationNode node); void Visit(StackFrameArrayTypeExpression node); - void Visit(StackFrameParameterNode stackFrameParameterNode); - void Visit(StackFrameCompilationUnit stackFrameCompilationUnit); + void Visit(StackFrameParameterNode node); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index f56b544394cac..361e34f413abd 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -11,6 +11,7 @@ internal enum StackFrameKind None = 0, // Nodes + CompilationUnit, MethodDeclaration, MemberAccess, ArrayTypeExpression, @@ -21,7 +22,6 @@ internal enum StackFrameKind ParameterList, ArrayExpression, FileInformation, - CompilationUnit, // Tokens EndOfLine, diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 09d862ec2cf67..d1c322abfc317 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -52,11 +52,14 @@ public VirtualCharSequence GetSubSequence(int start, int end) return CreateTrivia(StackFrameKind.SkippedTextTrivia, GetSubSequenceToCurrentPos(start)); } - public StackFrameToken? TryScanIdentifier(bool scanAtTrivia = false, bool scanWhitespace = false) + public StackFrameToken? TryScanIdentifier() + => TryScanIdentifier(scanAtTrivia: false, scanWhitespace: false); + + public StackFrameToken? TryScanIdentifier(bool scanAtTrivia, bool scanWhitespace) { var originalPosition = Position; var atTrivia = scanAtTrivia ? TryScanAtTrivia() : null; - var whitespaceTrivia = scanWhitespace ? TryScanWhiteSpace() : null; + var leadingWhitespace = scanWhitespace ? TryScanWhiteSpace() : null; var startPosition = Position; var ch = CurrentChar; @@ -75,24 +78,13 @@ public VirtualCharSequence GetSubSequence(int start, int end) ch = CurrentChar; } - return CreateToken(StackFrameKind.IdentifierToken, CreateTrivia(atTrivia, whitespaceTrivia), GetSubSequenceToCurrentPos(startPosition)); - } - - private StackFrameTrivia? TryScanWhiteSpace() - { - var startPosition = Position; - - while (IsBlank(CurrentChar)) - { - Position++; - } + var trailingWhitespace = scanWhitespace ? TryScanWhiteSpace() : null; - if (Position == startPosition) - { - return null; - } - - return CreateTrivia(StackFrameKind.WhitespaceTrivia, GetSubSequenceToCurrentPos(startPosition)); + return CreateToken( + StackFrameKind.IdentifierToken, + leadingTrivia: CreateTrivia(atTrivia, leadingWhitespace), + GetSubSequenceToCurrentPos(startPosition), + trailingTrivia: CreateTrivia(trailingWhitespace)); } public StackFrameToken CurrentCharAsToken() @@ -106,23 +98,6 @@ public StackFrameToken CurrentCharAsToken() return CreateToken(GetKind(ch), Text.GetSubSequence(new TextSpan(Position, 1))); } - public bool IsStringAtPosition(string val) - => TextAt(Position, val); - - private bool TextAt(int position, string val) - { - for (var i = 0; i < val.Length; i++) - { - if (position + i >= Text.Length || - Text[position + i] != val[i]) - { - return false; - } - } - - return true; - } - /// /// Progress the position by one if the current character /// matches the kind. @@ -130,7 +105,7 @@ private bool TextAt(int position, string val) /// /// if the position was incremented /// - internal bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, out StackFrameToken token) + public bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, out StackFrameToken token) => ScanCurrentCharAsTokenIfMatch(kind, scanTrailingWhitespace: false, out token); /// @@ -140,7 +115,7 @@ internal bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, out StackFrameT /// /// if the position was incremented /// - internal bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, bool scanTrailingWhitespace, out StackFrameToken token) + public bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, bool scanTrailingWhitespace, out StackFrameToken token) { if (GetKind(CurrentChar) == kind) { @@ -166,7 +141,7 @@ internal bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, bool scanTraili /// /// if the position was incremented /// - internal bool ScanCurrentCharAsTokenIfMatch(Func matchFn, out StackFrameToken token) + public bool ScanCurrentCharAsTokenIfMatch(Func matchFn, out StackFrameToken token) { if (matchFn(GetKind(CurrentChar))) { @@ -179,74 +154,22 @@ internal bool ScanCurrentCharAsTokenIfMatch(Func matchFn, return false; } - private static StackFrameKind GetKind(VirtualChar ch) - => ch.Value switch - { - '\n' => StackFrameKind.EndOfLine, - '&' => StackFrameKind.AmpersandToken, - '[' => StackFrameKind.OpenBracketToken, - ']' => StackFrameKind.CloseBracketToken, - '(' => StackFrameKind.OpenParenToken, - ')' => StackFrameKind.CloseParenToken, - '.' => StackFrameKind.DotToken, - '+' => StackFrameKind.PlusToken, - ',' => StackFrameKind.CommaToken, - ':' => StackFrameKind.ColonToken, - '=' => StackFrameKind.EqualsToken, - '>' => StackFrameKind.GreaterThanToken, - '<' => StackFrameKind.LessThanToken, - '-' => StackFrameKind.MinusToken, - '\'' => StackFrameKind.SingleQuoteToken, - '`' => StackFrameKind.GraveAccentToken, - '\\' => StackFrameKind.BackslashToken, - '/' => StackFrameKind.ForwardSlashToken, - _ => IsBlank(ch) - ? StackFrameKind.WhitespaceTrivia - : IsNumber(ch) - ? StackFrameKind.NumberToken - : StackFrameKind.SkippedTextTrivia - }; - - private static bool IsNumber(VirtualChar ch) - => ch.Value is >= '0' and <= '9'; - public StackFrameTrivia? TryScanAtTrivia() - { // TODO: Handle multiple languages? Right now we're going to only parse english - const string AtString = "at "; - - if (IsStringAtPosition(AtString)) - { - var start = Position; - Position += AtString.Length; - - return CreateTrivia(StackFrameKind.AtTrivia, GetSubSequenceToCurrentPos(start)); - } - - return null; - } - + => TryScanStringTrivia("at ", StackFrameKind.AtTrivia); public StackFrameTrivia? TryScanInTrivia() - { // TODO: Handle multiple languages? Right now we're going to only parse english - const string InString = " in "; - - if (IsStringAtPosition(InString)) - { - var start = Position; - Position += InString.Length; - - return CreateTrivia(StackFrameKind.InTrivia, GetSubSequenceToCurrentPos(start)); - } + => TryScanStringTrivia(" in ", StackFrameKind.InTrivia); - return null; - } + public StackFrameTrivia? TryScanLineTrivia() + // TODO: Handle multiple languages? Right now we're going to only parse english + => TryScanStringTrivia("line ", StackFrameKind.LineTrivia); /// /// Attempts to parse and a path following https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names /// Uses as a tool to determine if the path is correct for returning. /// - internal StackFrameToken? TryScanPath() + public StackFrameToken? TryScanPath() { var inTrivia = TryScanInTrivia(); if (!inTrivia.HasValue) @@ -300,7 +223,7 @@ private static bool IsNumber(VirtualChar ch) /// attached to it. If no numbers are found, returns a token with the trivia that was scanned. /// /// - internal StackFrameToken? TryScanLineNumber() + public StackFrameToken? TryScanLineNumber() { var lineTrivia = TryScanLineTrivia(); if (!lineTrivia.HasValue) @@ -321,22 +244,7 @@ private static bool IsNumber(VirtualChar ch) trailingTrivia: remainingTrivia.HasValue ? ImmutableArray.Create(remainingTrivia.Value) : ImmutableArray.Empty); } - internal StackFrameTrivia? TryScanLineTrivia() - { - // TODO: Handle multiple languages? Right now we're going to only parse english - const string LineString = "line "; - if (IsStringAtPosition(LineString)) - { - var start = Position; - Position += LineString.Length; - - return CreateTrivia(StackFrameKind.LineTrivia, GetSubSequenceToCurrentPos(start)); - } - - return null; - } - - internal StackFrameToken? TryScanNumbers() + public StackFrameToken? TryScanNumbers() { var start = Position; while (IsNumber(CurrentChar)) @@ -374,6 +282,9 @@ public static StackFrameToken CreateToken(StackFrameKind kind, VirtualCharSequen public static StackFrameToken CreateToken(StackFrameKind kind, ImmutableArray leadingTrivia, VirtualCharSequence virtualChars) => new(kind, leadingTrivia, virtualChars, ImmutableArray.Empty, ImmutableArray.Empty, value: null!); + public static StackFrameToken CreateToken(StackFrameKind kind, ImmutableArray leadingTrivia, VirtualCharSequence virtualChars, ImmutableArray trailingTrivia) + => new(kind, leadingTrivia, virtualChars, trailingTrivia, ImmutableArray.Empty, value: null!); + public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars) => CreateTrivia(kind, virtualChars, ImmutableArray.Empty); @@ -397,5 +308,83 @@ public static ImmutableArray CreateTrivia(params StackFrameTri return builder.ToImmutable(); } + + private bool IsStringAtPosition(string val) + => IsAtStartOfText(Position, val); + + private bool IsAtStartOfText(int position, string val) + { + for (var i = 0; i < val.Length; i++) + { + if (position + i >= Text.Length || + Text[position + i] != val[i]) + { + return false; + } + } + + return true; + } + + private StackFrameTrivia? TryScanStringTrivia(string valueToLookFor, StackFrameKind triviaKind) + { + if (IsStringAtPosition(valueToLookFor)) + { + var start = Position; + Position += valueToLookFor.Length; + + return CreateTrivia(triviaKind, GetSubSequenceToCurrentPos(start)); + } + + return null; + } + + private StackFrameTrivia? TryScanWhiteSpace() + { + var startPosition = Position; + + while (IsBlank(CurrentChar)) + { + Position++; + } + + if (Position == startPosition) + { + return null; + } + + return CreateTrivia(StackFrameKind.WhitespaceTrivia, GetSubSequenceToCurrentPos(startPosition)); + } + + private static StackFrameKind GetKind(VirtualChar ch) + => ch.Value switch + { + '\n' => StackFrameKind.EndOfLine, + '&' => StackFrameKind.AmpersandToken, + '[' => StackFrameKind.OpenBracketToken, + ']' => StackFrameKind.CloseBracketToken, + '(' => StackFrameKind.OpenParenToken, + ')' => StackFrameKind.CloseParenToken, + '.' => StackFrameKind.DotToken, + '+' => StackFrameKind.PlusToken, + ',' => StackFrameKind.CommaToken, + ':' => StackFrameKind.ColonToken, + '=' => StackFrameKind.EqualsToken, + '>' => StackFrameKind.GreaterThanToken, + '<' => StackFrameKind.LessThanToken, + '-' => StackFrameKind.MinusToken, + '\'' => StackFrameKind.SingleQuoteToken, + '`' => StackFrameKind.GraveAccentToken, + '\\' => StackFrameKind.BackslashToken, + '/' => StackFrameKind.ForwardSlashToken, + _ => IsBlank(ch) + ? StackFrameKind.WhitespaceTrivia + : IsNumber(ch) + ? StackFrameKind.NumberToken + : StackFrameKind.SkippedTextTrivia + }; + + private static bool IsNumber(VirtualChar ch) + => ch.Value is >= '0' and <= '9'; } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index ba38f154586a0..8b7ac374583b1 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -69,20 +69,6 @@ private SeparatedStackFrameNodeList() public StackFrameNodeOrToken this[int index] => NodesAndTokens[index]; public static SeparatedStackFrameNodeList Empty => new SeparatedStackFrameNodeList(); - - public ImmutableArray GetNodes() - { - using var _ = ArrayBuilder.GetInstance(out var builder); - - for (var i = 0; i < NodesAndTokens.Length; i = i + 2) - { - var node = NodesAndTokens[i].Node; - RoslynDebug.AssertNotNull(node); - builder.Add((TNode)node); - } - - return builder.ToImmutable(); - } } /// diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index c6879379f9003..e24cd6d61e423 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -260,7 +260,7 @@ private StackFrameParser(VirtualCharSequence text) var useCloseBracket = openToken.Kind is StackFrameKind.OpenBracketToken; using var _ = ArrayBuilder.GetInstance(out var builder); - var currentIdentifier = _lexer.TryScanIdentifier(); + var currentIdentifier = _lexer.TryScanIdentifier(scanWhitespace: true, scanAtTrivia: false); StackFrameToken closeToken = default; while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) @@ -357,7 +357,7 @@ private StackFrameParameterNode ParseParameterNode() typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier.Value, arrayIdentifiers); } - var identifier = _lexer.TryScanIdentifier(scanWhitespace: true); + var identifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanWhitespace: true); if (!identifier.HasValue) { throw new StackFrameParseException("Expected a parameter identifier"); From ceac888620429708e0d1d9002a48e90d3826f888 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 26 Oct 2021 22:53:39 -0700 Subject: [PATCH 22/59] Move to EmbeddedSeparatedSyntaxNodeList --- .../StackFrameParserTests.Utilities.cs | 2 +- .../StackFrame/StackFrameNodeDefinitions.cs | 41 +------ .../Core/CompilerExtensions.projitems | 1 + .../Common/EmbeddedSaparatedSyntaxNodeList.cs | 108 ++++++++++++++++++ .../Common/EmbeddedSyntaxNode.cs | 3 + .../Common/EmbeddedSyntaxToken.cs | 24 ++++ .../Common/EmbeddedSyntaxTrivia.cs | 7 ++ .../VirtualChars/VirtualCharSequence.cs | 8 ++ 8 files changed, 158 insertions(+), 36 deletions(-) create mode 100644 src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 6f00ce3f5f037..f66e8abe683c2 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -232,7 +232,7 @@ private static IEnumerable Enumerate(StackFrameNode node) yield return charSequence; } } - else + else if (nodeOrToken.Token != default) { foreach (var charSequence in Enumerate(nodeOrToken.Token)) { diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index 8b7ac374583b1..a6fb9fa94d88c 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -25,50 +25,21 @@ protected StackFrameNode(StackFrameKind kind) : base(kind) } public abstract void Accept(IStackFrameNodeVisitor visitor); - - public StackFrameNodeOrToken this[int index] => ChildAt(index); - public StackFrameNodeOrToken this[Index index] => this[index.GetOffset(this.ChildCount)]; } - internal sealed class SeparatedStackFrameNodeList where TNode : StackFrameNode + internal sealed class SeparatedStackFrameNodeList : EmbeddedSeparatedSyntaxNodeList> + where TNode : EmbeddedSyntaxNode { - public SeparatedStackFrameNodeList(ImmutableArray nodesAndTokens) + public SeparatedStackFrameNodeList(ImmutableArray nodeOrTokens) + : base(nodeOrTokens) { - Contract.ThrowIfTrue(nodesAndTokens.IsDefaultOrEmpty); - NodesAndTokens = nodesAndTokens; - -#if DEBUG - // Length should represent (nodes.Length) + (nodes.Length - 1), where the latter - // represents the number of separator tokens - Debug.Assert(nodesAndTokens.Length % 2 == 1); - for (var i = 0; i < nodesAndTokens.Length; i++) - { - if (i % 2 == 0) - { - // All even values should be TNode - Debug.Assert(nodesAndTokens[i].IsNode); - Debug.Assert(nodesAndTokens[i].Node is TNode); - } - else - { - // All odd values should be separator tokens - Debug.Assert(!nodesAndTokens[i].IsNode); - Debug.Assert(!nodesAndTokens[i].Token.IsMissing); - } - } -#endif } - private SeparatedStackFrameNodeList() + private SeparatedStackFrameNodeList() : base() { - NodesAndTokens = ImmutableArray.Empty; } - public ImmutableArray NodesAndTokens { get; } - public int Length => NodesAndTokens.Length; - public StackFrameNodeOrToken this[int index] => NodesAndTokens[index]; - - public static SeparatedStackFrameNodeList Empty => new SeparatedStackFrameNodeList(); + public static SeparatedStackFrameNodeList Empty { get; } = new(); } /// diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/CompilerExtensions.projitems b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/CompilerExtensions.projitems index f616833480ad6..270c7b1bace94 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/CompilerExtensions.projitems +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/CompilerExtensions.projitems @@ -192,6 +192,7 @@ + diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs new file mode 100644 index 0000000000000..f56a6a3577047 --- /dev/null +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs @@ -0,0 +1,108 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Immutable; +using System.Diagnostics; +using Roslyn.Utilities; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.Common +{ + internal abstract class EmbeddedSeparatedSyntaxNodeList + where TSyntaxKind : struct + where TSyntaxNode : EmbeddedSyntaxNode + where TList : EmbeddedSeparatedSyntaxNodeList + { + public EmbeddedSeparatedSyntaxNodeList(ImmutableArray> nodesAndTokens) + { + Contract.ThrowIfTrue(nodesAndTokens.IsDefaultOrEmpty); + NodesAndTokens = nodesAndTokens; + + // Verify size constraints for not empty list. If empty is desired the Empty static field should + // be used instead to retrieve an empty EmbeddedSeparatedSyntaxNodeList + Verify(); + + var allLength = NodesAndTokens.Length; + Length = (allLength + 1) / 2; + SeparatorLength = allLength / 2; + } + + protected EmbeddedSeparatedSyntaxNodeList() + { + NodesAndTokens = ImmutableArray>.Empty; + } + + [Conditional("DEBUG")] + private void Verify() + { + for (var i = 0; i < NodesAndTokens.Length; i++) + { + if ((i & 1) == 0) + { + // All even values should be TNode + Debug.Assert(NodesAndTokens[i].IsNode); + Debug.Assert(NodesAndTokens[i].Node is EmbeddedSyntaxNode); + } + else + { + // All odd values should be separator tokens + Debug.Assert(!NodesAndTokens[i].IsNode); + Debug.Assert(NodesAndTokens[i].Token != default); + } + } + } + + public ImmutableArray> NodesAndTokens { get; } + public int Length { get; } + public int SeparatorLength { get; } + + /// + /// Retrieves only nodes, skipping the separator tokens + /// + public EmbeddedSyntaxNodeOrToken this[int index] + { + get + { + if (index < Length && index > 0) + { + // x2 here to get only even indexed numbers. Follows same logic + // as SeparatedSyntaxList in that the separator tokens are not returned + return NodesAndTokens[index * 2]; + } + + throw new ArgumentOutOfRangeException(nameof(index)); + } + } + + public Enumerator GetEnumerator() => new(this); + + public struct Enumerator + { + private readonly EmbeddedSeparatedSyntaxNodeList _list; + private int _currentIndex; + + public Enumerator(EmbeddedSeparatedSyntaxNodeList list) + { + _list = list; + _currentIndex = -1; + Current = default; + } + + public EmbeddedSyntaxNodeOrToken Current { get; private set; } + + public bool MoveNext() + { + _currentIndex++; + if (_currentIndex >= _list.Length) + { + Current = null; + return false; + } + + Current = _list[_currentIndex]; + return true; + } + } + } +} diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxNode.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxNode.cs index 1d7c493a5381f..2c5689b88beb7 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxNode.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxNode.cs @@ -47,6 +47,9 @@ protected EmbeddedSyntaxNode(TSyntaxKind kind) internal abstract int ChildCount { get; } internal abstract EmbeddedSyntaxNodeOrToken ChildAt(int index); + public EmbeddedSyntaxNodeOrToken this[int index] => ChildAt(index); + public EmbeddedSyntaxNodeOrToken this[Index index] => this[index.GetOffset(this.ChildCount)]; + public TextSpan GetSpan() { var start = int.MaxValue; diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs index ddbeb7f618641..0d440e1390a6d 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs @@ -2,10 +2,13 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Immutable; using System.Diagnostics; +using System.Linq; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.Text; +using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.Common { @@ -69,5 +72,26 @@ public EmbeddedSyntaxToken With( public TextSpan GetSpan() => EmbeddedSyntaxHelpers.GetSpan(this.VirtualChars); + + public static bool operator ==(EmbeddedSyntaxToken t1, EmbeddedSyntaxToken t2) + => t1.Kind.Equals(t2.Kind) + && t1.LeadingTrivia.SequenceEqual(t2.LeadingTrivia) + && t1.VirtualChars == t2.VirtualChars + && t1.TrailingTrivia.SequenceEqual(t2.TrailingTrivia) + && t1.Diagnostics.SequenceEqual(t2.Diagnostics) + && t1.Value == t2.Value; + + public static bool operator !=(EmbeddedSyntaxToken t1, EmbeddedSyntaxToken t2) + => !(t1 == t2); + + public override bool Equals(object? obj) + => obj is EmbeddedSyntaxToken t && this == t; + + public override int GetHashCode() + => Hash.Combine(Kind.GetHashCode(), + Hash.Combine(LeadingTrivia.GetHashCode(), + Hash.Combine(VirtualChars.GetHashCode(), + Hash.Combine(TrailingTrivia.GetHashCode(), + Hash.Combine(Diagnostics.GetHashCode(), Value?.GetHashCode() ?? 0))))); } } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs index 7f06e3138c56e..bfe4bca006d91 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs @@ -29,5 +29,12 @@ public EmbeddedSyntaxTrivia(TSyntaxKind kind, VirtualCharSequence virtualChars, VirtualChars = virtualChars; Diagnostics = diagnostics; } + + public static bool operator ==(EmbeddedSyntaxTrivia t1, EmbeddedSyntaxTrivia t2) + => t1.Kind.Equals(t2.Kind) + && t1.VirtualChars.Equals(t2.VirtualChars); + + public static bool operator !=(EmbeddedSyntaxTrivia t1, EmbeddedSyntaxTrivia t2) + => !(t1 == t2); } } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs index 37dc47a5a64c3..fb5abe07e4cee 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs @@ -144,5 +144,13 @@ public static VirtualCharSequence FromBounds( chars1._leafCharacters, TextSpan.FromBounds(chars1._span.Start, chars2._span.End)); } + + public static bool operator ==(VirtualCharSequence v1, VirtualCharSequence v2) + => v1.Length == v2.Length + && v1._span == v2._span + && v1._leafCharacters == v2._leafCharacters; + + public static bool operator !=(VirtualCharSequence v1, VirtualCharSequence v2) + => !(v1 == v2); } } From f972f1100de86ee3d151a97efa146b8398106549 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 26 Oct 2021 23:42:38 -0700 Subject: [PATCH 23/59] Update for passing tests and include keywords as identifiers --- .../StackFrame/StackFrameParserTests.cs | 9 ++++++--- .../StackFrame/StackFrameSyntaxFactory.cs | 3 +++ .../EmbeddedLanguages/StackFrame/StackFrameLexer.cs | 11 ++++++----- .../StackFrame/StackFrameNodeDefinitions.cs | 10 ++++++---- .../EmbeddedLanguages/StackFrame/StackFrameParser.cs | 10 +++++----- .../Common/EmbeddedSaparatedSyntaxNodeList.cs | 2 +- .../EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs | 9 ++++++--- 7 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 9e48864a11b5e..093ec3994ef39 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -360,16 +360,19 @@ public void TestInvalidInputs(string input) [Theory] [InlineData("at ")] - [InlineData(" in ")] + [InlineData("in ")] + [InlineData("line ")] public void TestKeywordsAsIdentifiers(string keyword) => Verify(@$"MyNamespace.MyType.MyMethod[{keyword}]({keyword} {keyword})", methodDeclaration: MethodDeclaration( MemberAccessExpression("MyNamespace.MyType.MyMethod"), - typeArguments: TypeArgumentList(TypeArgument(keyword.Trim())), + typeArguments: TypeArgumentList(TypeArgument(Identifier(keyword.Trim(), trailingTrivia: SpaceTrivia()))), argumentList: ParameterList( OpenParenToken, CloseParenToken, - Parameter(Identifier(keyword), Identifier(keyword)))) + Parameter( + Identifier(keyword.Trim()), + Identifier(keyword.Trim(), leadingTrivia: SpaceTrivia(2), trailingTrivia: SpaceTrivia())))) ); } } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 4d13e639d4e5b..3e64d07d245a4 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -180,6 +180,9 @@ public static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, para public static StackFrameTypeArgumentNode TypeArgument(string identifier) => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); + public static StackFrameTypeArgumentNode TypeArgument(StackFrameToken identifier) + => new(identifier); + public static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken colon, StackFrameToken line) => new(path.With(leadingTrivia: CreateTriviaArray(InTrivia)), colon, line); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index d1c322abfc317..5e5877aff5de2 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -53,13 +53,13 @@ public VirtualCharSequence GetSubSequence(int start, int end) } public StackFrameToken? TryScanIdentifier() - => TryScanIdentifier(scanAtTrivia: false, scanWhitespace: false); + => TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: false, scanTrailingWhitespace: false); - public StackFrameToken? TryScanIdentifier(bool scanAtTrivia, bool scanWhitespace) + public StackFrameToken? TryScanIdentifier(bool scanAtTrivia, bool scanLeadingWhitespace, bool scanTrailingWhitespace) { var originalPosition = Position; var atTrivia = scanAtTrivia ? TryScanAtTrivia() : null; - var leadingWhitespace = scanWhitespace ? TryScanWhiteSpace() : null; + var leadingWhitespace = scanLeadingWhitespace ? TryScanWhiteSpace() : null; var startPosition = Position; var ch = CurrentChar; @@ -78,12 +78,13 @@ public VirtualCharSequence GetSubSequence(int start, int end) ch = CurrentChar; } - var trailingWhitespace = scanWhitespace ? TryScanWhiteSpace() : null; + var identifierSequence = GetSubSequenceToCurrentPos(startPosition); + var trailingWhitespace = scanTrailingWhitespace ? TryScanWhiteSpace() : null; return CreateToken( StackFrameKind.IdentifierToken, leadingTrivia: CreateTrivia(atTrivia, leadingWhitespace), - GetSubSequenceToCurrentPos(startPosition), + identifierSequence, trailingTrivia: CreateTrivia(trailingWhitespace)); } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index a6fb9fa94d88c..1085ae3512c48 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -255,7 +255,7 @@ public StackFrameTypeArgumentList(StackFrameToken openToken, SeparatedStackFrame TypeArguments = typeArguments; } - internal override int ChildCount => TypeArguments.Length + 2; + internal override int ChildCount => TypeArguments.NodesAndTokens.Length + 2; public override void Accept(IStackFrameNodeVisitor visitor) => visitor.Visit(this); @@ -277,7 +277,8 @@ internal override StackFrameNodeOrToken ChildAt(int index) return CloseToken; } - return TypeArguments[index - 1]; + // Includes both the nodes and separator tokens as children + return TypeArguments.NodesAndTokens[index - 1]; } } @@ -348,7 +349,7 @@ public StackFrameParameterList(StackFrameToken openToken, StackFrameToken closeT Parameters = parameters; } - internal override int ChildCount => 2 + Parameters.Length; + internal override int ChildCount => 2 + Parameters.NodesAndTokens.Length; public override void Accept(IStackFrameNodeVisitor visitor) { @@ -367,7 +368,8 @@ internal override StackFrameNodeOrToken ChildAt(int index) return CloseParen; } - return Parameters[index - 1]; + // Include both nodes and tokens here as children of the StackFrameParameterList + return Parameters.NodesAndTokens[index - 1]; } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index e24cd6d61e423..ab005d1a490d1 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -160,9 +160,9 @@ private StackFrameParser(VirtualCharSequence text) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) /// - private StackFrameNodeOrToken? TryParseIdentifierExpression(bool scanAtTrivia = false) + private StackFrameNodeOrToken? TryParseIdentifierExpression(bool scanAtTrivia) { - var currentIdentifer = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanWhitespace: true); + var currentIdentifer = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanLeadingWhitespace: true, scanTrailingWhitespace: false); if (!currentIdentifer.HasValue) { return null; @@ -260,7 +260,7 @@ private StackFrameParser(VirtualCharSequence text) var useCloseBracket = openToken.Kind is StackFrameKind.OpenBracketToken; using var _ = ArrayBuilder.GetInstance(out var builder); - var currentIdentifier = _lexer.TryScanIdentifier(scanWhitespace: true, scanAtTrivia: false); + var currentIdentifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: true, scanTrailingWhitespace: true); StackFrameToken closeToken = default; while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) @@ -345,7 +345,7 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameParameterNode ParseParameterNode() { - var typeIdentifier = TryParseIdentifierExpression(); + var typeIdentifier = TryParseIdentifierExpression(scanAtTrivia: false); if (!typeIdentifier.HasValue) { throw new StackFrameParseException("Expected type identifier when parsing parameters"); @@ -357,7 +357,7 @@ private StackFrameParameterNode ParseParameterNode() typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier.Value, arrayIdentifiers); } - var identifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanWhitespace: true); + var identifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: true, scanTrailingWhitespace: true); if (!identifier.HasValue) { throw new StackFrameParseException("Expected a parameter identifier"); diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs index f56a6a3577047..aafaf6f61d307 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs @@ -64,7 +64,7 @@ public EmbeddedSyntaxNodeOrToken this[int index] { get { - if (index < Length && index > 0) + if (index < Length && index >= 0) { // x2 here to get only even indexed numbers. Follows same logic // as SeparatedSyntaxList in that the separator tokens are not returned diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs index 0d440e1390a6d..1a99a2b986bb5 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs @@ -75,10 +75,13 @@ public TextSpan GetSpan() public static bool operator ==(EmbeddedSyntaxToken t1, EmbeddedSyntaxToken t2) => t1.Kind.Equals(t2.Kind) - && t1.LeadingTrivia.SequenceEqual(t2.LeadingTrivia) + && t1.LeadingTrivia.IsDefault == t2.LeadingTrivia.IsDefault + && (t1.LeadingTrivia.IsDefault || t1.LeadingTrivia.SequenceEqual(t2.LeadingTrivia)) && t1.VirtualChars == t2.VirtualChars - && t1.TrailingTrivia.SequenceEqual(t2.TrailingTrivia) - && t1.Diagnostics.SequenceEqual(t2.Diagnostics) + && t1.TrailingTrivia.IsDefault == t2.TrailingTrivia.IsDefault + && (t1.TrailingTrivia.IsDefault || t1.TrailingTrivia.SequenceEqual(t2.TrailingTrivia)) + && t1.Diagnostics.IsDefault == t2.Diagnostics.IsDefault + && (t1.Diagnostics.IsDefault || t1.Diagnostics.SequenceEqual(t2.Diagnostics)) && t1.Value == t2.Value; public static bool operator !=(EmbeddedSyntaxToken t1, EmbeddedSyntaxToken t2) From dd729d24332b6184988d67d3f3e404ffb66c5289 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 26 Oct 2021 23:45:35 -0700 Subject: [PATCH 24/59] Make creation of trivia private --- .../Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 5e5877aff5de2..0ef8fafc57c3d 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -286,10 +286,10 @@ public static StackFrameToken CreateToken(StackFrameKind kind, ImmutableArray leadingTrivia, VirtualCharSequence virtualChars, ImmutableArray trailingTrivia) => new(kind, leadingTrivia, virtualChars, trailingTrivia, ImmutableArray.Empty, value: null!); - public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars) + private static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars) => CreateTrivia(kind, virtualChars, ImmutableArray.Empty); - public static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars, ImmutableArray diagnostics) + private static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSequence virtualChars, ImmutableArray diagnostics) { // Empty trivia is not supported in StackFrames Debug.Assert(virtualChars.Length > 0); From ebabf0fc587c49428a8d148281979a50e5cae14e Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 26 Oct 2021 23:47:11 -0700 Subject: [PATCH 25/59] Missed one --- .../Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 0ef8fafc57c3d..74a3d42c2174c 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -296,7 +296,7 @@ private static StackFrameTrivia CreateTrivia(StackFrameKind kind, VirtualCharSeq return new(kind, virtualChars, diagnostics); } - public static ImmutableArray CreateTrivia(params StackFrameTrivia?[] triviaArray) + private static ImmutableArray CreateTrivia(params StackFrameTrivia?[] triviaArray) { using var _ = ArrayBuilder.GetInstance(out var builder); foreach (var trivia in triviaArray) From 80c7b770c04e1be123baa0ccb18b15bf51ed85c9 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 26 Oct 2021 23:54:01 -0700 Subject: [PATCH 26/59] Change StackFrameTree to inherit from EmbeddedSyntaxTree --- .../EmbeddedLanguages/StackFrame/StackFrameTree.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs index d2373c94630a9..49886856eb58c 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameTree.cs @@ -8,15 +8,11 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { - internal class StackFrameTree + internal class StackFrameTree : EmbeddedSyntaxTree { public StackFrameTree(VirtualCharSequence text, StackFrameCompilationUnit root) + : base(text, root, ImmutableArray.Empty) { - Text = text; - Root = root; } - - public VirtualCharSequence Text { get; } - public StackFrameCompilationUnit Root { get; } } } From 3904852a23dc0d8d24704df9d8ff6f3d6aab2e00 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 27 Oct 2021 13:19:50 -0700 Subject: [PATCH 27/59] PR feedback. Make name nodes better handled to avoid common cases where StackFrameNodeOrToken was used --- .../StackFrame/StackFrameParserTests.cs | 92 +++++------ .../StackFrame/StackFrameSyntaxFactory.cs | 69 ++++---- .../StackFrame/IStackFrameNodeVisitor.cs | 6 +- .../StackFrame/StackFrameNodeDefinitions.cs | 156 ++++++++++-------- ...ackFrameParser.StackFrameParseException.cs | 48 ++++++ .../StackFrame/StackFrameParser.cs | 96 ++++------- 6 files changed, 253 insertions(+), 214 deletions(-) create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.StackFrameParseException.cs diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 093ec3994ef39..e165035e104c6 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -14,7 +14,7 @@ public void TestNoParams() => Verify( @"at ConsoleApp4.MyClass.M()", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), + QualifiedName("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), argumentList: EmptyParams) ); @@ -27,8 +27,8 @@ public void TestArity(string typeName, int arity) => Verify( $"at ConsoleApp4.{typeName}`{arity}.M()", methodDeclaration: MethodDeclaration( - MemberAccessExpression( - MemberAccessExpression( + QualifiedName( + QualifiedName( Identifier("ConsoleApp4", leadingTrivia: AtTrivia), GenericType(typeName, arity)), Identifier("M")), @@ -41,7 +41,7 @@ public void TestTrailingTrivia() => Verify( @"at ConsoleApp4.MyClass.M() some other text", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), + QualifiedName("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(" some other text")) @@ -52,7 +52,7 @@ public void TestTrailingTrivia_InTriviaNoSpace() => Verify( @"at ConsoleApp4.MyClass.M() inC:\My\Path\C.cs:line 26", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), + QualifiedName("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(@" inC:\My\Path\C.cs:line 26")) @@ -63,7 +63,7 @@ public void TestTrailingTrivia_InTriviaNoSpace2() => Verify( @"at ConsoleApp4.MyClass.M()in C:\My\Path\C.cs:line 26", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), + QualifiedName("ConsoleApp4.MyClass.M", leadingTrivia: AtTrivia), argumentList: EmptyParams), eolTokenOpt: EOLToken.With(leadingTrivia: CreateTriviaArray(@"in C:\My\Path\C.cs:line 26")) @@ -74,7 +74,7 @@ public void TestNoParams_NoAtTrivia() => Verify( @"ConsoleApp4.MyClass.M()", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M"), + QualifiedName("ConsoleApp4.MyClass.M"), argumentList: EmptyParams) ); @@ -83,7 +83,7 @@ public void TestNoParams_SpaceInParams_NoAtTrivia() => Verify( @"ConsoleApp4.MyClass.M( )", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M"), + QualifiedName("ConsoleApp4.MyClass.M"), argumentList: ParameterList( OpenParenToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia(2))), CloseParenToken)) @@ -94,7 +94,7 @@ public void TestNoParams_SpaceTrivia() => Verify( @" ConsoleApp4.MyClass.M()", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: SpaceTrivia()), + QualifiedName("ConsoleApp4.MyClass.M", leadingTrivia: SpaceTrivia()), argumentList: EmptyParams) ); @@ -103,7 +103,7 @@ public void TestNoParams_SpaceTrivia2() => Verify( @" ConsoleApp4.MyClass.M()", methodDeclaration: MethodDeclaration( - MemberAccessExpression("ConsoleApp4.MyClass.M", leadingTrivia: SpaceTrivia(2)), + QualifiedName("ConsoleApp4.MyClass.M", leadingTrivia: SpaceTrivia(2)), argumentList: EmptyParams) ); @@ -112,18 +112,16 @@ public void TestMethodOneParam() => Verify( @"at ConsoleApp4.MyClass.M(string s)", methodDeclaration: MethodDeclaration( - MemberAccessExpression( - MemberAccessExpression( + QualifiedName( + QualifiedName( Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), argumentList: ParameterList( - OpenParenToken, - CloseParenToken, Parameter( Identifier("string"), - Identifier("s", leadingTrivia: SpaceTrivia()))) + IdentifierToken("s", leadingTrivia: SpaceTrivia()))) ) ); @@ -132,21 +130,19 @@ public void TestMethodTwoParam() => Verify( @"at ConsoleApp4.MyClass.M(string s, string t)", methodDeclaration: MethodDeclaration( - MemberAccessExpression( - MemberAccessExpression( + QualifiedName( + QualifiedName( Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), argumentList: ParameterList( - OpenParenToken, - CloseParenToken, Parameter( Identifier("string"), - Identifier("s", leadingTrivia: SpaceTrivia())), + IdentifierToken("s", leadingTrivia: SpaceTrivia())), Parameter( Identifier("string", leadingTrivia: SpaceTrivia()), - Identifier("t", leadingTrivia: SpaceTrivia()))) + IdentifierToken("t", leadingTrivia: SpaceTrivia()))) ) ); @@ -155,17 +151,15 @@ public void TestMethodArrayParam() => Verify( @"at ConsoleApp4.MyClass.M(string[] s)", methodDeclaration: MethodDeclaration( - MemberAccessExpression( - MemberAccessExpression( + QualifiedName( + QualifiedName( Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), argumentList: ParameterList( - OpenParenToken, - CloseParenToken, Parameter(ArrayExpression(Identifier("string"), ArrayRankSpecifier(trailingTrivia: SpaceTrivia())), - Identifier("s"))) + IdentifierToken("s"))) ) ); @@ -174,18 +168,16 @@ public void TestCommaArrayParam() => Verify( @"at ConsoleApp4.MyClass.M(string[,] s)", methodDeclaration: MethodDeclaration( - MemberAccessExpression( - MemberAccessExpression( + QualifiedName( + QualifiedName( Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), argumentList: ParameterList( - OpenParenToken, - CloseParenToken, Parameter( ArrayExpression(Identifier("string"), ArrayRankSpecifier(1, trailingTrivia: SpaceTrivia())), - Identifier("s"))) + IdentifierToken("s"))) ) ); @@ -202,18 +194,16 @@ public void TestGenericMethod_Brackets() => Verify( @"at ConsoleApp4.MyClass.M[T](T t)", methodDeclaration: MethodDeclaration( - MemberAccessExpression( - MemberAccessExpression( + QualifiedName( + QualifiedName( Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), typeArguments: TypeArgumentList(TypeArgument("T")), argumentList: ParameterList( - OpenParenToken, - CloseParenToken, Parameter( Identifier("T"), - Identifier("t", leadingTrivia: SpaceTrivia()))) + IdentifierToken("t", leadingTrivia: SpaceTrivia()))) ) ); @@ -222,18 +212,16 @@ public void TestGenericMethod() => Verify( @"at ConsoleApp4.MyClass.M(T t)", methodDeclaration: MethodDeclaration( - MemberAccessExpression( - MemberAccessExpression( + QualifiedName( + QualifiedName( Identifier("ConsoleApp4", leadingTrivia: AtTrivia), Identifier("MyClass")), Identifier("M")), typeArguments: TypeArgumentList(useBrackets: false, TypeArgument("T")), argumentList: ParameterList( - OpenParenToken, - CloseParenToken, Parameter( Identifier("T"), - Identifier("t", leadingTrivia: SpaceTrivia()))) + IdentifierToken("t", leadingTrivia: SpaceTrivia()))) ) ); @@ -250,14 +238,12 @@ public void TestIdentifierNames(string identifierName) => Verify( @$"at {identifierName}.{identifierName}[{identifierName}]({identifierName} {identifierName})", methodDeclaration: MethodDeclaration( - MemberAccessExpression($"{identifierName}.{identifierName}", leadingTrivia: AtTrivia), + QualifiedName($"{identifierName}.{identifierName}", leadingTrivia: AtTrivia), typeArguments: TypeArgumentList(TypeArgument(identifierName)), argumentList: ParameterList( - OpenParenToken, - CloseParenToken, Parameter( Identifier(identifierName), - Identifier(identifierName, leadingTrivia: SpaceTrivia()))) + IdentifierToken(identifierName, leadingTrivia: SpaceTrivia()))) ) ); @@ -266,7 +252,7 @@ public void TestAnonymousMethod() => Verify( @"Microsoft.VisualStudio.DesignTools.SurfaceDesigner.Tools.EventRouter.ScopeElement_MouseUp.AnonymousMethod__0()", methodDeclaration: MethodDeclaration( - MemberAccessExpression("Microsoft.VisualStudio.DesignTools.SurfaceDesigner.Tools.EventRouter.ScopeElement_MouseUp.AnonymousMethod__0"), + QualifiedName("Microsoft.VisualStudio.DesignTools.SurfaceDesigner.Tools.EventRouter.ScopeElement_MouseUp.AnonymousMethod__0"), argumentList: EmptyParams) ); @@ -275,7 +261,7 @@ public void TestFileInformation() => Verify( @"M.M() in C:\folder\m.cs:line 1", methodDeclaration: MethodDeclaration( - MemberAccessExpression("M.M"), + QualifiedName("M.M"), argumentList: EmptyParams), fileInformation: FileInformation( @@ -306,7 +292,7 @@ public void TestFilePaths(string path, int line) => Verify( $"M.M() in {path}:line {line}", methodDeclaration: MethodDeclaration( - MemberAccessExpression("M.M"), + QualifiedName("M.M"), argumentList: EmptyParams), fileInformation: FileInformation( @@ -320,7 +306,7 @@ public void TestFileInformation_TrailingTrivia() => Verify( @"M.M() in C:\folder\m.cs:line 1[trailingtrivia]", methodDeclaration: MethodDeclaration( - MemberAccessExpression("M.M"), + QualifiedName("M.M"), argumentList: EmptyParams), fileInformation: FileInformation( @@ -365,14 +351,12 @@ public void TestInvalidInputs(string input) public void TestKeywordsAsIdentifiers(string keyword) => Verify(@$"MyNamespace.MyType.MyMethod[{keyword}]({keyword} {keyword})", methodDeclaration: MethodDeclaration( - MemberAccessExpression("MyNamespace.MyType.MyMethod"), - typeArguments: TypeArgumentList(TypeArgument(Identifier(keyword.Trim(), trailingTrivia: SpaceTrivia()))), + QualifiedName("MyNamespace.MyType.MyMethod"), + typeArguments: TypeArgumentList(TypeArgument(IdentifierToken(keyword.Trim(), trailingTrivia: SpaceTrivia()))), argumentList: ParameterList( - OpenParenToken, - CloseParenToken, Parameter( Identifier(keyword.Trim()), - Identifier(keyword.Trim(), leadingTrivia: SpaceTrivia(2), trailingTrivia: SpaceTrivia())))) + IdentifierToken(keyword.Trim(), leadingTrivia: SpaceTrivia(2), trailingTrivia: SpaceTrivia())))) ); } } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 3e64d07d245a4..9072772f8dec9 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -57,16 +57,19 @@ public static ImmutableArray CreateTriviaArray(params string[] public static readonly StackFrameParameterList EmptyParams = ParameterList(OpenParenToken, CloseParenToken); - public static StackFrameParameterNode Parameter(StackFrameNodeOrToken type, StackFrameToken identifier) + public static StackFrameParameterNode Parameter(StackFrameNameNode type, StackFrameToken identifier) => new(type, identifier); + public static StackFrameParameterList ParameterList(params StackFrameParameterNode[] parameters) + => ParameterList(OpenParenToken, CloseParenToken, parameters); + public static StackFrameParameterList ParameterList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameParameterNode[] parameters) { var separatedList = parameters.Length == 0 ? SeparatedStackFrameNodeList.Empty : new(CommaSeparateList(parameters)); - return new(openToken, closeToken, separatedList); + return new(openToken, separatedList, closeToken); static ImmutableArray CommaSeparateList(StackFrameParameterNode[] parameters) { @@ -84,74 +87,80 @@ static ImmutableArray CommaSeparateList(StackFrameParamet } public static StackFrameMethodDeclarationNode MethodDeclaration( - StackFrameMemberAccessExpressionNode memberAccessExpression, + StackFrameQualifiedNameNode memberAccessExpression, StackFrameTypeArgumentList? typeArguments = null, StackFrameParameterList? argumentList = null) { return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, argumentList ?? ParameterList(OpenParenToken, CloseParenToken)); } - public static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) - => MemberAccessExpression(s, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); + public static StackFrameQualifiedNameNode QualifiedName(string s, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + => QualifiedName(s, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); - public static StackFrameMemberAccessExpressionNode MemberAccessExpression(string s, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) + public static StackFrameQualifiedNameNode QualifiedName(string s, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) { - StackFrameNodeOrToken? current = null; + StackFrameNameNode? current = null; var identifiers = s.Split('.'); for (var i = 0; i < identifiers.Length; i++) { var identifier = identifiers[i]; - if (!current.HasValue) + if (current is null) { - current = Identifier(identifier, leadingTrivia: leadingTrivia, trailingTrivia: ImmutableArray.Empty); + current = Identifier(IdentifierToken(identifier, leadingTrivia: leadingTrivia, trailingTrivia: ImmutableArray.Empty)); } else if (i == identifiers.Length - 1) { - current = MemberAccessExpression(current.Value, Identifier(identifier, leadingTrivia: ImmutableArray.Empty, trailingTrivia: trailingTrivia)); + var rhs = Identifier(IdentifierToken(identifier, leadingTrivia: ImmutableArray.Empty, trailingTrivia: trailingTrivia)); + current = QualifiedName(current, rhs); } else { - current = MemberAccessExpression(current.Value, Identifier(identifier)); + current = QualifiedName(current, Identifier(identifier)); } } - Assert.True(current.HasValue); - Assert.True(current!.Value.IsNode); - - var node = current.Value.Node; - AssertEx.NotNull(node); - return (StackFrameMemberAccessExpressionNode)node; + AssertEx.NotNull(current); + return (StackFrameQualifiedNameNode)current; } public static StackFrameTrivia SpaceTrivia(int count = 1) => CreateTrivia(StackFrameKind.WhitespaceTrivia, new string(' ', count)); - public static StackFrameMemberAccessExpressionNode MemberAccessExpression(StackFrameNodeOrToken left, StackFrameNodeOrToken right) + public static StackFrameQualifiedNameNode QualifiedName(StackFrameNameNode left, StackFrameSimpleNameNode right) => new(left, DotToken, right); - public static StackFrameToken Identifier(string identifierName) - => Identifier(identifierName, leadingTrivia: null, trailingTrivia: null); + public static StackFrameToken IdentifierToken(string identifierName) + => IdentifierToken(identifierName, leadingTrivia: null, trailingTrivia: null); - public static StackFrameToken Identifier(string identifierName, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) - => Identifier(identifierName, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); + public static StackFrameToken IdentifierToken(string identifierName, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + => IdentifierToken(identifierName, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); - public static StackFrameToken Identifier(string identifierName, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) + public static StackFrameToken IdentifierToken(string identifierName, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) => CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia); + public static StackFrameIdentifierNameNode Identifier(string name) + => Identifier(IdentifierToken(name)); + + public static StackFrameIdentifierNameNode Identifier(StackFrameToken identifier) + => new(identifier); + + public static StackFrameIdentifierNameNode Identifier(string name, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) + => Identifier(IdentifierToken(name, leadingTrivia, trailingTrivia)); + public static StackFrameArrayRankSpecifier ArrayRankSpecifier(int commaCount = 0, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) => new(OpenBracketToken.With(leadingTrivia: CreateTriviaArray(leadingTrivia)), CloseBracketToken.With(trailingTrivia: CreateTriviaArray(trailingTrivia)), Enumerable.Repeat(CommaToken, commaCount).ToImmutableArray()); - public static StackFrameArrayTypeExpression ArrayExpression(StackFrameNodeOrToken identifier, params StackFrameArrayRankSpecifier[] arrayTokens) + public static StackFrameArrayTypeExpression ArrayExpression(StackFrameNameNode identifier, params StackFrameArrayRankSpecifier[] arrayTokens) => new(identifier, arrayTokens.ToImmutableArray()); - public static StackFrameGenericTypeIdentifier GenericType(string identifierName, int arity) + public static StackFrameGenericNameNode GenericType(string identifierName, int arity) => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.NumberToken, arity.ToString())); - public static StackFrameTypeArgumentList TypeArgumentList(params StackFrameTypeArgumentNode[] typeArguments) + public static StackFrameTypeArgumentList TypeArgumentList(params StackFrameIdentifierNameNode[] typeArguments) => TypeArgumentList(useBrackets: true, typeArguments); - public static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameTypeArgumentNode[] typeArguments) + public static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, params StackFrameIdentifierNameNode[] typeArguments) { using var _ = PooledObjects.ArrayBuilder.GetInstance(out var builder); var openToken = useBrackets ? OpenBracketToken : LessThanToken; @@ -172,15 +181,15 @@ public static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, para builder.Add(typeArgument); } - var typeArgumentsList = new SeparatedStackFrameNodeList(builder.ToImmutable()); + var typeArgumentsList = new SeparatedStackFrameNodeList(builder.ToImmutable()); return new(openToken, typeArgumentsList, closeToken); } - public static StackFrameTypeArgumentNode TypeArgument(string identifier) + public static StackFrameIdentifierNameNode TypeArgument(string identifier) => new(CreateToken(StackFrameKind.IdentifierToken, identifier)); - public static StackFrameTypeArgumentNode TypeArgument(StackFrameToken identifier) + public static StackFrameIdentifierNameNode TypeArgument(StackFrameToken identifier) => new(identifier); public static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken colon, StackFrameToken line) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index e23d9da97a459..dca069eb7d836 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -8,11 +8,11 @@ internal interface IStackFrameNodeVisitor { void Visit(StackFrameCompilationUnit node); void Visit(StackFrameMethodDeclarationNode node); - void Visit(StackFrameMemberAccessExpressionNode node); + void Visit(StackFrameQualifiedNameNode node); void Visit(StackFrameTypeArgumentList node); void Visit(StackFrameParameterList node); - void Visit(StackFrameGenericTypeIdentifier node); - void Visit(StackFrameTypeArgumentNode node); + void Visit(StackFrameGenericNameNode node); + void Visit(StackFrameIdentifierNameNode node); void Visit(StackFrameArrayRankSpecifier node); void Visit(StackFrameFileInformationNode node); void Visit(StackFrameArrayTypeExpression node); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index 1085ae3512c48..13b33b2917c25 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -54,12 +54,12 @@ protected StackFrameExpressionNode(StackFrameKind kind) : base(kind) internal sealed class StackFrameMethodDeclarationNode : StackFrameNode { - public readonly StackFrameMemberAccessExpressionNode MemberAccessExpression; + public readonly StackFrameQualifiedNameNode MemberAccessExpression; public readonly StackFrameTypeArgumentList? TypeArguments; public readonly StackFrameParameterList ArgumentList; public StackFrameMethodDeclarationNode( - StackFrameMemberAccessExpressionNode memberAccessExpression, + StackFrameQualifiedNameNode memberAccessExpression, StackFrameTypeArgumentList? typeArguments, StackFrameParameterList argumentList) : base(StackFrameKind.MethodDeclaration) @@ -84,18 +84,45 @@ internal override StackFrameNodeOrToken ChildAt(int index) }; } - internal sealed class StackFrameMemberAccessExpressionNode : StackFrameExpressionNode + /// + /// Base class for all name nodes + /// + internal abstract class StackFrameNameNode : StackFrameNode + { + protected StackFrameNameNode(StackFrameKind kind) : base(kind) + { + } + } + + /// + /// Base class for and + /// + internal abstract class StackFrameSimpleNameNode : StackFrameNameNode + { + public readonly StackFrameToken Identifier; + + protected StackFrameSimpleNameNode(StackFrameToken identifier, StackFrameKind kind) : base(kind) + { + Debug.Assert(identifier.Kind == StackFrameKind.IdentifierToken); + Identifier = identifier; + } + } + + /// + /// Represents a qualified name, such as "MyClass.MyMethod" + /// + internal sealed class StackFrameQualifiedNameNode : StackFrameNameNode { - public readonly StackFrameNodeOrToken Left; - public readonly StackFrameToken Operator; - public readonly StackFrameNodeOrToken Right; + public readonly StackFrameNameNode Left; + public readonly StackFrameToken DotToken; + public readonly StackFrameSimpleNameNode Right; - public StackFrameMemberAccessExpressionNode(StackFrameNodeOrToken left, StackFrameToken operatorToken, StackFrameNodeOrToken right) : base(StackFrameKind.MemberAccess) + public StackFrameQualifiedNameNode(StackFrameNameNode left, StackFrameToken dotToken, StackFrameSimpleNameNode right) : base(StackFrameKind.MemberAccess) { - Debug.Assert(left.IsNode || left.Token.Kind == StackFrameKind.IdentifierToken); - Debug.Assert(right.IsNode || right.Token.Kind == StackFrameKind.IdentifierToken); + Debug.Assert(dotToken.Kind == StackFrameKind.DotToken); + Left = left; - Operator = operatorToken; + DotToken = dotToken; Right = right; } @@ -108,31 +135,55 @@ internal override StackFrameNodeOrToken ChildAt(int index) => index switch { 0 => Left, - 1 => Operator, + 1 => DotToken, 2 => Right, _ => throw new InvalidOperationException() }; } - internal abstract class StackFrameBaseIdentifierNode : StackFrameExpressionNode + /// + /// The simplest identifier node, which wraps a + /// + internal sealed class StackFrameIdentifierNameNode : StackFrameSimpleNameNode { - protected StackFrameBaseIdentifierNode(StackFrameKind kind) : base(kind) + internal override int ChildCount => 1; + + public StackFrameIdentifierNameNode(StackFrameToken identifier) + : base(identifier, StackFrameKind.TypeIdentifier) { } + + public override void Accept(IStackFrameNodeVisitor visitor) + => visitor.Visit(this); + + internal override StackFrameNodeOrToken ChildAt(int index) + => index switch + { + 0 => Identifier, + _ => throw new InvalidOperationException() + }; } - internal sealed class StackFrameGenericTypeIdentifier : StackFrameBaseIdentifierNode + /// + /// An identifier with an arity, such as "MyNamespace.MyClass`1" + /// + internal sealed class StackFrameGenericNameNode : StackFrameSimpleNameNode { - public readonly StackFrameToken Identifier; + /// + /// The "`" token in arity identifiers. Must be + /// public readonly StackFrameToken ArityToken; + public readonly StackFrameToken ArityNumericToken; internal override int ChildCount => 3; - public StackFrameGenericTypeIdentifier(StackFrameToken identifier, StackFrameToken arityToken, StackFrameToken arityNumericToken) - : base(StackFrameKind.GenericTypeIdentifier) + public StackFrameGenericNameNode(StackFrameToken identifier, StackFrameToken arityToken, StackFrameToken arityNumericToken) + : base(identifier, StackFrameKind.GenericTypeIdentifier) { - Identifier = identifier; + Debug.Assert(arityToken.Kind == StackFrameKind.GraveAccentToken); + Debug.Assert(arityNumericToken.Kind == StackFrameKind.NumberToken); + ArityToken = arityToken; ArityNumericToken = arityNumericToken; } @@ -153,14 +204,14 @@ internal override StackFrameNodeOrToken ChildAt(int index) /// /// Represents an array type declaration, such as string[,][] /// - internal sealed class StackFrameArrayTypeExpression : StackFrameExpressionNode + internal sealed class StackFrameArrayTypeExpression : StackFrameNameNode { /// /// The type identifier without the array indicators. /// string[][] /// ^----^ /// - public readonly StackFrameNodeOrToken TypeIdentifier; + public readonly StackFrameNameNode TypeIdentifier; /// /// Each unique array identifier for the type @@ -170,7 +221,7 @@ internal sealed class StackFrameArrayTypeExpression : StackFrameExpressionNode /// public ImmutableArray ArrayExpressions; - public StackFrameArrayTypeExpression(StackFrameNodeOrToken typeIdentifier, ImmutableArray arrayExpressions) : base(StackFrameKind.ArrayTypeExpression) + public StackFrameArrayTypeExpression(StackFrameNameNode typeIdentifier, ImmutableArray arrayExpressions) : base(StackFrameKind.ArrayTypeExpression) { Contract.ThrowIfTrue(arrayExpressions.IsDefaultOrEmpty); TypeIdentifier = typeIdentifier; @@ -180,9 +231,7 @@ public StackFrameArrayTypeExpression(StackFrameNodeOrToken typeIdentifier, Immut internal override int ChildCount => 1 + ArrayExpressions.Length; public override void Accept(IStackFrameNodeVisitor visitor) - { - visitor.Visit(this); - } + => visitor.Visit(this); internal override StackFrameNodeOrToken ChildAt(int index) => index switch @@ -191,11 +240,12 @@ internal override StackFrameNodeOrToken ChildAt(int index) _ => ArrayExpressions[index - 1] }; } - internal sealed class StackFrameArrayRankSpecifier : StackFrameExpressionNode + + internal sealed class StackFrameArrayRankSpecifier : StackFrameNode { public readonly StackFrameToken OpenBracket; public readonly StackFrameToken CloseBracket; - public ImmutableArray CommaTokens; + public readonly ImmutableArray CommaTokens; public StackFrameArrayRankSpecifier(StackFrameToken openBracket, StackFrameToken closeBracket, ImmutableArray commaTokens) : base(StackFrameKind.ArrayExpression) @@ -203,7 +253,7 @@ public StackFrameArrayRankSpecifier(StackFrameToken openBracket, StackFrameToken Contract.ThrowIfTrue(commaTokens.IsDefault); Debug.Assert(openBracket.Kind == StackFrameKind.OpenBracketToken); Debug.Assert(closeBracket.Kind == StackFrameKind.CloseBracketToken); - Debug.Assert(commaTokens.All(t => t.Kind == StackFrameKind.CommaToken)); + Debug.Assert(commaTokens.All(static t => t.Kind == StackFrameKind.CommaToken)); OpenBracket = openBracket; CloseBracket = closeBracket; @@ -233,26 +283,30 @@ internal override StackFrameNodeOrToken ChildAt(int index) /// /// The type argument list for a method declaration. + /// + /// /// Ex: MyType.MyMethod[T, U, V](T t, U u, V v) /// ^----------------------- "[" = Open Token /// ^------^ ------------ "T, U, V" = SeparatedStackFrameNodeList<StackFrameTypeArgumentNode> /// ^-------------- "]" = Close Token + /// + /// /// internal sealed class StackFrameTypeArgumentList : StackFrameNode { public readonly StackFrameToken OpenToken; + public readonly SeparatedStackFrameNodeList TypeArguments; public readonly StackFrameToken CloseToken; - public readonly SeparatedStackFrameNodeList TypeArguments; - public StackFrameTypeArgumentList(StackFrameToken openToken, SeparatedStackFrameNodeList typeArguments, StackFrameToken closeToken) : base(StackFrameKind.TypeArgument) + public StackFrameTypeArgumentList(StackFrameToken openToken, SeparatedStackFrameNodeList typeArguments, StackFrameToken closeToken) : base(StackFrameKind.TypeArgument) { Debug.Assert(openToken.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken); - Debug.Assert(openToken.Kind == StackFrameKind.OpenBracketToken ? closeToken.Kind == StackFrameKind.CloseBracketToken : closeToken.Kind == StackFrameKind.GreaterThanToken); Debug.Assert(typeArguments.Length > 0); + Debug.Assert(openToken.Kind == StackFrameKind.OpenBracketToken ? closeToken.Kind == StackFrameKind.CloseBracketToken : closeToken.Kind == StackFrameKind.GreaterThanToken); OpenToken = openToken; - CloseToken = closeToken; TypeArguments = typeArguments; + CloseToken = closeToken; } internal override int ChildCount => TypeArguments.NodesAndTokens.Length + 2; @@ -282,40 +336,17 @@ internal override StackFrameNodeOrToken ChildAt(int index) } } - internal sealed class StackFrameTypeArgumentNode : StackFrameBaseIdentifierNode - { - public readonly StackFrameToken Identifier; - - internal override int ChildCount => 1; - - public StackFrameTypeArgumentNode(StackFrameToken identifier) - : base(StackFrameKind.TypeIdentifier) - { - Identifier = identifier; - } - - public override void Accept(IStackFrameNodeVisitor visitor) - => visitor.Visit(this); - - internal override StackFrameNodeOrToken ChildAt(int index) - => index switch - { - 0 => Identifier, - _ => throw new InvalidOperationException() - }; - } - internal sealed class StackFrameParameterNode : StackFrameExpressionNode { - public readonly StackFrameNodeOrToken Type; + public readonly StackFrameNameNode Type; public readonly StackFrameToken Identifier; internal override int ChildCount => 2; - public StackFrameParameterNode(StackFrameNodeOrToken type, StackFrameToken identifier) + public StackFrameParameterNode(StackFrameNameNode type, StackFrameToken identifier) : base(StackFrameKind.Parameter) { - Debug.Assert(type.IsNode || !type.Token.IsMissing); + Debug.Assert(identifier.Kind == StackFrameKind.IdentifierToken); Type = type; Identifier = identifier; } @@ -335,18 +366,18 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameParameterList : StackFrameExpressionNode { public readonly StackFrameToken OpenParen; - public readonly StackFrameToken CloseParen; public readonly SeparatedStackFrameNodeList Parameters; + public readonly StackFrameToken CloseParen; - public StackFrameParameterList(StackFrameToken openToken, StackFrameToken closeToken, SeparatedStackFrameNodeList parameters) + public StackFrameParameterList(StackFrameToken openToken, SeparatedStackFrameNodeList parameters, StackFrameToken closeToken) : base(StackFrameKind.ParameterList) { Debug.Assert(openToken.Kind == StackFrameKind.OpenParenToken); Debug.Assert(closeToken.Kind == StackFrameKind.CloseParenToken); OpenParen = openToken; - CloseParen = closeToken; Parameters = parameters; + CloseParen = closeToken; } internal override int ChildCount => 2 + Parameters.NodesAndTokens.Length; @@ -402,10 +433,5 @@ internal override StackFrameNodeOrToken ChildAt(int index) 2 => Line.HasValue ? Line.Value : null, _ => throw new InvalidOperationException() }; - - internal StackFrameFileInformationNode WithLeadingTrivia(StackFrameTrivia inTrivia) - => new(Path.With(leadingTrivia: ImmutableArray.Create(inTrivia)), - Colon, - Line); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.StackFrameParseException.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.StackFrameParseException.cs new file mode 100644 index 0000000000000..3cf93b8d2a2ad --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.StackFrameParseException.cs @@ -0,0 +1,48 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; + + internal partial struct StackFrameParser + { + /// + /// Exception type for when parsing encounters an exceptional state and must halt. This varies based on input, + /// but some examples are + /// * Open type arguments without closing + /// * Missing required identifier (such as parameter with type but no identifier name) + /// * Missing close bracket on array type + /// + private class StackFrameParseException : Exception + { + public StackFrameParseException(StackFrameKind expectedKind, StackFrameNodeOrToken actual) + : this($"Expected {expectedKind} instead of {GetDetails(actual)}") + { + } + + private static string GetDetails(StackFrameNodeOrToken actual) + { + if (actual.IsNode) + { + var node = actual.Node; + return $"'{node.Kind}' at {node.GetSpan().Start}"; + } + else + { + var token = actual.Token; + return $"'{token.VirtualChars.CreateString()}' at {token.GetSpan().Start}"; + } + } + + public StackFrameParseException(string message) + : base(message) + { + } + } + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index ab005d1a490d1..9e5aa7579c0a4 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -3,18 +3,15 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.PooledObjects; -using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { - using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; using StackFrameToken = EmbeddedSyntaxToken; using StackFrameTrivia = EmbeddedSyntaxTrivia; @@ -24,43 +21,16 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame /// defined as a string line in a StackTrace. See https://docs.microsoft.com/en-us/dotnet/api/system.environment.stacktrace for /// more documentation on dotnet stack traces. /// - internal struct StackFrameParser + internal partial struct StackFrameParser { - private class StackFrameParseException : Exception + private StackFrameParser(VirtualCharSequence text) { - public StackFrameParseException(StackFrameKind expectedKind, StackFrameNodeOrToken actual) - : this($"Expected {expectedKind} instead of {GetDetails(actual)}") - { - } - - private static string GetDetails(StackFrameNodeOrToken actual) - { - if (actual.IsNode) - { - var node = actual.Node; - return $"'{node.Kind}' at {node.GetSpan().Start}"; - } - else - { - var token = actual.Token; - return $"'{token.VirtualChars.CreateString()}' at {token.GetSpan().Start}"; - } - } - - public StackFrameParseException(string message) - : base(message) - { - } + _lexer = new(text); } private StackFrameLexer _lexer; private StackFrameToken CurrentCharAsToken() => _lexer.CurrentCharAsToken(); - private StackFrameParser(VirtualCharSequence text) - { - _lexer = new(text); - } - /// /// Given an input text, and set of options, parses out a fully representative syntax tree /// and list of diagnostics. Parsing should always succeed, except in the case of the stack @@ -126,13 +96,13 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameMethodDeclarationNode? TryParseMethodDeclaration() { - var identifierExpression = TryParseIdentifierExpression(scanAtTrivia: true); - if (!(identifierExpression.HasValue && identifierExpression.Value.IsNode)) + var identifierNode = TryParseIdentifierNode(scanAtTrivia: true); + if (identifierNode is null) { return null; } - if (identifierExpression.Value.Node is not StackFrameMemberAccessExpressionNode memberAccessExpression) + if (identifierNode is not StackFrameQualifiedNameNode memberAccessExpression) { return null; } @@ -149,8 +119,8 @@ private StackFrameParser(VirtualCharSequence text) } /// - /// Parses an identifier expression which could either be a or . Combines - /// identifiers that are separated by into . + /// Parses an identifier expression which could either be a or . Combines + /// identifiers that are separated by into . /// /// Identifiers will be parsed for arity but not generic type arguments. /// @@ -160,7 +130,7 @@ private StackFrameParser(VirtualCharSequence text) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) /// - private StackFrameNodeOrToken? TryParseIdentifierExpression(bool scanAtTrivia) + private StackFrameNameNode? TryParseIdentifierNode(bool scanAtTrivia) { var currentIdentifer = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanLeadingWhitespace: true, scanTrailingWhitespace: false); if (!currentIdentifer.HasValue) @@ -168,10 +138,10 @@ private StackFrameParser(VirtualCharSequence text) return null; } - var lhs = TryScanGenericTypeIdentifier(currentIdentifer.Value) - ?? (StackFrameNodeOrToken)currentIdentifer.Value; + StackFrameSimpleNameNode? lhs = TryScanGenericTypeIdentifier(currentIdentifer.Value); + lhs ??= new StackFrameIdentifierNameNode(currentIdentifer.Value); - var memberAccess = TryScanMemberAccessExpression(lhs); + var memberAccess = TryScanQualifiedNameNode(lhs); if (memberAccess is null) { return lhs; @@ -179,7 +149,7 @@ private StackFrameParser(VirtualCharSequence text) while (true) { - var newMemberAccess = TryScanMemberAccessExpression(memberAccess); + var newMemberAccess = TryScanQualifiedNameNode(memberAccess); if (newMemberAccess is null) { return memberAccess; @@ -191,13 +161,10 @@ private StackFrameParser(VirtualCharSequence text) /// /// Given an existing left hand side node or token, which can either be - /// an or + /// an or /// - private StackFrameMemberAccessExpressionNode? TryScanMemberAccessExpression(StackFrameNodeOrToken lhs) + private StackFrameQualifiedNameNode? TryScanQualifiedNameNode(StackFrameNameNode lhs) { - Debug.Assert((lhs.IsNode && lhs.Node is StackFrameMemberAccessExpressionNode) || - lhs.Token.Kind == StackFrameKind.IdentifierToken); - if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) { return null; @@ -209,10 +176,10 @@ private StackFrameParser(VirtualCharSequence text) throw new StackFrameParseException(StackFrameKind.IdentifierToken, CurrentCharAsToken()); } - var rhs = TryScanGenericTypeIdentifier(identifier.Value) - ?? (StackFrameNodeOrToken)identifier.Value; + StackFrameSimpleNameNode? rhs = TryScanGenericTypeIdentifier(identifier.Value); + rhs ??= new StackFrameIdentifierNameNode(identifier.Value); - return new StackFrameMemberAccessExpressionNode(lhs, dotToken, rhs); + return new StackFrameQualifiedNameNode(lhs, dotToken, rhs); } /// @@ -222,7 +189,7 @@ private StackFrameParser(VirtualCharSequence text) /// ^-------------- Grave token /// ^------------- Arity token of "1" /// - private StackFrameGenericTypeIdentifier? TryScanGenericTypeIdentifier(StackFrameToken identifierToken) + private StackFrameGenericNameNode? TryScanGenericTypeIdentifier(StackFrameToken identifierToken) { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { @@ -235,7 +202,7 @@ private StackFrameParser(VirtualCharSequence text) throw new StackFrameParseException(StackFrameKind.NumberToken, CurrentCharAsToken()); } - return new StackFrameGenericTypeIdentifier(identifierToken, graveAccentToken, arity.Value); + return new StackFrameGenericNameNode(identifierToken, graveAccentToken, arity.Value); } /// @@ -265,7 +232,7 @@ private StackFrameParser(VirtualCharSequence text) while (currentIdentifier.HasValue && currentIdentifier.Value.Kind == StackFrameKind.IdentifierToken) { - builder.Add(new StackFrameTypeArgumentNode(currentIdentifier.Value)); + builder.Add(new StackFrameIdentifierNameNode(currentIdentifier.Value)); if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseBracketToken, out var closeBracket)) { @@ -293,12 +260,17 @@ private StackFrameParser(VirtualCharSequence text) currentIdentifier = _lexer.TryScanIdentifier(); } + if (builder.Count == 0) + { + throw new StackFrameParseException(StackFrameKind.TypeArgument, CurrentCharAsToken()); + } + if (closeToken.IsMissing) { - return null; + throw new StackFrameParseException("No close token for type arguments"); } - return new StackFrameTypeArgumentList(openToken, new SeparatedStackFrameNodeList(builder.ToImmutable()), closeToken); + return new StackFrameTypeArgumentList(openToken, new SeparatedStackFrameNodeList(builder.ToImmutable()), closeToken); } /// @@ -317,7 +289,7 @@ private StackFrameParser(VirtualCharSequence text) if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) { - return new StackFrameParameterList(openParen, closeParen, SeparatedStackFrameNodeList.Empty); + return new StackFrameParameterList(openParen, SeparatedStackFrameNodeList.Empty, closeParen); } using var _ = ArrayBuilder.GetInstance(out var builder); @@ -334,7 +306,7 @@ private StackFrameParser(VirtualCharSequence text) } var parameters = new SeparatedStackFrameNodeList(builder.ToImmutable()); - return new StackFrameParameterList(openParen, closeParen, parameters); + return new StackFrameParameterList(openParen, parameters, closeParen); } /// @@ -345,8 +317,8 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameParameterNode ParseParameterNode() { - var typeIdentifier = TryParseIdentifierExpression(scanAtTrivia: false); - if (!typeIdentifier.HasValue) + var typeIdentifier = TryParseIdentifierNode(scanAtTrivia: false); + if (typeIdentifier is null) { throw new StackFrameParseException("Expected type identifier when parsing parameters"); } @@ -354,7 +326,7 @@ private StackFrameParameterNode ParseParameterNode() if (CurrentCharAsToken().Kind == StackFrameKind.OpenBracketToken) { var arrayIdentifiers = ParseArrayIdentifiers(); - typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier.Value, arrayIdentifiers); + typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier, arrayIdentifiers); } var identifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: true, scanTrailingWhitespace: true); @@ -363,7 +335,7 @@ private StackFrameParameterNode ParseParameterNode() throw new StackFrameParseException("Expected a parameter identifier"); } - return new StackFrameParameterNode(typeIdentifier.Value, identifier.Value); + return new StackFrameParameterNode(typeIdentifier, identifier.Value); } /// From 598571c0704a364b8a983e03b6ce26ad0e6be77e Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 27 Oct 2021 13:27:17 -0700 Subject: [PATCH 28/59] Clean up type hierarchy by including declaration node as a type. Remove expression node --- .../StackFrame/StackFrameNodeDefinitions.cs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index 13b33b2917c25..d3e4996241504 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -42,17 +42,14 @@ private SeparatedStackFrameNodeList() : base() public static SeparatedStackFrameNodeList Empty { get; } = new(); } - /// - /// Root of all expression nodes. - /// - internal abstract class StackFrameExpressionNode : StackFrameNode + internal abstract class StackFrameDeclarationNode : StackFrameNode { - protected StackFrameExpressionNode(StackFrameKind kind) : base(kind) + protected StackFrameDeclarationNode(StackFrameKind kind) : base(kind) { } } - internal sealed class StackFrameMethodDeclarationNode : StackFrameNode + internal sealed class StackFrameMethodDeclarationNode : StackFrameDeclarationNode { public readonly StackFrameQualifiedNameNode MemberAccessExpression; public readonly StackFrameTypeArgumentList? TypeArguments; @@ -336,7 +333,7 @@ internal override StackFrameNodeOrToken ChildAt(int index) } } - internal sealed class StackFrameParameterNode : StackFrameExpressionNode + internal sealed class StackFrameParameterNode : StackFrameDeclarationNode { public readonly StackFrameNameNode Type; public readonly StackFrameToken Identifier; @@ -363,7 +360,7 @@ internal override StackFrameNodeOrToken ChildAt(int index) }; } - internal sealed class StackFrameParameterList : StackFrameExpressionNode + internal sealed class StackFrameParameterList : StackFrameNode { public readonly StackFrameToken OpenParen; public readonly SeparatedStackFrameNodeList Parameters; From ad40dfaf02718b2c1d73a6fb18bd895ecb9324e1 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 27 Oct 2021 15:07:32 -0700 Subject: [PATCH 29/59] Remove exception usage from StackFrameParser --- ...ackFrameParser.StackFrameParseException.cs | 48 ----- .../StackFrame/StackFrameParser.cs | 194 +++++++++++------- 2 files changed, 121 insertions(+), 121 deletions(-) delete mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.StackFrameParseException.cs diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.StackFrameParseException.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.StackFrameParseException.cs deleted file mode 100644 index 3cf93b8d2a2ad..0000000000000 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.StackFrameParseException.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; - -namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame -{ - using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; - - internal partial struct StackFrameParser - { - /// - /// Exception type for when parsing encounters an exceptional state and must halt. This varies based on input, - /// but some examples are - /// * Open type arguments without closing - /// * Missing required identifier (such as parameter with type but no identifier name) - /// * Missing close bracket on array type - /// - private class StackFrameParseException : Exception - { - public StackFrameParseException(StackFrameKind expectedKind, StackFrameNodeOrToken actual) - : this($"Expected {expectedKind} instead of {GetDetails(actual)}") - { - } - - private static string GetDetails(StackFrameNodeOrToken actual) - { - if (actual.IsNode) - { - var node = actual.Node; - return $"'{node.Kind}' at {node.GetSpan().Start}"; - } - else - { - var token = actual.Token; - return $"'{token.VirtualChars.CreateString()}' at {token.GetSpan().Start}"; - } - } - - public StackFrameParseException(string message) - : base(message) - { - } - } - } -} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 9e5aa7579c0a4..ceb65f45d1f15 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -23,6 +23,8 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame /// internal partial struct StackFrameParser { + private record struct ParseResult(bool Success, T Value); + private StackFrameParser(VirtualCharSequence text) { _lexer = new(text); @@ -47,11 +49,6 @@ private StackFrameParser(VirtualCharSequence text) { return new StackFrameParser(text).TryParseTree(); } - catch (StackFrameParseException) - { - // Should we report why parsing failed here? - return null; - } catch (InsufficientExecutionStackException) { return null; @@ -69,13 +66,18 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameTree? TryParseTree() { - var methodDeclaration = TryParseMethodDeclaration(); + var (_, methodDeclaration) = TryParseMethodDeclaration(); if (methodDeclaration is null) { return null; } - var fileInformation = TryParseFileInformation(); + var fileInformationResult = TryParseFileInformation(); + if (!fileInformationResult.Success) + { + return null; + } + var remainingTrivia = _lexer.TryScanRemainingTrivia(); var eolToken = CurrentCharAsToken().With(leadingTrivia: remainingTrivia.HasValue ? ImmutableArray.Create(remainingTrivia.Value) : ImmutableArray.Empty); @@ -83,7 +85,7 @@ private StackFrameParser(VirtualCharSequence text) Debug.Assert(_lexer.Position == _lexer.Text.Length); Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); - var root = new StackFrameCompilationUnit(methodDeclaration, fileInformation, eolToken); + var root = new StackFrameCompilationUnit(methodDeclaration, fileInformationResult.Value, eolToken); return new(_lexer.Text, root); } @@ -94,65 +96,88 @@ private StackFrameParser(VirtualCharSequence text) /// /// Ex: [|MyClass.MyMethod(string s)|] /// - private StackFrameMethodDeclarationNode? TryParseMethodDeclaration() + private ParseResult TryParseMethodDeclaration() { - var identifierNode = TryParseIdentifierNode(scanAtTrivia: true); - if (identifierNode is null) + var (success, identifierNode) = TryParseNameNode(scanAtTrivia: true); + if (!success) { - return null; + return new(false, null); } if (identifierNode is not StackFrameQualifiedNameNode memberAccessExpression) { - return null; + return new(true, null); + } + + var typeArgumentsResult = TryParseTypeArguments(); + if (!typeArgumentsResult.Success) + { + return new(false, null); } - var typeArguments = TryParseTypeArguments(); - var arguments = TryParseMethodParameters(); + var argumentsResult = TryParseMethodParameters(); - if (arguments is null) + if (!argumentsResult.Success || argumentsResult.Value is null) { - return null; + return new(false, Value: null); } - return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, arguments); + return new(true, new(memberAccessExpression, typeArgumentsResult.Value, argumentsResult.Value)); } /// - /// Parses an identifier expression which could either be a or . Combines - /// identifiers that are separated by into . + /// Parses a which could either be a or . /// - /// Identifiers will be parsed for arity but not generic type arguments. + /// Nodes will be parsed for arity but not generic type arguments. /// - /// All of the following are valid identifiers, where "$$" marks the parsing starting point, and "[|" + "|]" mark the endpoints of the parsed identifier including trivia + /// All of the following are valid nodes, where "$$" marks the parsing starting point, and "[|" + "|]" mark the endpoints of the parsed node excluding trivia /// * [|$$MyNamespace.MyClass.MyMethod|](string s) - /// * MyClass.MyMethod([|$$string |]s) + /// * MyClass.MyMethod([|$$string|] s) + /// * MyClass.MyMethod([|$$string[]|] s) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) /// - private StackFrameNameNode? TryParseIdentifierNode(bool scanAtTrivia) + private ParseResult TryParseNameNode(bool scanAtTrivia) { var currentIdentifer = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanLeadingWhitespace: true, scanTrailingWhitespace: false); if (!currentIdentifer.HasValue) { - return null; + return new(Success: true, Value: null); } - StackFrameSimpleNameNode? lhs = TryScanGenericTypeIdentifier(currentIdentifer.Value); + var identifierParseResult = TryScanGenericTypeIdentifier(currentIdentifer.Value); + if (!identifierParseResult.Success) + { + return new(false, null); + } + + StackFrameSimpleNameNode? lhs = identifierParseResult.Value; lhs ??= new StackFrameIdentifierNameNode(currentIdentifer.Value); - var memberAccess = TryScanQualifiedNameNode(lhs); + var parseResult = TryScanQualifiedNameNode(lhs); + if (!parseResult.Success) + { + return new(false, null); + } + + var memberAccess = parseResult.Value; if (memberAccess is null) { - return lhs; + return new(true, lhs); } while (true) { - var newMemberAccess = TryScanQualifiedNameNode(memberAccess); + parseResult = TryScanQualifiedNameNode(memberAccess); + if (!parseResult.Success) + { + return new(false, null); + } + + var newMemberAccess = parseResult.Value; if (newMemberAccess is null) { - return memberAccess; + return new(true, memberAccess); } memberAccess = newMemberAccess; @@ -163,46 +188,55 @@ private StackFrameParser(VirtualCharSequence text) /// Given an existing left hand side node or token, which can either be /// an or /// - private StackFrameQualifiedNameNode? TryScanQualifiedNameNode(StackFrameNameNode lhs) + private ParseResult TryScanQualifiedNameNode(StackFrameNameNode lhs) { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) { - return null; + return new(true, null); } var identifier = _lexer.TryScanIdentifier(); if (!identifier.HasValue) { - throw new StackFrameParseException(StackFrameKind.IdentifierToken, CurrentCharAsToken()); + return new(false, null); + } + + (var success, StackFrameSimpleNameNode? rhs) = TryScanGenericTypeIdentifier(identifier.Value); + if (!success) + { + return new(false, null); } - StackFrameSimpleNameNode? rhs = TryScanGenericTypeIdentifier(identifier.Value); rhs ??= new StackFrameIdentifierNameNode(identifier.Value); - return new StackFrameQualifiedNameNode(lhs, dotToken, rhs); + return new(true, new(lhs, dotToken, rhs)); } /// /// Given an identifier, attempts to parse the type identifier arity for it. + /// + /// /// ex: MyNamespace.MyClass`1.MyMethod() /// ^--------------------- MyClass would be the identifier passed in /// ^-------------- Grave token /// ^------------- Arity token of "1" + /// + /// /// - private StackFrameGenericNameNode? TryScanGenericTypeIdentifier(StackFrameToken identifierToken) + private ParseResult TryScanGenericTypeIdentifier(StackFrameToken identifierToken) { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { - return null; + return new(Success: true, null); } var arity = _lexer.TryScanNumbers(); if (!arity.HasValue) { - throw new StackFrameParseException(StackFrameKind.NumberToken, CurrentCharAsToken()); + return new(Success: false, Value: null); } - return new StackFrameGenericNameNode(identifierToken, graveAccentToken, arity.Value); + return new(Success: true, new(identifierToken, graveAccentToken, arity.Value)); } /// @@ -215,13 +249,13 @@ private StackFrameParser(VirtualCharSequence text) /// Assumes the identifier "MyMethod" has already been parsed, and the type arguments will need to be parsed. /// Returns null if no type arguments are found, and throw a if they are malformed /// - private StackFrameTypeArgumentList? TryParseTypeArguments() + private ParseResult TryParseTypeArguments() { if (!_lexer.ScanCurrentCharAsTokenIfMatch( kind => kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken, out var openToken)) { - return null; + return new(true, null); } var useCloseBracket = openToken.Kind is StackFrameKind.OpenBracketToken; @@ -242,14 +276,14 @@ private StackFrameParser(VirtualCharSequence text) break; } - throw new StackFrameParseException(StackFrameKind.GreaterThanToken, closeBracket); + return new(false, null); } if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GreaterThanToken, out var greaterThanToken)) { if (useCloseBracket) { - throw new StackFrameParseException(StackFrameKind.CloseBracketToken, greaterThanToken); + return new(false, null); } closeToken = greaterThanToken; @@ -262,51 +296,60 @@ private StackFrameParser(VirtualCharSequence text) if (builder.Count == 0) { - throw new StackFrameParseException(StackFrameKind.TypeArgument, CurrentCharAsToken()); + return new(false, null); } if (closeToken.IsMissing) { - throw new StackFrameParseException("No close token for type arguments"); + return new(false, null); } - return new StackFrameTypeArgumentList(openToken, new SeparatedStackFrameNodeList(builder.ToImmutable()), closeToken); + var separatedList = new SeparatedStackFrameNodeList(builder.ToImmutable()); + return new(Success: true, Value: new(openToken, separatedList, closeToken)); } /// /// MyNamespace.MyClass.MyMethod[|(string s1, string s2, int i1)|] - /// Takes parameter declarations from method text and parses them into a . - /// - /// Returns null in cases where the opening paren is not found. Throws if the - /// opening paren exists but the remaining parameter definitions are malformed. + /// Takes parameter declarations from method text and parses them into a . /// - private StackFrameParameterList? TryParseMethodParameters() + private ParseResult TryParseMethodParameters() { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.OpenParenToken, scanTrailingWhitespace: true, out var openParen)) { - return null; + return new(true, null); } if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) { - return new StackFrameParameterList(openParen, SeparatedStackFrameNodeList.Empty, closeParen); + return new(Success: true, Value: new(openParen, SeparatedStackFrameNodeList.Empty, closeParen)); } using var _ = ArrayBuilder.GetInstance(out var builder); - builder.Add(ParseParameterNode()); + var (success, parameterNode) = ParseParameterNode(); + if (!success) + { + return new(false, null); + } + while (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, out var commaToken)) { builder.Add(commaToken); - builder.Add(ParseParameterNode()); + (success, parameterNode) = ParseParameterNode(); + if (!success) + { + return new(false, null); + } + + builder.Add(parameterNode); } if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out closeParen)) { - throw new StackFrameParseException(StackFrameKind.CloseParenToken, CurrentCharAsToken()); + return new(true, null); } var parameters = new SeparatedStackFrameNodeList(builder.ToImmutable()); - return new StackFrameParameterList(openParen, parameters, closeParen); + return new(true, new(openParen, parameters, closeParen)); } /// @@ -315,27 +358,32 @@ private StackFrameParser(VirtualCharSequence text) /// ^--------------^ -- Type = "System.String[]" /// ^-- Identifier = "s" /// - private StackFrameParameterNode ParseParameterNode() + private ParseResult ParseParameterNode() { - var typeIdentifier = TryParseIdentifierNode(scanAtTrivia: false); - if (typeIdentifier is null) + var (success, typeIdentifier) = TryParseNameNode(scanAtTrivia: false); + if (!success || typeIdentifier is null) { - throw new StackFrameParseException("Expected type identifier when parsing parameters"); + return new(false, null); } if (CurrentCharAsToken().Kind == StackFrameKind.OpenBracketToken) { - var arrayIdentifiers = ParseArrayIdentifiers(); + (success, var arrayIdentifiers) = ParseArrayRankSpecifiers(); + if (!success || arrayIdentifiers.IsDefault) + { + return new(false, null); + } + typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier, arrayIdentifiers); } var identifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: true, scanTrailingWhitespace: true); if (!identifier.HasValue) { - throw new StackFrameParseException("Expected a parameter identifier"); + return new(false, null); } - return new StackFrameParameterNode(typeIdentifier, identifier.Value); + return new(true, new(typeIdentifier, identifier.Value)); } /// @@ -345,7 +393,7 @@ private StackFrameParameterNode ParseParameterNode() /// 0: "[,] /// 1: "[]" /// - private ImmutableArray ParseArrayIdentifiers() + private ParseResult> ParseArrayRankSpecifiers() { using var _ = ArrayBuilder.GetInstance(out var builder); using var _1 = ArrayBuilder.GetInstance(out var commaBuilder); @@ -354,7 +402,7 @@ private ImmutableArray ParseArrayIdentifiers() { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.OpenBracketToken, scanTrailingWhitespace: true, out var openBracket)) { - return builder.ToImmutable(); + return new(true, builder.ToImmutable()); } commaBuilder.Clear(); @@ -365,7 +413,7 @@ private ImmutableArray ParseArrayIdentifiers() if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseBracketToken, scanTrailingWhitespace: true, out var closeBracket)) { - throw new StackFrameParseException(StackFrameKind.CloseBracketToken, CurrentCharAsToken()); + return new(false, default); } builder.Add(new StackFrameArrayRankSpecifier(openBracket, closeBracket, commaBuilder.ToImmutable())); @@ -381,22 +429,22 @@ private ImmutableArray ParseArrayIdentifiers() /// Can return if only a path is available but not line numbers, but throws if the value after the path is a colon as the expectation /// is that line number should follow. /// - private StackFrameFileInformationNode? TryParseFileInformation() + private ParseResult TryParseFileInformation() { var path = _lexer.TryScanPath(); if (!path.HasValue) { - return null; + return new(true, null); } if (path.Value.Kind != StackFrameKind.PathToken) { - throw new StackFrameParseException("'In ' trivia was present "); + return new(false, null); } if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.ColonToken, out var colonToken)) { - return new(path.Value, null, null); + return new(true, new(path.Value, null, null)); } var lineNumber = _lexer.TryScanLineNumber(); @@ -405,10 +453,10 @@ private ImmutableArray ParseArrayIdentifiers() // to bail in error and consider this malformed. if (!lineNumber.HasValue || lineNumber.Value.Kind != StackFrameKind.NumberToken) { - throw new StackFrameParseException("Expected line number to exist after colon token"); + return new(false, null); } - return new(path.Value, colonToken, lineNumber.Value); + return new(true, new(path.Value, colonToken, lineNumber.Value)); } } } From 1beb9dcb47e6e68e9313498d3626532436b789b6 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 27 Oct 2021 15:44:22 -0700 Subject: [PATCH 30/59] Fix tests --- .../StackFrame/StackFrameParser.cs | 22 ++++++++++--------- .../Common/EmbeddedSaparatedSyntaxNodeList.cs | 5 +---- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index ceb65f45d1f15..8517a72add744 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -119,7 +119,7 @@ private StackFrameParser(VirtualCharSequence text) if (!argumentsResult.Success || argumentsResult.Value is null) { - return new(false, Value: null); + return new(false, null); } return new(true, new(memberAccessExpression, typeArgumentsResult.Value, argumentsResult.Value)); @@ -142,7 +142,7 @@ private StackFrameParser(VirtualCharSequence text) var currentIdentifer = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanLeadingWhitespace: true, scanTrailingWhitespace: false); if (!currentIdentifer.HasValue) { - return new(Success: true, Value: null); + return new(true, null); } var identifierParseResult = TryScanGenericTypeIdentifier(currentIdentifer.Value); @@ -227,16 +227,16 @@ private StackFrameParser(VirtualCharSequence text) { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { - return new(Success: true, null); + return new(true, null); } var arity = _lexer.TryScanNumbers(); if (!arity.HasValue) { - return new(Success: false, Value: null); + return new(false, null); } - return new(Success: true, new(identifierToken, graveAccentToken, arity.Value)); + return new(true, new(identifierToken, graveAccentToken, arity.Value)); } /// @@ -305,7 +305,7 @@ private StackFrameParser(VirtualCharSequence text) } var separatedList = new SeparatedStackFrameNodeList(builder.ToImmutable()); - return new(Success: true, Value: new(openToken, separatedList, closeToken)); + return new(true, new(openToken, separatedList, closeToken)); } /// @@ -321,21 +321,23 @@ private StackFrameParser(VirtualCharSequence text) if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) { - return new(Success: true, Value: new(openParen, SeparatedStackFrameNodeList.Empty, closeParen)); + return new(true, new(openParen, SeparatedStackFrameNodeList.Empty, closeParen)); } - using var _ = ArrayBuilder.GetInstance(out var builder); var (success, parameterNode) = ParseParameterNode(); - if (!success) + if (!success || parameterNode is null) { return new(false, null); } + using var _ = ArrayBuilder.GetInstance(out var builder); + builder.Add(parameterNode); + while (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, out var commaToken)) { builder.Add(commaToken); (success, parameterNode) = ParseParameterNode(); - if (!success) + if (!success || parameterNode is null) { return new(false, null); } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs index aafaf6f61d307..0a7165a805f81 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs @@ -16,11 +16,8 @@ internal abstract class EmbeddedSeparatedSyntaxNodeList> nodesAndTokens) { - Contract.ThrowIfTrue(nodesAndTokens.IsDefaultOrEmpty); + Contract.ThrowIfTrue(nodesAndTokens.IsDefault); NodesAndTokens = nodesAndTokens; - - // Verify size constraints for not empty list. If empty is desired the Empty static field should - // be used instead to retrieve an empty EmbeddedSeparatedSyntaxNodeList Verify(); var allLength = NodesAndTokens.Length; From cba32de7e4af5b5ffa008b7b7addea32d4c7be2d Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 27 Oct 2021 16:11:17 -0700 Subject: [PATCH 31/59] Fix spelling --- .../Compiler/Core/CompilerExtensions.projitems | 2 +- ...atedSyntaxNodeList.cs => EmbeddedSeparatedSyntaxNodeList.cs} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/{EmbeddedSaparatedSyntaxNodeList.cs => EmbeddedSeparatedSyntaxNodeList.cs} (100%) diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/CompilerExtensions.projitems b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/CompilerExtensions.projitems index 270c7b1bace94..d2868a4b2cbe8 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/CompilerExtensions.projitems +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/CompilerExtensions.projitems @@ -192,7 +192,7 @@ - + diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs similarity index 100% rename from src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSaparatedSyntaxNodeList.cs rename to src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs From ab1a662ff5698953af2da3c5ad29026fdc71b362 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 27 Oct 2021 16:30:10 -0700 Subject: [PATCH 32/59] Add required overrides --- .../EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs | 2 +- .../EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs | 10 +++++++++- .../VirtualChars/VirtualCharSequence.cs | 7 +++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs index 1a99a2b986bb5..705d9efc3be6a 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs @@ -78,7 +78,7 @@ public TextSpan GetSpan() && t1.LeadingTrivia.IsDefault == t2.LeadingTrivia.IsDefault && (t1.LeadingTrivia.IsDefault || t1.LeadingTrivia.SequenceEqual(t2.LeadingTrivia)) && t1.VirtualChars == t2.VirtualChars - && t1.TrailingTrivia.IsDefault == t2.TrailingTrivia.IsDefault + && t1.TrailingTrivia.IsDefault == t2.TrailingTrivia.IsDefault && (t1.TrailingTrivia.IsDefault || t1.TrailingTrivia.SequenceEqual(t2.TrailingTrivia)) && t1.Diagnostics.IsDefault == t2.Diagnostics.IsDefault && (t1.Diagnostics.IsDefault || t1.Diagnostics.SequenceEqual(t2.Diagnostics)) diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs index bfe4bca006d91..3c354e782d272 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs @@ -5,6 +5,7 @@ using System.Collections.Immutable; using System.Diagnostics; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; +using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.Common { @@ -30,7 +31,14 @@ public EmbeddedSyntaxTrivia(TSyntaxKind kind, VirtualCharSequence virtualChars, Diagnostics = diagnostics; } - public static bool operator ==(EmbeddedSyntaxTrivia t1, EmbeddedSyntaxTrivia t2) + public override bool Equals(object? obj) + => obj is EmbeddedSyntaxTrivia trivia && this == trivia; + + public override int GetHashCode() + => Hash.Combine(Kind.GetHashCode(), + Hash.Combine(VirtualChars.GetHashCode(), Diagnostics.GetHashCode())); + + public static bool operator ==(EmbeddedSyntaxTrivia t1, EmbeddedSyntaxTrivia t2) => t1.Kind.Equals(t2.Kind) && t1.VirtualChars.Equals(t2.VirtualChars); diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs index fb5abe07e4cee..f4279b38a4d60 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs @@ -7,6 +7,7 @@ using System.Diagnostics; using Microsoft.CodeAnalysis.PooledObjects; using Microsoft.CodeAnalysis.Text; +using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars { @@ -145,6 +146,12 @@ public static VirtualCharSequence FromBounds( TextSpan.FromBounds(chars1._span.Start, chars2._span.End)); } + public override bool Equals(object? obj) + => obj is VirtualCharSequence seq && this == seq; + + public override int GetHashCode() + => Hash.Combine(_leafCharacters.GetHashCode(), _span.GetHashCode()); + public static bool operator ==(VirtualCharSequence v1, VirtualCharSequence v2) => v1.Length == v2.Length && v1._span == v2._span From eeefa4810e947e5fa4267f4843c9d44f21f25f12 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Thu, 28 Oct 2021 12:04:04 -0700 Subject: [PATCH 33/59] Remove StackFrameParseException reference in comment --- .../Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 8517a72add744..4a663f5a41366 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -246,8 +246,7 @@ private StackFrameParser(VirtualCharSequence text) /// ex: MyNamespace.MyClass.MyMethod[T](T t) /// ex: MyNamespace.MyClass.MyMethod<T<(T t) /// - /// Assumes the identifier "MyMethod" has already been parsed, and the type arguments will need to be parsed. - /// Returns null if no type arguments are found, and throw a if they are malformed + /// Assumes the identifier "MyMethod" has already been parsed, and the type arguments will need to be parsed. /// private ParseResult TryParseTypeArguments() { From 3958af9fdb415635ae33fd9612f65518b784506c Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Thu, 28 Oct 2021 17:12:00 -0700 Subject: [PATCH 34/59] PR feedback --- .../StackFrame/StackFrameSyntaxFactory.cs | 10 +- .../StackFrame/IStackFrameNodeVisitor.cs | 2 +- .../StackFrame/StackFrameKind.cs | 1 + .../StackFrame/StackFrameLexer.cs | 3 +- .../StackFrame/StackFrameNodeDefinitions.cs | 26 +-- .../StackFrameParser.ParseResult.cs | 36 ++++ .../StackFrame/StackFrameParser.cs | 167 ++++++++---------- .../Common/EmbeddedSeparatedSyntaxNodeList.cs | 13 +- 8 files changed, 143 insertions(+), 115 deletions(-) create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 9072772f8dec9..a8397c78134cf 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -57,21 +57,21 @@ public static ImmutableArray CreateTriviaArray(params string[] public static readonly StackFrameParameterList EmptyParams = ParameterList(OpenParenToken, CloseParenToken); - public static StackFrameParameterNode Parameter(StackFrameNameNode type, StackFrameToken identifier) + public static StackFrameParameterDeclarationNode Parameter(StackFrameNameNode type, StackFrameToken identifier) => new(type, identifier); - public static StackFrameParameterList ParameterList(params StackFrameParameterNode[] parameters) + public static StackFrameParameterList ParameterList(params StackFrameParameterDeclarationNode[] parameters) => ParameterList(OpenParenToken, CloseParenToken, parameters); - public static StackFrameParameterList ParameterList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameParameterNode[] parameters) + public static StackFrameParameterList ParameterList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameParameterDeclarationNode[] parameters) { var separatedList = parameters.Length == 0 - ? SeparatedStackFrameNodeList.Empty + ? SeparatedStackFrameNodeList.Empty : new(CommaSeparateList(parameters)); return new(openToken, separatedList, closeToken); - static ImmutableArray CommaSeparateList(StackFrameParameterNode[] parameters) + static ImmutableArray CommaSeparateList(StackFrameParameterDeclarationNode[] parameters) { var builder = ImmutableArray.CreateBuilder(); builder.Add(parameters[0]); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index dca069eb7d836..b7ee6c4586289 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -16,6 +16,6 @@ internal interface IStackFrameNodeVisitor void Visit(StackFrameArrayRankSpecifier node); void Visit(StackFrameFileInformationNode node); void Visit(StackFrameArrayTypeExpression node); - void Visit(StackFrameParameterNode node); + void Visit(StackFrameParameterDeclarationNode node); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 361e34f413abd..51cbcada7a902 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -44,6 +44,7 @@ internal enum StackFrameKind ForwardSlashToken, IdentifierToken, PathToken, + InvalidPathToken, NumberToken, // Trivia diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 74a3d42c2174c..36182c9672613 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -158,6 +158,7 @@ public bool ScanCurrentCharAsTokenIfMatch(Func matchFn, ou public StackFrameTrivia? TryScanAtTrivia() // TODO: Handle multiple languages? Right now we're going to only parse english => TryScanStringTrivia("at ", StackFrameKind.AtTrivia); + public StackFrameTrivia? TryScanInTrivia() // TODO: Handle multiple languages? Right now we're going to only parse english => TryScanStringTrivia(" in ", StackFrameKind.InTrivia); @@ -213,7 +214,7 @@ public bool ScanCurrentCharAsTokenIfMatch(Func matchFn, ou if (startPosition == Position) { - return CreateToken(StackFrameKind.None, ImmutableArray.Create(inTrivia.Value), VirtualCharSequence.Empty); + return CreateToken(StackFrameKind.InvalidPathToken, ImmutableArray.Create(inTrivia.Value), VirtualCharSequence.Empty); } return CreateToken(StackFrameKind.PathToken, ImmutableArray.Create(inTrivia.Value), GetSubSequenceToCurrentPos(startPosition)); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index d3e4996241504..e074a5704ce7b 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -169,20 +169,20 @@ internal sealed class StackFrameGenericNameNode : StackFrameSimpleNameNode /// /// The "`" token in arity identifiers. Must be /// - public readonly StackFrameToken ArityToken; + public readonly StackFrameToken GraveAccentToken; - public readonly StackFrameToken ArityNumericToken; + public readonly StackFrameToken NumberToken; internal override int ChildCount => 3; - public StackFrameGenericNameNode(StackFrameToken identifier, StackFrameToken arityToken, StackFrameToken arityNumericToken) + public StackFrameGenericNameNode(StackFrameToken identifier, StackFrameToken graveAccentToken, StackFrameToken numberToken) : base(identifier, StackFrameKind.GenericTypeIdentifier) { - Debug.Assert(arityToken.Kind == StackFrameKind.GraveAccentToken); - Debug.Assert(arityNumericToken.Kind == StackFrameKind.NumberToken); + Debug.Assert(graveAccentToken.Kind == StackFrameKind.GraveAccentToken); + Debug.Assert(numberToken.Kind == StackFrameKind.NumberToken); - ArityToken = arityToken; - ArityNumericToken = arityNumericToken; + GraveAccentToken = graveAccentToken; + NumberToken = numberToken; } public override void Accept(IStackFrameNodeVisitor visitor) @@ -192,8 +192,8 @@ internal override StackFrameNodeOrToken ChildAt(int index) => index switch { 0 => Identifier, - 1 => ArityToken, - 2 => ArityNumericToken, + 1 => GraveAccentToken, + 2 => NumberToken, _ => throw new InvalidOperationException() }; } @@ -333,14 +333,14 @@ internal override StackFrameNodeOrToken ChildAt(int index) } } - internal sealed class StackFrameParameterNode : StackFrameDeclarationNode + internal sealed class StackFrameParameterDeclarationNode : StackFrameDeclarationNode { public readonly StackFrameNameNode Type; public readonly StackFrameToken Identifier; internal override int ChildCount => 2; - public StackFrameParameterNode(StackFrameNameNode type, StackFrameToken identifier) + public StackFrameParameterDeclarationNode(StackFrameNameNode type, StackFrameToken identifier) : base(StackFrameKind.Parameter) { Debug.Assert(identifier.Kind == StackFrameKind.IdentifierToken); @@ -363,10 +363,10 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameParameterList : StackFrameNode { public readonly StackFrameToken OpenParen; - public readonly SeparatedStackFrameNodeList Parameters; + public readonly SeparatedStackFrameNodeList Parameters; public readonly StackFrameToken CloseParen; - public StackFrameParameterList(StackFrameToken openToken, SeparatedStackFrameNodeList parameters, StackFrameToken closeToken) + public StackFrameParameterList(StackFrameToken openToken, SeparatedStackFrameNodeList parameters, StackFrameToken closeToken) : base(StackFrameKind.ParameterList) { Debug.Assert(openToken.Kind == StackFrameKind.OpenParenToken); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs new file mode 100644 index 0000000000000..1d88ff042ac2d --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + internal partial struct StackFrameParser + { + private struct ParseResult + { + public readonly bool Success; + public readonly T? Value; + + public static readonly ParseResult Abort = new(false, default); + public static readonly ParseResult Empty = new(true, default); + + public ParseResult(T? value) + : this(true, value) + { } + + private ParseResult(bool success, T? value) + { + Success = success; + Value = value; + } + + public void Deconstruct(out bool success, out T? value) + { + success = Success; + value = Value; + } + } + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 4a663f5a41366..0f34fae93071a 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -23,8 +23,6 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame /// internal partial struct StackFrameParser { - private record struct ParseResult(bool Success, T Value); - private StackFrameParser(VirtualCharSequence text) { _lexer = new(text); @@ -96,33 +94,32 @@ private StackFrameParser(VirtualCharSequence text) /// /// Ex: [|MyClass.MyMethod(string s)|] /// - private ParseResult TryParseMethodDeclaration() + private ParseResult TryParseMethodDeclaration() { var (success, identifierNode) = TryParseNameNode(scanAtTrivia: true); if (!success) { - return new(false, null); + return ParseResult.Abort; } if (identifierNode is not StackFrameQualifiedNameNode memberAccessExpression) { - return new(true, null); + return ParseResult.Abort; } var typeArgumentsResult = TryParseTypeArguments(); if (!typeArgumentsResult.Success) { - return new(false, null); + return ParseResult.Abort; } - var argumentsResult = TryParseMethodParameters(); - + var argumentsResult = ParseMethodParameters(); if (!argumentsResult.Success || argumentsResult.Value is null) { - return new(false, null); + return ParseResult.Abort; } - return new(true, new(memberAccessExpression, typeArgumentsResult.Value, argumentsResult.Value)); + return new(new(memberAccessExpression, typeArgumentsResult.Value, argumentsResult.Value)); } /// @@ -137,47 +134,49 @@ private StackFrameParser(VirtualCharSequence text) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) /// - private ParseResult TryParseNameNode(bool scanAtTrivia) + private ParseResult TryParseNameNode(bool scanAtTrivia) { var currentIdentifer = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanLeadingWhitespace: true, scanTrailingWhitespace: false); if (!currentIdentifer.HasValue) { - return new(true, null); + // If we can't parse an identifier, no need to abort the parser entirely here. + // The caller should determine if an identifier is required or not. + return ParseResult.Empty; } var identifierParseResult = TryScanGenericTypeIdentifier(currentIdentifer.Value); if (!identifierParseResult.Success) { - return new(false, null); + return ParseResult.Abort; } - StackFrameSimpleNameNode? lhs = identifierParseResult.Value; - lhs ??= new StackFrameIdentifierNameNode(currentIdentifer.Value); + RoslynDebug.AssertNotNull(identifierParseResult.Value); + var lhs = identifierParseResult.Value; - var parseResult = TryScanQualifiedNameNode(lhs); + var parseResult = TryParseQualifiedName(lhs); if (!parseResult.Success) { - return new(false, null); + return ParseResult.Abort; } var memberAccess = parseResult.Value; if (memberAccess is null) { - return new(true, lhs); + return new(lhs); } while (true) { - parseResult = TryScanQualifiedNameNode(memberAccess); + parseResult = TryParseQualifiedName(memberAccess); if (!parseResult.Success) { - return new(false, null); + return ParseResult.Abort; } var newMemberAccess = parseResult.Value; if (newMemberAccess is null) { - return new(true, memberAccess); + return new(memberAccess); } memberAccess = newMemberAccess; @@ -188,28 +187,27 @@ private StackFrameParser(VirtualCharSequence text) /// Given an existing left hand side node or token, which can either be /// an or /// - private ParseResult TryScanQualifiedNameNode(StackFrameNameNode lhs) + private ParseResult TryParseQualifiedName(StackFrameNameNode lhs) { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) { - return new(true, null); + return ParseResult.Empty; } var identifier = _lexer.TryScanIdentifier(); if (!identifier.HasValue) { - return new(false, null); + return ParseResult.Abort; } - (var success, StackFrameSimpleNameNode? rhs) = TryScanGenericTypeIdentifier(identifier.Value); + var (success, rhs) = TryScanGenericTypeIdentifier(identifier.Value); if (!success) { - return new(false, null); + return ParseResult.Abort; } - rhs ??= new StackFrameIdentifierNameNode(identifier.Value); - - return new(true, new(lhs, dotToken, rhs)); + RoslynDebug.AssertNotNull(rhs); + return new(new(lhs, dotToken, rhs)); } /// @@ -223,20 +221,20 @@ private StackFrameParser(VirtualCharSequence text) /// /// /// - private ParseResult TryScanGenericTypeIdentifier(StackFrameToken identifierToken) + private ParseResult TryScanGenericTypeIdentifier(StackFrameToken identifierToken) { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { - return new(true, null); + return new(new StackFrameIdentifierNameNode(identifierToken)); } var arity = _lexer.TryScanNumbers(); if (!arity.HasValue) { - return new(false, null); + return ParseResult.Abort; } - return new(true, new(identifierToken, graveAccentToken, arity.Value)); + return new(new StackFrameGenericNameNode(identifierToken, graveAccentToken, arity.Value)); } /// @@ -248,16 +246,18 @@ private StackFrameParser(VirtualCharSequence text) /// /// Assumes the identifier "MyMethod" has already been parsed, and the type arguments will need to be parsed. /// - private ParseResult TryParseTypeArguments() + private ParseResult TryParseTypeArguments() { if (!_lexer.ScanCurrentCharAsTokenIfMatch( kind => kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken, out var openToken)) { - return new(true, null); + return ParseResult.Empty; } - var useCloseBracket = openToken.Kind is StackFrameKind.OpenBracketToken; + var closeBrackeKind = openToken.Kind is StackFrameKind.OpenBracketToken + ? StackFrameKind.CloseBracketToken + : StackFrameKind.GreaterThanToken; using var _ = ArrayBuilder.GetInstance(out var builder); var currentIdentifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: true, scanTrailingWhitespace: true); @@ -267,104 +267,92 @@ private StackFrameParser(VirtualCharSequence text) { builder.Add(new StackFrameIdentifierNameNode(currentIdentifier.Value)); - if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseBracketToken, out var closeBracket)) + if (_lexer.ScanCurrentCharAsTokenIfMatch(closeBrackeKind, out closeToken)) { - if (useCloseBracket) - { - closeToken = closeBracket; - break; - } - - return new(false, null); + break; } - if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GreaterThanToken, out var greaterThanToken)) + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, out var commaToken)) { - if (useCloseBracket) - { - return new(false, null); - } - - closeToken = greaterThanToken; - break; + return ParseResult.Abort; } - builder.Add(CurrentCharAsToken()); + builder.Add(commaToken); currentIdentifier = _lexer.TryScanIdentifier(); } if (builder.Count == 0) { - return new(false, null); + return ParseResult.Abort; } if (closeToken.IsMissing) { - return new(false, null); + return ParseResult.Abort; } var separatedList = new SeparatedStackFrameNodeList(builder.ToImmutable()); - return new(true, new(openToken, separatedList, closeToken)); + return new(new(openToken, separatedList, closeToken)); } /// /// MyNamespace.MyClass.MyMethod[|(string s1, string s2, int i1)|] /// Takes parameter declarations from method text and parses them into a . /// - private ParseResult TryParseMethodParameters() + private ParseResult ParseMethodParameters() { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.OpenParenToken, scanTrailingWhitespace: true, out var openParen)) { - return new(true, null); + return ParseResult.Abort; } if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) { - return new(true, new(openParen, SeparatedStackFrameNodeList.Empty, closeParen)); - } - - var (success, parameterNode) = ParseParameterNode(); - if (!success || parameterNode is null) - { - return new(false, null); + return new(new(openParen, SeparatedStackFrameNodeList.Empty, closeParen)); } using var _ = ArrayBuilder.GetInstance(out var builder); - builder.Add(parameterNode); - while (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, out var commaToken)) + while (true) { - builder.Add(commaToken); - (success, parameterNode) = ParseParameterNode(); - if (!success || parameterNode is null) + var (success, parameterNode) = ParseParameterNode(); + if (!success) { - return new(false, null); + return ParseResult.Abort; } + RoslynDebug.AssertNotNull(parameterNode); builder.Add(parameterNode); + + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, out var commaToken)) + { + break; + } + + builder.Add(commaToken); } if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out closeParen)) { - return new(true, null); + return ParseResult.Empty; } - var parameters = new SeparatedStackFrameNodeList(builder.ToImmutable()); - return new(true, new(openParen, parameters, closeParen)); + var parameters = new SeparatedStackFrameNodeList(builder.ToImmutable()); + return new(new(openParen, parameters, closeParen)); } /// - /// Parses a by parsing identifiers first representing the type and then the parameter identifier. + /// Parses a by parsing identifiers first representing the type and then the parameter identifier. /// Ex: System.String[] s /// ^--------------^ -- Type = "System.String[]" /// ^-- Identifier = "s" /// - private ParseResult ParseParameterNode() + private ParseResult ParseParameterNode() { var (success, typeIdentifier) = TryParseNameNode(scanAtTrivia: false); if (!success || typeIdentifier is null) { - return new(false, null); + return ParseResult.Abort; } if (CurrentCharAsToken().Kind == StackFrameKind.OpenBracketToken) @@ -372,7 +360,7 @@ private StackFrameParser(VirtualCharSequence text) (success, var arrayIdentifiers) = ParseArrayRankSpecifiers(); if (!success || arrayIdentifiers.IsDefault) { - return new(false, null); + return ParseResult.Abort; } typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier, arrayIdentifiers); @@ -381,10 +369,10 @@ private StackFrameParser(VirtualCharSequence text) var identifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: true, scanTrailingWhitespace: true); if (!identifier.HasValue) { - return new(false, null); + return ParseResult.Abort; } - return new(true, new(typeIdentifier, identifier.Value)); + return new(new(typeIdentifier, identifier.Value)); } /// @@ -403,10 +391,9 @@ private ParseResult> ParseArrayRank { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.OpenBracketToken, scanTrailingWhitespace: true, out var openBracket)) { - return new(true, builder.ToImmutable()); + return new(builder.ToImmutable()); } - commaBuilder.Clear(); while (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, scanTrailingWhitespace: true, out var commaToken)) { commaBuilder.Add(commaToken); @@ -414,10 +401,10 @@ private ParseResult> ParseArrayRank if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseBracketToken, scanTrailingWhitespace: true, out var closeBracket)) { - return new(false, default); + return ParseResult>.Abort; } - builder.Add(new StackFrameArrayRankSpecifier(openBracket, closeBracket, commaBuilder.ToImmutable())); + builder.Add(new StackFrameArrayRankSpecifier(openBracket, closeBracket, commaBuilder.ToImmutableAndClear())); } throw ExceptionUtilities.Unreachable; @@ -430,22 +417,22 @@ private ParseResult> ParseArrayRank /// Can return if only a path is available but not line numbers, but throws if the value after the path is a colon as the expectation /// is that line number should follow. /// - private ParseResult TryParseFileInformation() + private ParseResult TryParseFileInformation() { var path = _lexer.TryScanPath(); if (!path.HasValue) { - return new(true, null); + return ParseResult.Empty; } if (path.Value.Kind != StackFrameKind.PathToken) { - return new(false, null); + return ParseResult.Abort; } if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.ColonToken, out var colonToken)) { - return new(true, new(path.Value, null, null)); + return new(new(path.Value, null, null)); } var lineNumber = _lexer.TryScanLineNumber(); @@ -454,10 +441,10 @@ private ParseResult> ParseArrayRank // to bail in error and consider this malformed. if (!lineNumber.HasValue || lineNumber.Value.Kind != StackFrameKind.NumberToken) { - return new(false, null); + return ParseResult.Abort; } - return new(true, new(path.Value, colonToken, lineNumber.Value)); + return new(new(path.Value, colonToken, lineNumber.Value)); } } } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs index 0a7165a805f81..cb1b89f0a0122 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs @@ -14,6 +14,10 @@ internal abstract class EmbeddedSeparatedSyntaxNodeList where TList : EmbeddedSeparatedSyntaxNodeList { + public ImmutableArray> NodesAndTokens { get; } + public int Length { get; } + public int SeparatorLength { get; } + public EmbeddedSeparatedSyntaxNodeList(ImmutableArray> nodesAndTokens) { Contract.ThrowIfTrue(nodesAndTokens.IsDefault); @@ -50,14 +54,11 @@ private void Verify() } } - public ImmutableArray> NodesAndTokens { get; } - public int Length { get; } - public int SeparatorLength { get; } /// /// Retrieves only nodes, skipping the separator tokens /// - public EmbeddedSyntaxNodeOrToken this[int index] + public TSyntaxNode this[int index] { get { @@ -65,7 +66,9 @@ public EmbeddedSyntaxNodeOrToken this[int index] { // x2 here to get only even indexed numbers. Follows same logic // as SeparatedSyntaxList in that the separator tokens are not returned - return NodesAndTokens[index * 2]; + var nodeOrToken = NodesAndTokens[index * 2]; + Debug.Assert(nodeOrToken.IsNode); + return nodeOrToken.Node; } throw new ArgumentOutOfRangeException(nameof(index)); From 29754a2b4fdc136711dbd6f68888954287d27e2b Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Thu, 28 Oct 2021 17:51:25 -0700 Subject: [PATCH 35/59] shakes fist at blank lines --- .../EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs index cb1b89f0a0122..aad227ae38175 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs @@ -54,7 +54,6 @@ private void Verify() } } - /// /// Retrieves only nodes, skipping the separator tokens /// From 1abd3bab0344e420ec95de51944160b02ccb64c1 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Thu, 28 Oct 2021 19:27:35 -0700 Subject: [PATCH 36/59] add assert for non null --- .../EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs index aad227ae38175..e6804c4cf12c4 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs @@ -67,6 +67,7 @@ public TSyntaxNode this[int index] // as SeparatedSyntaxList in that the separator tokens are not returned var nodeOrToken = NodesAndTokens[index * 2]; Debug.Assert(nodeOrToken.IsNode); + RoslynDebug.AssertNotNull(nodeOrToken.Node); return nodeOrToken.Node; } From a017ec206e990204e06c91a68b82e06d03971f6a Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Fri, 29 Oct 2021 10:55:21 -0700 Subject: [PATCH 37/59] Remove equality overloads --- .../Common/EmbeddedSeparatedSyntaxNodeList.cs | 1 - .../Common/EmbeddedSyntaxToken.cs | 24 ------------------- .../Common/EmbeddedSyntaxTrivia.cs | 14 ----------- .../VirtualChars/VirtualCharSequence.cs | 14 ----------- 4 files changed, 53 deletions(-) diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs index e6804c4cf12c4..f5e83365eaaf9 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs @@ -49,7 +49,6 @@ private void Verify() { // All odd values should be separator tokens Debug.Assert(!NodesAndTokens[i].IsNode); - Debug.Assert(NodesAndTokens[i].Token != default); } } } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs index 705d9efc3be6a..ab872841f8aa0 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs @@ -72,29 +72,5 @@ public EmbeddedSyntaxToken With( public TextSpan GetSpan() => EmbeddedSyntaxHelpers.GetSpan(this.VirtualChars); - - public static bool operator ==(EmbeddedSyntaxToken t1, EmbeddedSyntaxToken t2) - => t1.Kind.Equals(t2.Kind) - && t1.LeadingTrivia.IsDefault == t2.LeadingTrivia.IsDefault - && (t1.LeadingTrivia.IsDefault || t1.LeadingTrivia.SequenceEqual(t2.LeadingTrivia)) - && t1.VirtualChars == t2.VirtualChars - && t1.TrailingTrivia.IsDefault == t2.TrailingTrivia.IsDefault - && (t1.TrailingTrivia.IsDefault || t1.TrailingTrivia.SequenceEqual(t2.TrailingTrivia)) - && t1.Diagnostics.IsDefault == t2.Diagnostics.IsDefault - && (t1.Diagnostics.IsDefault || t1.Diagnostics.SequenceEqual(t2.Diagnostics)) - && t1.Value == t2.Value; - - public static bool operator !=(EmbeddedSyntaxToken t1, EmbeddedSyntaxToken t2) - => !(t1 == t2); - - public override bool Equals(object? obj) - => obj is EmbeddedSyntaxToken t && this == t; - - public override int GetHashCode() - => Hash.Combine(Kind.GetHashCode(), - Hash.Combine(LeadingTrivia.GetHashCode(), - Hash.Combine(VirtualChars.GetHashCode(), - Hash.Combine(TrailingTrivia.GetHashCode(), - Hash.Combine(Diagnostics.GetHashCode(), Value?.GetHashCode() ?? 0))))); } } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs index 3c354e782d272..ea31aeb8ada70 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs @@ -30,19 +30,5 @@ public EmbeddedSyntaxTrivia(TSyntaxKind kind, VirtualCharSequence virtualChars, VirtualChars = virtualChars; Diagnostics = diagnostics; } - - public override bool Equals(object? obj) - => obj is EmbeddedSyntaxTrivia trivia && this == trivia; - - public override int GetHashCode() - => Hash.Combine(Kind.GetHashCode(), - Hash.Combine(VirtualChars.GetHashCode(), Diagnostics.GetHashCode())); - - public static bool operator ==(EmbeddedSyntaxTrivia t1, EmbeddedSyntaxTrivia t2) - => t1.Kind.Equals(t2.Kind) - && t1.VirtualChars.Equals(t2.VirtualChars); - - public static bool operator !=(EmbeddedSyntaxTrivia t1, EmbeddedSyntaxTrivia t2) - => !(t1 == t2); } } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs index f4279b38a4d60..40a6c0e9438aa 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs @@ -145,19 +145,5 @@ public static VirtualCharSequence FromBounds( chars1._leafCharacters, TextSpan.FromBounds(chars1._span.Start, chars2._span.End)); } - - public override bool Equals(object? obj) - => obj is VirtualCharSequence seq && this == seq; - - public override int GetHashCode() - => Hash.Combine(_leafCharacters.GetHashCode(), _span.GetHashCode()); - - public static bool operator ==(VirtualCharSequence v1, VirtualCharSequence v2) - => v1.Length == v2.Length - && v1._span == v2._span - && v1._leafCharacters == v2._leafCharacters; - - public static bool operator !=(VirtualCharSequence v1, VirtualCharSequence v2) - => !(v1 == v2); } } From 4a4b2ebc802b383c03cef56640b1747a455384b9 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Fri, 29 Oct 2021 10:58:19 -0700 Subject: [PATCH 38/59] cleanup unused usings --- .../Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs | 3 --- .../Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs | 1 - .../Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs | 1 - 3 files changed, 5 deletions(-) diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs index ab872841f8aa0..ddbeb7f618641 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxToken.cs @@ -2,13 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.Collections.Immutable; using System.Diagnostics; -using System.Linq; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.Text; -using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.Common { diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs index ea31aeb8ada70..7f06e3138c56e 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSyntaxTrivia.cs @@ -5,7 +5,6 @@ using System.Collections.Immutable; using System.Diagnostics; using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; -using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.Common { diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs index 40a6c0e9438aa..37dc47a5a64c3 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/VirtualChars/VirtualCharSequence.cs @@ -7,7 +7,6 @@ using System.Diagnostics; using Microsoft.CodeAnalysis.PooledObjects; using Microsoft.CodeAnalysis.Text; -using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars { From 6f73bb6a14ee5ef61f93cf4b669c79ff4deadd9b Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Sun, 31 Oct 2021 17:25:35 -0700 Subject: [PATCH 39/59] fix default comparison --- .../StackFrame/StackFrameParserTests.Utilities.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index f66e8abe683c2..8fa87009120db 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -232,13 +232,22 @@ private static IEnumerable Enumerate(StackFrameNode node) yield return charSequence; } } - else if (nodeOrToken.Token != default) + else if (nodeOrToken.Token.Kind != StackFrameKind.None) { foreach (var charSequence in Enumerate(nodeOrToken.Token)) { yield return charSequence; } } + else + { + // If we encounter a None token make sure it has default values + Assert.True(nodeOrToken.Token.IsMissing); + Assert.True(nodeOrToken.Token.LeadingTrivia.IsDefault); + Assert.True(nodeOrToken.Token.TrailingTrivia.IsDefault); + Assert.Null(nodeOrToken.Token.Value); + Assert.True(nodeOrToken.Token.VirtualChars.IsDefault); + } } } From 0401dc448b32b4d74d41d2661725e32e141d3a19 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Sun, 31 Oct 2021 17:48:11 -0700 Subject: [PATCH 40/59] PR feedback. Add more tests for invalid identifiers --- .../EmbeddedLanguages/StackFrame/StackFrameParserTests.cs | 4 ++++ .../EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs | 2 ++ .../Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs | 3 +++ 3 files changed, 9 insertions(+) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index e165035e104c6..c28583f0647b0 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -341,6 +341,10 @@ public void TestFileInformation_InvalidDirectory() [InlineData(@"at M.M(string[[]] s)")] [InlineData(@"at M.N`.P()")] // Missing numeric for arity [InlineData(@"at M.N`9N.P()")] // Invalid character after arity + [InlineData("M.N.P.()")] // Trailing . with no identifier before arguments + [InlineData("M.N(X.Y. x)")] // Trailing . in argument type + [InlineData("M.N[T.Y]()")] // Generic type arguments should not be qualified types + [InlineData("M.N(X.Y x.y)")] // argument names should not be qualified public void TestInvalidInputs(string input) => Verify(input, expectFailure: true); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index a8397c78134cf..b79b27c7c9ba4 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -100,6 +100,8 @@ public static StackFrameQualifiedNameNode QualifiedName(string s, StackFrameTriv public static StackFrameQualifiedNameNode QualifiedName(string s, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) { StackFrameNameNode? current = null; + Assert.True(s.Contains('.')); + var identifiers = s.Split('.'); for (var i = 0; i < identifiers.Length; i++) { diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 0f34fae93071a..34034a2b9ef83 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -127,12 +127,15 @@ private ParseResult TryParseMethodDeclaration() /// /// Nodes will be parsed for arity but not generic type arguments. /// + /// /// All of the following are valid nodes, where "$$" marks the parsing starting point, and "[|" + "|]" mark the endpoints of the parsed node excluding trivia /// * [|$$MyNamespace.MyClass.MyMethod|](string s) /// * MyClass.MyMethod([|$$string|] s) /// * MyClass.MyMethod([|$$string[]|] s) /// * [|$$MyClass`1.MyMethod|](string s) /// * [|$$MyClass.MyMethod|][T](T t) + /// + /// /// private ParseResult TryParseNameNode(bool scanAtTrivia) { From 06f0a696a49143546b791f09b36b855c886fcb2f Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Sun, 31 Oct 2021 17:50:48 -0700 Subject: [PATCH 41/59] Add some asserts --- .../Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 34034a2b9ef83..dfb4e77d90d0a 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -165,6 +165,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) var memberAccess = parseResult.Value; if (memberAccess is null) { + Debug.Assert(lhs is StackFrameSimpleNameNode); return new(lhs); } @@ -179,6 +180,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) var newMemberAccess = parseResult.Value; if (newMemberAccess is null) { + Debug.Assert(memberAccess is StackFrameQualifiedNameNode); return new(memberAccess); } From 665463738dced6fa86a7d4a7a2accd3c3e080828 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Sun, 31 Oct 2021 23:06:16 -0700 Subject: [PATCH 42/59] Typing --- .../StackFrame/StackFrameNodeDefinitions.cs | 30 +++++++------------ .../StackFrame/StackFrameParser.cs | 6 ++-- .../Common/EmbeddedSeparatedSyntaxNodeList.cs | 30 +++++++++---------- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index e074a5704ce7b..e0887cddb6271 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -27,21 +27,6 @@ protected StackFrameNode(StackFrameKind kind) : base(kind) public abstract void Accept(IStackFrameNodeVisitor visitor); } - internal sealed class SeparatedStackFrameNodeList : EmbeddedSeparatedSyntaxNodeList> - where TNode : EmbeddedSyntaxNode - { - public SeparatedStackFrameNodeList(ImmutableArray nodeOrTokens) - : base(nodeOrTokens) - { - } - - private SeparatedStackFrameNodeList() : base() - { - } - - public static SeparatedStackFrameNodeList Empty { get; } = new(); - } - internal abstract class StackFrameDeclarationNode : StackFrameNode { protected StackFrameDeclarationNode(StackFrameKind kind) : base(kind) @@ -292,10 +277,14 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameTypeArgumentList : StackFrameNode { public readonly StackFrameToken OpenToken; - public readonly SeparatedStackFrameNodeList TypeArguments; + public readonly EmbeddedSeparatedSyntaxNodeList TypeArguments; public readonly StackFrameToken CloseToken; - public StackFrameTypeArgumentList(StackFrameToken openToken, SeparatedStackFrameNodeList typeArguments, StackFrameToken closeToken) : base(StackFrameKind.TypeArgument) + public StackFrameTypeArgumentList( + StackFrameToken openToken, + EmbeddedSeparatedSyntaxNodeList typeArguments, + StackFrameToken closeToken) + : base(StackFrameKind.TypeArgument) { Debug.Assert(openToken.Kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken); Debug.Assert(typeArguments.Length > 0); @@ -363,10 +352,13 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameParameterList : StackFrameNode { public readonly StackFrameToken OpenParen; - public readonly SeparatedStackFrameNodeList Parameters; + public readonly EmbeddedSeparatedSyntaxNodeList Parameters; public readonly StackFrameToken CloseParen; - public StackFrameParameterList(StackFrameToken openToken, SeparatedStackFrameNodeList parameters, StackFrameToken closeToken) + public StackFrameParameterList( + StackFrameToken openToken, + EmbeddedSeparatedSyntaxNodeList parameters, + StackFrameToken closeToken) : base(StackFrameKind.ParameterList) { Debug.Assert(openToken.Kind == StackFrameKind.OpenParenToken); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index dfb4e77d90d0a..ce298afd75587 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -296,7 +296,7 @@ private ParseResult TryParseTypeArguments() return ParseResult.Abort; } - var separatedList = new SeparatedStackFrameNodeList(builder.ToImmutable()); + var separatedList = new EmbeddedSeparatedSyntaxNodeList(builder.ToImmutable()); return new(new(openToken, separatedList, closeToken)); } @@ -313,7 +313,7 @@ private ParseResult ParseMethodParameters() if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) { - return new(new(openParen, SeparatedStackFrameNodeList.Empty, closeParen)); + return new(new(openParen, EmbeddedSeparatedSyntaxNodeList.Empty, closeParen)); } using var _ = ArrayBuilder.GetInstance(out var builder); @@ -342,7 +342,7 @@ private ParseResult ParseMethodParameters() return ParseResult.Empty; } - var parameters = new SeparatedStackFrameNodeList(builder.ToImmutable()); + var parameters = new EmbeddedSeparatedSyntaxNodeList(builder.ToImmutable()); return new(new(openParen, parameters, closeParen)); } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs index f5e83365eaaf9..afd8eb6f425dd 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs @@ -9,29 +9,29 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.Common { - internal abstract class EmbeddedSeparatedSyntaxNodeList + internal struct EmbeddedSeparatedSyntaxNodeList where TSyntaxKind : struct where TSyntaxNode : EmbeddedSyntaxNode - where TList : EmbeddedSeparatedSyntaxNodeList + where TDerivedNode : TSyntaxNode { public ImmutableArray> NodesAndTokens { get; } public int Length { get; } public int SeparatorLength { get; } - public EmbeddedSeparatedSyntaxNodeList(ImmutableArray> nodesAndTokens) + public static readonly EmbeddedSeparatedSyntaxNodeList Empty + = new(ImmutableArray>.Empty); + + public EmbeddedSeparatedSyntaxNodeList( + ImmutableArray> nodesAndTokens) { Contract.ThrowIfTrue(nodesAndTokens.IsDefault); NodesAndTokens = nodesAndTokens; - Verify(); var allLength = NodesAndTokens.Length; Length = (allLength + 1) / 2; SeparatorLength = allLength / 2; - } - protected EmbeddedSeparatedSyntaxNodeList() - { - NodesAndTokens = ImmutableArray>.Empty; + Verify(); } [Conditional("DEBUG")] @@ -56,7 +56,7 @@ private void Verify() /// /// Retrieves only nodes, skipping the separator tokens /// - public TSyntaxNode this[int index] + public TDerivedNode this[int index] { get { @@ -67,7 +67,7 @@ public TSyntaxNode this[int index] var nodeOrToken = NodesAndTokens[index * 2]; Debug.Assert(nodeOrToken.IsNode); RoslynDebug.AssertNotNull(nodeOrToken.Node); - return nodeOrToken.Node; + return (TDerivedNode)nodeOrToken.Node; } throw new ArgumentOutOfRangeException(nameof(index)); @@ -78,24 +78,24 @@ public TSyntaxNode this[int index] public struct Enumerator { - private readonly EmbeddedSeparatedSyntaxNodeList _list; + private readonly EmbeddedSeparatedSyntaxNodeList _list; private int _currentIndex; - public Enumerator(EmbeddedSeparatedSyntaxNodeList list) + public Enumerator(EmbeddedSeparatedSyntaxNodeList list) { _list = list; _currentIndex = -1; - Current = default; + Current = null!; } - public EmbeddedSyntaxNodeOrToken Current { get; private set; } + public TDerivedNode Current { get; private set; } public bool MoveNext() { _currentIndex++; if (_currentIndex >= _list.Length) { - Current = null; + Current = null!; return false; } From e74a7f14445ae000ffa3251d4b79841a653fb335 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Sun, 31 Oct 2021 23:49:42 -0700 Subject: [PATCH 43/59] cleanup test factory code --- .../EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index b79b27c7c9ba4..b22fcf0c3c24f 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -66,7 +66,7 @@ public static StackFrameParameterList ParameterList(params StackFrameParameterDe public static StackFrameParameterList ParameterList(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameParameterDeclarationNode[] parameters) { var separatedList = parameters.Length == 0 - ? SeparatedStackFrameNodeList.Empty + ? EmbeddedSeparatedSyntaxNodeList.Empty : new(CommaSeparateList(parameters)); return new(openToken, separatedList, closeToken); @@ -183,7 +183,7 @@ public static StackFrameTypeArgumentList TypeArgumentList(bool useBrackets, para builder.Add(typeArgument); } - var typeArgumentsList = new SeparatedStackFrameNodeList(builder.ToImmutable()); + var typeArgumentsList = new EmbeddedSeparatedSyntaxNodeList(builder.ToImmutable()); return new(openToken, typeArgumentsList, closeToken); } From 99bf2bf862daf0119e17a97a4bc4f22664edab33 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 13:10:36 -0700 Subject: [PATCH 44/59] PR cleanup 1 --- .../StackFrame/StackFrameParserTests.cs | 3 +- .../StackFrame/StackFrameSyntaxFactory.cs | 9 +- .../StackFrame/StackFrameKind.cs | 1 + .../StackFrame/StackFrameLexer.cs | 6 +- .../StackFrame/StackFrameNodeDefinitions.cs | 4 +- .../StackFrameParser.ParseResult.cs | 4 +- .../StackFrame/StackFrameParser.cs | 151 +++++++++--------- .../Common/EmbeddedSeparatedSyntaxNodeList.cs | 2 +- 8 files changed, 91 insertions(+), 89 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index c28583f0647b0..eeb6827d3d7ae 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using Xunit; +using System.Collections.Immutable; using static Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame.StackFrameSyntaxFactory; namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame @@ -85,7 +86,7 @@ public void TestNoParams_SpaceInParams_NoAtTrivia() methodDeclaration: MethodDeclaration( QualifiedName("ConsoleApp4.MyClass.M"), argumentList: ParameterList( - OpenParenToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia(2))), + OpenParenToken.With(trailingTrivia: ImmutableArray.Create(SpaceTrivia(2))), CloseParenToken)) ); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index b22fcf0c3c24f..66e33e52d8f75 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -30,9 +30,6 @@ public static StackFrameToken CreateToken(StackFrameKind kind, string s, Immutab public static StackFrameTrivia CreateTrivia(StackFrameKind kind, string text) => new(kind, CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, text), ImmutableArray.Empty); - public static ImmutableArray CreateTriviaArray(params StackFrameTrivia[] trivia) - => ImmutableArray.Create(trivia); - public static ImmutableArray CreateTriviaArray(StackFrameTrivia? trivia) => trivia.HasValue ? ImmutableArray.Create(trivia.Value) : ImmutableArray.Empty; @@ -47,7 +44,7 @@ public static ImmutableArray CreateTriviaArray(params string[] public static readonly StackFrameToken CloseBracketToken = CreateToken(StackFrameKind.CloseBracketToken, "]"); public static readonly StackFrameToken LessThanToken = CreateToken(StackFrameKind.LessThanToken, "<"); public static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); - public static readonly StackFrameToken AccentGraveToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); + public static readonly StackFrameToken GraveAccentToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); public static readonly StackFrameToken EOLToken = CreateToken(StackFrameKind.EndOfLine, ""); public static readonly StackFrameToken ColonToken = CreateToken(StackFrameKind.ColonToken, ":"); @@ -157,7 +154,7 @@ public static StackFrameArrayTypeExpression ArrayExpression(StackFrameNameNode i => new(identifier, arrayTokens.ToImmutableArray()); public static StackFrameGenericNameNode GenericType(string identifierName, int arity) - => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), AccentGraveToken, CreateToken(StackFrameKind.NumberToken, arity.ToString())); + => new(CreateToken(StackFrameKind.IdentifierToken, identifierName), GraveAccentToken, CreateToken(StackFrameKind.NumberToken, arity.ToString())); public static StackFrameTypeArgumentList TypeArgumentList(params StackFrameIdentifierNameNode[] typeArguments) => TypeArgumentList(useBrackets: true, typeArguments); @@ -195,7 +192,7 @@ public static StackFrameIdentifierNameNode TypeArgument(StackFrameToken identifi => new(identifier); public static StackFrameFileInformationNode FileInformation(StackFrameToken path, StackFrameToken colon, StackFrameToken line) - => new(path.With(leadingTrivia: CreateTriviaArray(InTrivia)), colon, line); + => new(path.With(leadingTrivia: ImmutableArray.Create(InTrivia)), colon, line); public static StackFrameToken Path(string path) => CreateToken(StackFrameKind.PathToken, path); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 51cbcada7a902..2d50f8363224c 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -46,6 +46,7 @@ internal enum StackFrameKind PathToken, InvalidPathToken, NumberToken, + InvalidNumberToken, // Trivia WhitespaceTrivia, diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 36182c9672613..63dc270b77ec5 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -142,9 +142,9 @@ public bool ScanCurrentCharAsTokenIfMatch(StackFrameKind kind, bool scanTrailing /// /// if the position was incremented /// - public bool ScanCurrentCharAsTokenIfMatch(Func matchFn, out StackFrameToken token) + public bool ScanCurrentCharAsTokenIfMatch(Func isMatch, out StackFrameToken token) { - if (matchFn(GetKind(CurrentChar))) + if (isMatch(GetKind(CurrentChar))) { token = CurrentCharAsToken(); Position++; @@ -236,7 +236,7 @@ public bool ScanCurrentCharAsTokenIfMatch(Func matchFn, ou var numberToken = TryScanNumbers(); if (!numberToken.HasValue) { - return CreateToken(StackFrameKind.None, ImmutableArray.Create(lineTrivia.Value), VirtualCharSequence.Empty); + return CreateToken(StackFrameKind.InvalidNumberToken, ImmutableArray.Create(lineTrivia.Value), VirtualCharSequence.Empty); } var remainingTrivia = TryScanRemainingTrivia(); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index e0887cddb6271..d2822e3466ac5 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -205,7 +205,7 @@ internal sealed class StackFrameArrayTypeExpression : StackFrameNameNode public StackFrameArrayTypeExpression(StackFrameNameNode typeIdentifier, ImmutableArray arrayExpressions) : base(StackFrameKind.ArrayTypeExpression) { - Contract.ThrowIfTrue(arrayExpressions.IsDefaultOrEmpty); + Debug.Assert(!arrayExpressions.IsDefaultOrEmpty); TypeIdentifier = typeIdentifier; ArrayExpressions = arrayExpressions; } @@ -232,7 +232,7 @@ internal sealed class StackFrameArrayRankSpecifier : StackFrameNode public StackFrameArrayRankSpecifier(StackFrameToken openBracket, StackFrameToken closeBracket, ImmutableArray commaTokens) : base(StackFrameKind.ArrayExpression) { - Contract.ThrowIfTrue(commaTokens.IsDefault); + Debug.Assert(!commaTokens.IsDefault); Debug.Assert(openBracket.Kind == StackFrameKind.OpenBracketToken); Debug.Assert(closeBracket.Kind == StackFrameKind.CloseBracketToken); Debug.Assert(commaTokens.All(static t => t.Kind == StackFrameKind.CommaToken)); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs index 1d88ff042ac2d..4b8fcf1261da2 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs @@ -8,7 +8,7 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { internal partial struct StackFrameParser { - private struct ParseResult + private readonly struct ParseResult { public readonly bool Success; public readonly T? Value; @@ -31,6 +31,8 @@ public void Deconstruct(out bool success, out T? value) success = Success; value = Value; } + + public static implicit operator ParseResult(T value) => new(value); } } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index ce298afd75587..135522ff0daed 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -23,12 +23,13 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame /// internal partial struct StackFrameParser { + private StackFrameLexer _lexer; + private StackFrameParser(VirtualCharSequence text) { _lexer = new(text); } - private StackFrameLexer _lexer; private StackFrameToken CurrentCharAsToken() => _lexer.CurrentCharAsToken(); /// @@ -64,8 +65,8 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameTree? TryParseTree() { - var (_, methodDeclaration) = TryParseMethodDeclaration(); - if (methodDeclaration is null) + var (success, methodDeclaration) = TryParseMethodDeclaration(); + if (!success || methodDeclaration is null) { return null; } @@ -80,8 +81,8 @@ private StackFrameParser(VirtualCharSequence text) var eolToken = CurrentCharAsToken().With(leadingTrivia: remainingTrivia.HasValue ? ImmutableArray.Create(remainingTrivia.Value) : ImmutableArray.Empty); - Debug.Assert(_lexer.Position == _lexer.Text.Length); - Debug.Assert(eolToken.Kind == StackFrameKind.EndOfLine); + Contract.ThrowIfFalse(_lexer.Position == _lexer.Text.Length); + Contract.ThrowIfFalse(eolToken.Kind == StackFrameKind.EndOfLine); var root = new StackFrameCompilationUnit(methodDeclaration, fileInformationResult.Value, eolToken); @@ -107,19 +108,19 @@ private ParseResult TryParseMethodDeclaration() return ParseResult.Abort; } - var typeArgumentsResult = TryParseTypeArguments(); - if (!typeArgumentsResult.Success) + (success, var typeArguments) = TryParseTypeArguments(); + if (!success) { return ParseResult.Abort; } - var argumentsResult = ParseMethodParameters(); - if (!argumentsResult.Success || argumentsResult.Value is null) + var methodParameters = TryParseRequiredMethodParameters(); + if (methodParameters is null) { return ParseResult.Abort; } - return new(new(memberAccessExpression, typeArgumentsResult.Value, argumentsResult.Value)); + return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, methodParameters); } /// @@ -147,7 +148,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) return ParseResult.Empty; } - var identifierParseResult = TryScanGenericTypeIdentifier(currentIdentifer.Value); + var identifierParseResult = TryScanGenericTypeIdentifier(_lexer, currentIdentifer.Value); if (!identifierParseResult.Success) { return ParseResult.Abort; @@ -156,7 +157,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) RoslynDebug.AssertNotNull(identifierParseResult.Value); var lhs = identifierParseResult.Value; - var parseResult = TryParseQualifiedName(lhs); + var parseResult = TryParseQualifiedName(_lexer, lhs); if (!parseResult.Success) { return ParseResult.Abort; @@ -171,7 +172,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) while (true) { - parseResult = TryParseQualifiedName(memberAccess); + parseResult = TryParseQualifiedName(_lexer, memberAccess); if (!parseResult.Success) { return ParseResult.Abort; @@ -186,60 +187,57 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) memberAccess = newMemberAccess; } - } - - /// - /// Given an existing left hand side node or token, which can either be - /// an or - /// - private ParseResult TryParseQualifiedName(StackFrameNameNode lhs) - { - if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) - { - return ParseResult.Empty; - } - var identifier = _lexer.TryScanIdentifier(); - if (!identifier.HasValue) + // + // Given an existing left hand side node or token, which can either be + // an or + // + static ParseResult TryParseQualifiedName(StackFrameLexer lexer, StackFrameNameNode lhs) { - return ParseResult.Abort; - } + if (!lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) + { + return ParseResult.Empty; + } - var (success, rhs) = TryScanGenericTypeIdentifier(identifier.Value); - if (!success) - { - return ParseResult.Abort; - } + var identifier = lexer.TryScanIdentifier(); + if (!identifier.HasValue) + { + return ParseResult.Abort; + } - RoslynDebug.AssertNotNull(rhs); - return new(new(lhs, dotToken, rhs)); - } + var (success, rhs) = TryScanGenericTypeIdentifier(lexer, identifier.Value); + if (!success) + { + return ParseResult.Abort; + } - /// - /// Given an identifier, attempts to parse the type identifier arity for it. - /// - /// - /// ex: MyNamespace.MyClass`1.MyMethod() - /// ^--------------------- MyClass would be the identifier passed in - /// ^-------------- Grave token - /// ^------------- Arity token of "1" - /// - /// - /// - private ParseResult TryScanGenericTypeIdentifier(StackFrameToken identifierToken) - { - if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) - { - return new(new StackFrameIdentifierNameNode(identifierToken)); + RoslynDebug.AssertNotNull(rhs); + return new StackFrameQualifiedNameNode(lhs, dotToken, rhs); } - var arity = _lexer.TryScanNumbers(); - if (!arity.HasValue) + // + // Given an identifier, attempts to parse the type identifier arity for it. + // + // ex: MyNamespace.MyClass`1.MyMethod() + // ^--------------------- MyClass would be the identifier passed in + // ^-------------- Grave token + // ^------------- Arity token of "1" + // + static ParseResult TryScanGenericTypeIdentifier(StackFrameLexer lexer, StackFrameToken identifierToken) { - return ParseResult.Abort; - } + if (!lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) + { + return new(new StackFrameIdentifierNameNode(identifierToken)); + } + + var arity = lexer.TryScanNumbers(); + if (!arity.HasValue) + { + return ParseResult.Abort; + } - return new(new StackFrameGenericNameNode(identifierToken, graveAccentToken, arity.Value)); + return new StackFrameGenericNameNode(identifierToken, graveAccentToken, arity.Value); + } } /// @@ -297,23 +295,27 @@ private ParseResult TryParseTypeArguments() } var separatedList = new EmbeddedSeparatedSyntaxNodeList(builder.ToImmutable()); - return new(new(openToken, separatedList, closeToken)); + return new StackFrameTypeArgumentList(openToken, separatedList, closeToken); } /// /// MyNamespace.MyClass.MyMethod[|(string s1, string s2, int i1)|] /// Takes parameter declarations from method text and parses them into a . /// - private ParseResult ParseMethodParameters() + /// + /// This method assumes that the caller requires method parameters, and returns null for all failure cases. The caller + /// should escalate to abort parsing on null values. + /// + private StackFrameParameterList? TryParseRequiredMethodParameters() { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.OpenParenToken, scanTrailingWhitespace: true, out var openParen)) { - return ParseResult.Abort; + return null; } if (_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out var closeParen)) { - return new(new(openParen, EmbeddedSeparatedSyntaxNodeList.Empty, closeParen)); + return new(openParen, EmbeddedSeparatedSyntaxNodeList.Empty, closeParen); } using var _ = ArrayBuilder.GetInstance(out var builder); @@ -323,7 +325,7 @@ private ParseResult ParseMethodParameters() var (success, parameterNode) = ParseParameterNode(); if (!success) { - return ParseResult.Abort; + return null; } RoslynDebug.AssertNotNull(parameterNode); @@ -339,11 +341,11 @@ private ParseResult ParseMethodParameters() if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseParenToken, out closeParen)) { - return ParseResult.Empty; + return null; } var parameters = new EmbeddedSeparatedSyntaxNodeList(builder.ToImmutable()); - return new(new(openParen, parameters, closeParen)); + return new(openParen, parameters, closeParen); } /// @@ -377,7 +379,7 @@ private ParseResult ParseParameterNode() return ParseResult.Abort; } - return new(new(typeIdentifier, identifier.Value)); + return new StackFrameParameterDeclarationNode(typeIdentifier, identifier.Value); } /// @@ -411,8 +413,6 @@ private ParseResult> ParseArrayRank builder.Add(new StackFrameArrayRankSpecifier(openBracket, closeBracket, commaBuilder.ToImmutableAndClear())); } - - throw ExceptionUtilities.Unreachable; } /// @@ -430,26 +430,27 @@ private ParseResult TryParseFileInformation() return ParseResult.Empty; } - if (path.Value.Kind != StackFrameKind.PathToken) + if (path.Value.Kind == StackFrameKind.InvalidPathToken) { return ParseResult.Abort; } + Debug.Assert(path.Value.Kind == StackFrameKind.PathToken); + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.ColonToken, out var colonToken)) { - return new(new(path.Value, null, null)); + return new StackFrameFileInformationNode(path.Value, colon: null, line: null); } var lineNumber = _lexer.TryScanLineNumber(); - - // TryScanLineNumber can return a token that isn't a number, in which case we want - // to bail in error and consider this malformed. - if (!lineNumber.HasValue || lineNumber.Value.Kind != StackFrameKind.NumberToken) + if (!lineNumber.HasValue || lineNumber.Value.Kind == StackFrameKind.InvalidNumberToken) { return ParseResult.Abort; } - return new(new(path.Value, colonToken, lineNumber.Value)); + Debug.Assert(lineNumber.Value.Kind == StackFrameKind.NumberToken); + + return new StackFrameFileInformationNode(path.Value, colonToken, lineNumber.Value); } } } diff --git a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs index afd8eb6f425dd..4cd837cabae59 100644 --- a/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs +++ b/src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/EmbeddedLanguages/Common/EmbeddedSeparatedSyntaxNodeList.cs @@ -9,7 +9,7 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.Common { - internal struct EmbeddedSeparatedSyntaxNodeList + internal readonly struct EmbeddedSeparatedSyntaxNodeList where TSyntaxKind : struct where TSyntaxNode : EmbeddedSyntaxNode where TDerivedNode : TSyntaxNode From 87ba143d19abed5dcc0c47c657efb86951e724d9 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 13:20:46 -0700 Subject: [PATCH 45/59] Fix type hierarchy for ParameterDeclaration --- .../StackFrame/StackFrameSyntaxFactory.cs | 2 +- .../StackFrame/IStackFrameNodeVisitor.cs | 2 +- .../StackFrame/StackFrameNodeDefinitions.cs | 25 +++++++++++++++---- .../StackFrame/StackFrameParser.cs | 7 +++--- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 66e33e52d8f75..37eac42eaad84 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -150,7 +150,7 @@ public static StackFrameIdentifierNameNode Identifier(string name, StackFrameTri public static StackFrameArrayRankSpecifier ArrayRankSpecifier(int commaCount = 0, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) => new(OpenBracketToken.With(leadingTrivia: CreateTriviaArray(leadingTrivia)), CloseBracketToken.With(trailingTrivia: CreateTriviaArray(trailingTrivia)), Enumerable.Repeat(CommaToken, commaCount).ToImmutableArray()); - public static StackFrameArrayTypeExpression ArrayExpression(StackFrameNameNode identifier, params StackFrameArrayRankSpecifier[] arrayTokens) + public static StackFrameArrayTypeNode ArrayExpression(StackFrameNameNode identifier, params StackFrameArrayRankSpecifier[] arrayTokens) => new(identifier, arrayTokens.ToImmutableArray()); public static StackFrameGenericNameNode GenericType(string identifierName, int arity) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs index b7ee6c4586289..fa948cc6e9bae 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/IStackFrameNodeVisitor.cs @@ -15,7 +15,7 @@ internal interface IStackFrameNodeVisitor void Visit(StackFrameIdentifierNameNode node); void Visit(StackFrameArrayRankSpecifier node); void Visit(StackFrameFileInformationNode node); - void Visit(StackFrameArrayTypeExpression node); + void Visit(StackFrameArrayTypeNode node); void Visit(StackFrameParameterDeclarationNode node); } } diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index d2822e3466ac5..d3844e71fb4a9 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -66,10 +66,25 @@ internal override StackFrameNodeOrToken ChildAt(int index) }; } + /// + /// Base class for all type nodes + /// + internal abstract class StackFrameTypeNode : StackFrameNode + { + protected StackFrameTypeNode(StackFrameKind kind) : base(kind) + { + } + } + /// /// Base class for all name nodes /// - internal abstract class StackFrameNameNode : StackFrameNode + /// + /// All of these are . If a node requires an identifier or name that + /// is not a type then it should use with + /// directly. + /// + internal abstract class StackFrameNameNode : StackFrameTypeNode { protected StackFrameNameNode(StackFrameKind kind) : base(kind) { @@ -186,7 +201,7 @@ internal override StackFrameNodeOrToken ChildAt(int index) /// /// Represents an array type declaration, such as string[,][] /// - internal sealed class StackFrameArrayTypeExpression : StackFrameNameNode + internal sealed class StackFrameArrayTypeNode : StackFrameTypeNode { /// /// The type identifier without the array indicators. @@ -203,7 +218,7 @@ internal sealed class StackFrameArrayTypeExpression : StackFrameNameNode /// public ImmutableArray ArrayExpressions; - public StackFrameArrayTypeExpression(StackFrameNameNode typeIdentifier, ImmutableArray arrayExpressions) : base(StackFrameKind.ArrayTypeExpression) + public StackFrameArrayTypeNode(StackFrameNameNode typeIdentifier, ImmutableArray arrayExpressions) : base(StackFrameKind.ArrayTypeExpression) { Debug.Assert(!arrayExpressions.IsDefaultOrEmpty); TypeIdentifier = typeIdentifier; @@ -324,12 +339,12 @@ internal override StackFrameNodeOrToken ChildAt(int index) internal sealed class StackFrameParameterDeclarationNode : StackFrameDeclarationNode { - public readonly StackFrameNameNode Type; + public readonly StackFrameTypeNode Type; public readonly StackFrameToken Identifier; internal override int ChildCount => 2; - public StackFrameParameterDeclarationNode(StackFrameNameNode type, StackFrameToken identifier) + public StackFrameParameterDeclarationNode(StackFrameTypeNode type, StackFrameToken identifier) : base(StackFrameKind.Parameter) { Debug.Assert(identifier.Kind == StackFrameKind.IdentifierToken); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 135522ff0daed..23be728b5a6aa 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -356,12 +356,13 @@ private ParseResult TryParseTypeArguments() /// private ParseResult ParseParameterNode() { - var (success, typeIdentifier) = TryParseNameNode(scanAtTrivia: false); - if (!success || typeIdentifier is null) + var (success, nameNode) = TryParseNameNode(scanAtTrivia: false); + if (!success || nameNode is null) { return ParseResult.Abort; } + StackFrameTypeNode? typeIdentifier = nameNode; if (CurrentCharAsToken().Kind == StackFrameKind.OpenBracketToken) { (success, var arrayIdentifiers) = ParseArrayRankSpecifiers(); @@ -370,7 +371,7 @@ private ParseResult ParseParameterNode() return ParseResult.Abort; } - typeIdentifier = new StackFrameArrayTypeExpression(typeIdentifier, arrayIdentifiers); + typeIdentifier = new StackFrameArrayTypeNode(nameNode, arrayIdentifiers); } var identifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: true, scanTrailingWhitespace: true); From 9eaa65e5ca379fa438625c1cdc470b0372915f76 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 13:25:42 -0700 Subject: [PATCH 46/59] Fix test build breaks --- .../EmbeddedLanguages/StackFrame/StackFrameParserTests.cs | 4 ++-- .../EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index eeb6827d3d7ae..efa589fe847cc 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -159,7 +159,7 @@ public void TestMethodArrayParam() Identifier("M")), argumentList: ParameterList( - Parameter(ArrayExpression(Identifier("string"), ArrayRankSpecifier(trailingTrivia: SpaceTrivia())), + Parameter(ArrayType(Identifier("string"), ArrayRankSpecifier(trailingTrivia: SpaceTrivia())), IdentifierToken("s"))) ) ); @@ -177,7 +177,7 @@ public void TestCommaArrayParam() argumentList: ParameterList( Parameter( - ArrayExpression(Identifier("string"), ArrayRankSpecifier(1, trailingTrivia: SpaceTrivia())), + ArrayType(Identifier("string"), ArrayRankSpecifier(1, trailingTrivia: SpaceTrivia())), IdentifierToken("s"))) ) ); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 37eac42eaad84..7f7491f722f10 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -54,7 +54,7 @@ public static ImmutableArray CreateTriviaArray(params string[] public static readonly StackFrameParameterList EmptyParams = ParameterList(OpenParenToken, CloseParenToken); - public static StackFrameParameterDeclarationNode Parameter(StackFrameNameNode type, StackFrameToken identifier) + public static StackFrameParameterDeclarationNode Parameter(StackFrameTypeNode type, StackFrameToken identifier) => new(type, identifier); public static StackFrameParameterList ParameterList(params StackFrameParameterDeclarationNode[] parameters) @@ -150,7 +150,7 @@ public static StackFrameIdentifierNameNode Identifier(string name, StackFrameTri public static StackFrameArrayRankSpecifier ArrayRankSpecifier(int commaCount = 0, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) => new(OpenBracketToken.With(leadingTrivia: CreateTriviaArray(leadingTrivia)), CloseBracketToken.With(trailingTrivia: CreateTriviaArray(trailingTrivia)), Enumerable.Repeat(CommaToken, commaCount).ToImmutableArray()); - public static StackFrameArrayTypeNode ArrayExpression(StackFrameNameNode identifier, params StackFrameArrayRankSpecifier[] arrayTokens) + public static StackFrameArrayTypeNode ArrayType(StackFrameNameNode identifier, params StackFrameArrayRankSpecifier[] arrayTokens) => new(identifier, arrayTokens.ToImmutableArray()); public static StackFrameGenericNameNode GenericType(string identifierName, int arity) From 7dec0eb3e3353e71844c557568087d08d91de796 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 15:56:01 -0700 Subject: [PATCH 47/59] Fix tests and add more --- .../StackFrame/StackFrameParserTests.cs | 25 ++++++++++++++++++- .../StackFrame/StackFrameParser.cs | 12 ++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index efa589fe847cc..fc7387da54ee2 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -234,7 +234,9 @@ public void TestGenericMethod() [InlineData("uʶ")] // character and modifier character [InlineData("a\u00AD")] // Soft hyphen formatting character [InlineData("a‿")] // Connecting punctuation (combining character) - + [InlineData("at")] + [InlineData("line")] + [InlineData("in")] public void TestIdentifierNames(string identifierName) => Verify( @$"at {identifierName}.{identifierName}[{identifierName}]({identifierName} {identifierName})", @@ -248,6 +250,26 @@ public void TestIdentifierNames(string identifierName) ) ); + [Fact] + public void TestInvalidSpacingBeforeQualifiedName() + => Verify( + @"at MyNamespace. MyClass.MyMethod()", expectFailure: true); + + [Fact] + public void TestInvalidSpacingAfterQualifiedName2() + => Verify( + @"at MyNamespace.MyClass .MyMethod()", expectFailure: true); + + [Fact] + public void TestWhitespaceAroundBrackets() + => Verify( + @"at MyNamespace.MyClass.MyMethod[ T ]()", + methodDeclaration: MethodDeclaration( + QualifiedName("MyNamespace.MyClass.MyMethod", leadingTrivia: AtTrivia), + typeArguments: TypeArgumentList( + TypeArgument(IdentifierToken("T", leadingTrivia: SpaceTrivia(), trailingTrivia: SpaceTrivia())))) + ); + [Fact] public void TestAnonymousMethod() => Verify( @@ -346,6 +368,7 @@ public void TestFileInformation_InvalidDirectory() [InlineData("M.N(X.Y. x)")] // Trailing . in argument type [InlineData("M.N[T.Y]()")] // Generic type arguments should not be qualified types [InlineData("M.N(X.Y x.y)")] // argument names should not be qualified + [InlineData("M.N(params)")] // argument with type but no name public void TestInvalidInputs(string input) => Verify(input, expectFailure: true); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 23be728b5a6aa..6217dce1afd94 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -148,7 +148,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) return ParseResult.Empty; } - var identifierParseResult = TryScanGenericTypeIdentifier(_lexer, currentIdentifer.Value); + var identifierParseResult = TryScanGenericTypeIdentifier(ref _lexer, currentIdentifer.Value); if (!identifierParseResult.Success) { return ParseResult.Abort; @@ -157,7 +157,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) RoslynDebug.AssertNotNull(identifierParseResult.Value); var lhs = identifierParseResult.Value; - var parseResult = TryParseQualifiedName(_lexer, lhs); + var parseResult = TryParseQualifiedName(ref _lexer, lhs); if (!parseResult.Success) { return ParseResult.Abort; @@ -172,7 +172,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) while (true) { - parseResult = TryParseQualifiedName(_lexer, memberAccess); + parseResult = TryParseQualifiedName(ref _lexer, memberAccess); if (!parseResult.Success) { return ParseResult.Abort; @@ -192,7 +192,7 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) // Given an existing left hand side node or token, which can either be // an or // - static ParseResult TryParseQualifiedName(StackFrameLexer lexer, StackFrameNameNode lhs) + static ParseResult TryParseQualifiedName(ref StackFrameLexer lexer, StackFrameNameNode lhs) { if (!lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) { @@ -205,7 +205,7 @@ static ParseResult TryParseQualifiedName(StackFrame return ParseResult.Abort; } - var (success, rhs) = TryScanGenericTypeIdentifier(lexer, identifier.Value); + var (success, rhs) = TryScanGenericTypeIdentifier(ref lexer, identifier.Value); if (!success) { return ParseResult.Abort; @@ -223,7 +223,7 @@ static ParseResult TryParseQualifiedName(StackFrame // ^-------------- Grave token // ^------------- Arity token of "1" // - static ParseResult TryScanGenericTypeIdentifier(StackFrameLexer lexer, StackFrameToken identifierToken) + static ParseResult TryScanGenericTypeIdentifier(ref StackFrameLexer lexer, StackFrameToken identifierToken) { if (!lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { From 73d38e46dd59281f3be3794d06653637d1b9b51e Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 16:17:33 -0700 Subject: [PATCH 48/59] Refactor --- .../StackFrame/StackFrameParser.cs | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 6217dce1afd94..f999928d44f0a 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -155,42 +155,32 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) } RoslynDebug.AssertNotNull(identifierParseResult.Value); - var lhs = identifierParseResult.Value; - - var parseResult = TryParseQualifiedName(ref _lexer, lhs); - if (!parseResult.Success) - { - return ParseResult.Abort; - } - - var memberAccess = parseResult.Value; - if (memberAccess is null) - { - Debug.Assert(lhs is StackFrameSimpleNameNode); - return new(lhs); - } + StackFrameNameNode nameNode = identifierParseResult.Value; while (true) { - parseResult = TryParseQualifiedName(ref _lexer, memberAccess); - if (!parseResult.Success) + var (success, memberAccess) = TryParseQualifiedName(ref _lexer, nameNode); + if (!success) { return ParseResult.Abort; } - var newMemberAccess = parseResult.Value; - if (newMemberAccess is null) + if (memberAccess is null) { - Debug.Assert(memberAccess is StackFrameQualifiedNameNode); - return new(memberAccess); + Debug.Assert(nameNode is StackFrameQualifiedNameNode or StackFrameSimpleNameNode); + return new(nameNode); } - memberAccess = newMemberAccess; + nameNode = memberAccess; } // - // Given an existing left hand side node or token, which can either be - // an or + // Local Functions + // + + // + // Given an existing left hand side node or token, parse a qualified name if possible. Returns + // null with success if a dot token is not available // static ParseResult TryParseQualifiedName(ref StackFrameLexer lexer, StackFrameNameNode lhs) { From 07007cc191661cd53bb363c56c1830115ac63be9 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 16:45:16 -0700 Subject: [PATCH 49/59] Add space test for array commas --- .../StackFrame/StackFrameParserTests.cs | 21 +++++++++++++++++++ .../StackFrame/StackFrameSyntaxFactory.cs | 3 +++ 2 files changed, 24 insertions(+) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index fc7387da54ee2..ff1dc90255761 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -164,6 +164,25 @@ public void TestMethodArrayParam() ) ); + [Fact] + public void TestMethodArrayParamWithSpace() + => Verify( + "M.N(string[ , , ] s)", + methodDeclaration: MethodDeclaration( + QualifiedName("M.N"), + argumentList: ParameterList( + Parameter( + ArrayType(Identifier("string"), + ArrayRankSpecifier( + OpenBracketToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia())), + CloseBracketToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia())), + CommaToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia())), + CommaToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia())))), + IdentifierToken("s") + ) + )) + ); + [Fact] public void TestCommaArrayParam() => Verify( @@ -369,6 +388,8 @@ public void TestFileInformation_InvalidDirectory() [InlineData("M.N[T.Y]()")] // Generic type arguments should not be qualified types [InlineData("M.N(X.Y x.y)")] // argument names should not be qualified [InlineData("M.N(params)")] // argument with type but no name + [InlineData("M.N [T]()")] // Space between identifier and bracket is not allowed + [InlineData("M.N(string [] s)")] // Space between type and array brackets not allowed public void TestInvalidInputs(string input) => Verify(input, expectFailure: true); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 7f7491f722f10..59b154ae3fb27 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -150,6 +150,9 @@ public static StackFrameIdentifierNameNode Identifier(string name, StackFrameTri public static StackFrameArrayRankSpecifier ArrayRankSpecifier(int commaCount = 0, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) => new(OpenBracketToken.With(leadingTrivia: CreateTriviaArray(leadingTrivia)), CloseBracketToken.With(trailingTrivia: CreateTriviaArray(trailingTrivia)), Enumerable.Repeat(CommaToken, commaCount).ToImmutableArray()); + public static StackFrameArrayRankSpecifier ArrayRankSpecifier(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameToken[] commaTokens) + => new(openToken, closeToken, commaTokens.ToImmutableArray()); + public static StackFrameArrayTypeNode ArrayType(StackFrameNameNode identifier, params StackFrameArrayRankSpecifier[] arrayTokens) => new(identifier, arrayTokens.ToImmutableArray()); From d283218f8feae78a70ba6e28b4bed5e0620487ff Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 17:10:13 -0700 Subject: [PATCH 50/59] Change to TryParseRequired --- .../StackFrame/StackFrameParser.cs | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index f999928d44f0a..abacaa4213ced 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -65,8 +65,8 @@ private StackFrameParser(VirtualCharSequence text) /// private StackFrameTree? TryParseTree() { - var (success, methodDeclaration) = TryParseMethodDeclaration(); - if (!success || methodDeclaration is null) + var methodDeclaration = TryParseRequiredMethodDeclaration(); + if (methodDeclaration is null) { return null; } @@ -95,29 +95,25 @@ private StackFrameParser(VirtualCharSequence text) /// /// Ex: [|MyClass.MyMethod(string s)|] /// - private ParseResult TryParseMethodDeclaration() + private StackFrameMethodDeclarationNode? TryParseRequiredMethodDeclaration() { - var (success, identifierNode) = TryParseNameNode(scanAtTrivia: true); - if (!success) - { - return ParseResult.Abort; - } + var identifierNode = TryParseRequiredNameNode(scanAtTrivia: true); if (identifierNode is not StackFrameQualifiedNameNode memberAccessExpression) { - return ParseResult.Abort; + return null; } - (success, var typeArguments) = TryParseTypeArguments(); + var (success, typeArguments) = TryParseTypeArguments(); if (!success) { - return ParseResult.Abort; + return null; } var methodParameters = TryParseRequiredMethodParameters(); if (methodParameters is null) { - return ParseResult.Abort; + return null; } return new StackFrameMethodDeclarationNode(memberAccessExpression, typeArguments, methodParameters); @@ -138,20 +134,18 @@ private ParseResult TryParseMethodDeclaration() /// /// /// - private ParseResult TryParseNameNode(bool scanAtTrivia) + private StackFrameNameNode? TryParseRequiredNameNode(bool scanAtTrivia) { var currentIdentifer = _lexer.TryScanIdentifier(scanAtTrivia: scanAtTrivia, scanLeadingWhitespace: true, scanTrailingWhitespace: false); if (!currentIdentifer.HasValue) { - // If we can't parse an identifier, no need to abort the parser entirely here. - // The caller should determine if an identifier is required or not. - return ParseResult.Empty; + return null; } var identifierParseResult = TryScanGenericTypeIdentifier(ref _lexer, currentIdentifer.Value); if (!identifierParseResult.Success) { - return ParseResult.Abort; + return null; } RoslynDebug.AssertNotNull(identifierParseResult.Value); @@ -162,13 +156,13 @@ private ParseResult TryParseNameNode(bool scanAtTrivia) var (success, memberAccess) = TryParseQualifiedName(ref _lexer, nameNode); if (!success) { - return ParseResult.Abort; + return null; } if (memberAccess is null) { Debug.Assert(nameNode is StackFrameQualifiedNameNode or StackFrameSimpleNameNode); - return new(nameNode); + return nameNode; } nameNode = memberAccess; @@ -346,8 +340,8 @@ private ParseResult TryParseTypeArguments() /// private ParseResult ParseParameterNode() { - var (success, nameNode) = TryParseNameNode(scanAtTrivia: false); - if (!success || nameNode is null) + var nameNode = TryParseRequiredNameNode(scanAtTrivia: false); + if (nameNode is null) { return ParseResult.Abort; } @@ -355,7 +349,7 @@ private ParseResult ParseParameterNode() StackFrameTypeNode? typeIdentifier = nameNode; if (CurrentCharAsToken().Kind == StackFrameKind.OpenBracketToken) { - (success, var arrayIdentifiers) = ParseArrayRankSpecifiers(); + var (success, arrayIdentifiers) = ParseArrayRankSpecifiers(); if (!success || arrayIdentifiers.IsDefault) { return ParseResult.Abort; From 80e5fcc0bf1cabdb5223b3209032d08ec3e16a80 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 17:23:12 -0700 Subject: [PATCH 51/59] Move local functions back out --- .../StackFrame/StackFrameParser.cs | 98 +++++++++---------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index abacaa4213ced..67efbf95f2648 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -142,18 +142,18 @@ private StackFrameParser(VirtualCharSequence text) return null; } - var identifierParseResult = TryScanGenericTypeIdentifier(ref _lexer, currentIdentifer.Value); - if (!identifierParseResult.Success) + var (success, genericIdentifier) = TryScanGenericTypeIdentifier(currentIdentifer.Value); + if (!success) { return null; } - RoslynDebug.AssertNotNull(identifierParseResult.Value); - StackFrameNameNode nameNode = identifierParseResult.Value; + RoslynDebug.AssertNotNull(genericIdentifier); + StackFrameNameNode nameNode = genericIdentifier; while (true) { - var (success, memberAccess) = TryParseQualifiedName(ref _lexer, nameNode); + (success, var memberAccess) = TryParseQualifiedName(nameNode); if (!success) { return null; @@ -167,61 +167,59 @@ private StackFrameParser(VirtualCharSequence text) nameNode = memberAccess; } + } - // - // Local Functions - // + /// + /// Given an existing left hand side node or token, parse a qualified name if possible. Returns + /// null with success if a dot token is not available + /// + private ParseResult TryParseQualifiedName(StackFrameNameNode lhs) + { + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) + { + return ParseResult.Empty; + } - // - // Given an existing left hand side node or token, parse a qualified name if possible. Returns - // null with success if a dot token is not available - // - static ParseResult TryParseQualifiedName(ref StackFrameLexer lexer, StackFrameNameNode lhs) + var identifier = _lexer.TryScanIdentifier(); + if (!identifier.HasValue) { - if (!lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) - { - return ParseResult.Empty; - } + return ParseResult.Abort; + } - var identifier = lexer.TryScanIdentifier(); - if (!identifier.HasValue) - { - return ParseResult.Abort; - } + var (success, rhs) = TryScanGenericTypeIdentifier(identifier.Value); + if (!success) + { + return ParseResult.Abort; + } - var (success, rhs) = TryScanGenericTypeIdentifier(ref lexer, identifier.Value); - if (!success) - { - return ParseResult.Abort; - } + RoslynDebug.AssertNotNull(rhs); + return new StackFrameQualifiedNameNode(lhs, dotToken, rhs); + } - RoslynDebug.AssertNotNull(rhs); - return new StackFrameQualifiedNameNode(lhs, dotToken, rhs); + /// + /// Given an identifier, attempts to parse the type identifier arity for it. + /// + /// + /// ex: MyNamespace.MyClass`1.MyMethod() + /// ^--------------------- MyClass would be the identifier passed in + /// ^-------------- Grave token + /// ^------------- Arity token of "1" + /// + /// + private ParseResult TryScanGenericTypeIdentifier(StackFrameToken identifierToken) + { + if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) + { + return new(new StackFrameIdentifierNameNode(identifierToken)); } - // - // Given an identifier, attempts to parse the type identifier arity for it. - // - // ex: MyNamespace.MyClass`1.MyMethod() - // ^--------------------- MyClass would be the identifier passed in - // ^-------------- Grave token - // ^------------- Arity token of "1" - // - static ParseResult TryScanGenericTypeIdentifier(ref StackFrameLexer lexer, StackFrameToken identifierToken) + var arity = _lexer.TryScanNumbers(); + if (!arity.HasValue) { - if (!lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) - { - return new(new StackFrameIdentifierNameNode(identifierToken)); - } - - var arity = lexer.TryScanNumbers(); - if (!arity.HasValue) - { - return ParseResult.Abort; - } - - return new StackFrameGenericNameNode(identifierToken, graveAccentToken, arity.Value); + return ParseResult.Abort; } + + return new StackFrameGenericNameNode(identifierToken, graveAccentToken, arity.Value); } /// From 604ec771fb00a98273a6bb80f5c71339aa80c20e Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 17:33:38 -0700 Subject: [PATCH 52/59] Change to Result instead of ParseResult. Allow lexer to have stronger indication of Abort --- .../EmbeddedLanguages/StackFrame/Result.cs | 33 ++++++++++ .../StackFrame/StackFrameKind.cs | 4 -- .../StackFrame/StackFrameLexer.cs | 12 ++-- .../StackFrame/StackFrameNodeDefinitions.cs | 5 +- .../StackFrameParser.ParseResult.cs | 38 ----------- .../StackFrame/StackFrameParser.cs | 65 ++++++++++--------- 6 files changed, 73 insertions(+), 84 deletions(-) create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/Result.cs delete mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/Result.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/Result.cs new file mode 100644 index 0000000000000..253c454ce6459 --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/Result.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + internal readonly struct Result + { + public readonly bool Success; + public readonly T? Value; + + public static readonly Result Abort = new(false, default); + public static readonly Result Empty = new(true, default); + + public Result(T? value) + : this(true, value) + { } + + private Result(bool success, T? value) + { + Success = success; + Value = value; + } + + public void Deconstruct(out bool success, out T? value) + { + success = Success; + value = Value; + } + + public static implicit operator Result(T value) => new(value); + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 2d50f8363224c..70883d434b844 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -2,8 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; - namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { internal enum StackFrameKind @@ -44,9 +42,7 @@ internal enum StackFrameKind ForwardSlashToken, IdentifierToken, PathToken, - InvalidPathToken, NumberToken, - InvalidNumberToken, // Trivia WhitespaceTrivia, diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 63dc270b77ec5..2508d19927f00 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -171,12 +171,12 @@ public bool ScanCurrentCharAsTokenIfMatch(Func isMatch, ou /// Attempts to parse and a path following https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#file-and-directory-names /// Uses as a tool to determine if the path is correct for returning. /// - public StackFrameToken? TryScanPath() + public Result TryScanPath() { var inTrivia = TryScanInTrivia(); if (!inTrivia.HasValue) { - return null; + return Result.Empty; } var startPosition = Position; @@ -214,7 +214,7 @@ public bool ScanCurrentCharAsTokenIfMatch(Func isMatch, ou if (startPosition == Position) { - return CreateToken(StackFrameKind.InvalidPathToken, ImmutableArray.Create(inTrivia.Value), VirtualCharSequence.Empty); + return Result.Abort; } return CreateToken(StackFrameKind.PathToken, ImmutableArray.Create(inTrivia.Value), GetSubSequenceToCurrentPos(startPosition)); @@ -225,18 +225,18 @@ public bool ScanCurrentCharAsTokenIfMatch(Func isMatch, ou /// attached to it. If no numbers are found, returns a token with the trivia that was scanned. /// /// - public StackFrameToken? TryScanLineNumber() + public Result TryScanLineNumber() { var lineTrivia = TryScanLineTrivia(); if (!lineTrivia.HasValue) { - return null; + return Result.Empty; } var numberToken = TryScanNumbers(); if (!numberToken.HasValue) { - return CreateToken(StackFrameKind.InvalidNumberToken, ImmutableArray.Create(lineTrivia.Value), VirtualCharSequence.Empty); + return Result.Abort; } var remainingTrivia = TryScanRemainingTrivia(); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs index d3844e71fb4a9..5c1c921b0cb1d 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameNodeDefinitions.cs @@ -3,20 +3,16 @@ // See the LICENSE file in the project root for more information. using System; -using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; using System.Linq; -using System.Text; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; -using Microsoft.CodeAnalysis.PooledObjects; using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame { using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; using StackFrameToken = EmbeddedSyntaxToken; - using StackFrameTrivia = EmbeddedSyntaxTrivia; internal abstract class StackFrameNode : EmbeddedSyntaxNode { @@ -416,6 +412,7 @@ internal sealed class StackFrameFileInformationNode : StackFrameNode public StackFrameFileInformationNode(StackFrameToken path, StackFrameToken? colon, StackFrameToken? line) : base(StackFrameKind.FileInformation) { + Debug.Assert(path.Kind == StackFrameKind.PathToken); Debug.Assert(colon.HasValue == line.HasValue); Debug.Assert(!line.HasValue || line.Value.Kind == StackFrameKind.NumberToken); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs deleted file mode 100644 index 4b8fcf1261da2..0000000000000 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.ParseResult.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; - -namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame -{ - internal partial struct StackFrameParser - { - private readonly struct ParseResult - { - public readonly bool Success; - public readonly T? Value; - - public static readonly ParseResult Abort = new(false, default); - public static readonly ParseResult Empty = new(true, default); - - public ParseResult(T? value) - : this(true, value) - { } - - private ParseResult(bool success, T? value) - { - Success = success; - Value = value; - } - - public void Deconstruct(out bool success, out T? value) - { - success = Success; - value = Value; - } - - public static implicit operator ParseResult(T value) => new(value); - } - } -} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 67efbf95f2648..bd8f604e41381 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -21,7 +21,7 @@ namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame /// defined as a string line in a StackTrace. See https://docs.microsoft.com/en-us/dotnet/api/system.environment.stacktrace for /// more documentation on dotnet stack traces. /// - internal partial struct StackFrameParser + internal struct StackFrameParser { private StackFrameLexer _lexer; @@ -173,23 +173,23 @@ private StackFrameParser(VirtualCharSequence text) /// Given an existing left hand side node or token, parse a qualified name if possible. Returns /// null with success if a dot token is not available /// - private ParseResult TryParseQualifiedName(StackFrameNameNode lhs) + private Result TryParseQualifiedName(StackFrameNameNode lhs) { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.DotToken, out var dotToken)) { - return ParseResult.Empty; + return Result.Empty; } var identifier = _lexer.TryScanIdentifier(); if (!identifier.HasValue) { - return ParseResult.Abort; + return Result.Abort; } var (success, rhs) = TryScanGenericTypeIdentifier(identifier.Value); if (!success) { - return ParseResult.Abort; + return Result.Abort; } RoslynDebug.AssertNotNull(rhs); @@ -206,7 +206,7 @@ private ParseResult TryParseQualifiedName(StackFram /// ^------------- Arity token of "1" /// /// - private ParseResult TryScanGenericTypeIdentifier(StackFrameToken identifierToken) + private Result TryScanGenericTypeIdentifier(StackFrameToken identifierToken) { if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.GraveAccentToken, out var graveAccentToken)) { @@ -216,7 +216,7 @@ private ParseResult TryScanGenericTypeIdentifier(Stack var arity = _lexer.TryScanNumbers(); if (!arity.HasValue) { - return ParseResult.Abort; + return Result.Abort; } return new StackFrameGenericNameNode(identifierToken, graveAccentToken, arity.Value); @@ -231,13 +231,13 @@ private ParseResult TryScanGenericTypeIdentifier(Stack /// /// Assumes the identifier "MyMethod" has already been parsed, and the type arguments will need to be parsed. /// - private ParseResult TryParseTypeArguments() + private Result TryParseTypeArguments() { if (!_lexer.ScanCurrentCharAsTokenIfMatch( kind => kind is StackFrameKind.OpenBracketToken or StackFrameKind.LessThanToken, out var openToken)) { - return ParseResult.Empty; + return Result.Empty; } var closeBrackeKind = openToken.Kind is StackFrameKind.OpenBracketToken @@ -259,7 +259,7 @@ private ParseResult TryParseTypeArguments() if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CommaToken, out var commaToken)) { - return ParseResult.Abort; + return Result.Abort; } builder.Add(commaToken); @@ -268,12 +268,12 @@ private ParseResult TryParseTypeArguments() if (builder.Count == 0) { - return ParseResult.Abort; + return Result.Abort; } if (closeToken.IsMissing) { - return ParseResult.Abort; + return Result.Abort; } var separatedList = new EmbeddedSeparatedSyntaxNodeList(builder.ToImmutable()); @@ -336,12 +336,12 @@ private ParseResult TryParseTypeArguments() /// ^--------------^ -- Type = "System.String[]" /// ^-- Identifier = "s" /// - private ParseResult ParseParameterNode() + private Result ParseParameterNode() { var nameNode = TryParseRequiredNameNode(scanAtTrivia: false); if (nameNode is null) { - return ParseResult.Abort; + return Result.Abort; } StackFrameTypeNode? typeIdentifier = nameNode; @@ -350,7 +350,7 @@ private ParseResult ParseParameterNode() var (success, arrayIdentifiers) = ParseArrayRankSpecifiers(); if (!success || arrayIdentifiers.IsDefault) { - return ParseResult.Abort; + return Result.Abort; } typeIdentifier = new StackFrameArrayTypeNode(nameNode, arrayIdentifiers); @@ -359,7 +359,7 @@ private ParseResult ParseParameterNode() var identifier = _lexer.TryScanIdentifier(scanAtTrivia: false, scanLeadingWhitespace: true, scanTrailingWhitespace: true); if (!identifier.HasValue) { - return ParseResult.Abort; + return Result.Abort; } return new StackFrameParameterDeclarationNode(typeIdentifier, identifier.Value); @@ -372,7 +372,7 @@ private ParseResult ParseParameterNode() /// 0: "[,] /// 1: "[]" /// - private ParseResult> ParseArrayRankSpecifiers() + private Result> ParseArrayRankSpecifiers() { using var _ = ArrayBuilder.GetInstance(out var builder); using var _1 = ArrayBuilder.GetInstance(out var commaBuilder); @@ -391,7 +391,7 @@ private ParseResult> ParseArrayRank if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.CloseBracketToken, scanTrailingWhitespace: true, out var closeBracket)) { - return ParseResult>.Abort; + return Result>.Abort; } builder.Add(new StackFrameArrayRankSpecifier(openBracket, closeBracket, commaBuilder.ToImmutableAndClear())); @@ -405,35 +405,36 @@ private ParseResult> ParseArrayRank /// Can return if only a path is available but not line numbers, but throws if the value after the path is a colon as the expectation /// is that line number should follow. /// - private ParseResult TryParseFileInformation() + private Result TryParseFileInformation() { - var path = _lexer.TryScanPath(); - if (!path.HasValue) + var (success, path) = _lexer.TryScanPath(); + if (!success) { - return ParseResult.Empty; + return Result.Abort; } - if (path.Value.Kind == StackFrameKind.InvalidPathToken) + if (path.Kind == StackFrameKind.None) { - return ParseResult.Abort; + return Result.Empty; } - Debug.Assert(path.Value.Kind == StackFrameKind.PathToken); - if (!_lexer.ScanCurrentCharAsTokenIfMatch(StackFrameKind.ColonToken, out var colonToken)) { - return new StackFrameFileInformationNode(path.Value, colon: null, line: null); + return new StackFrameFileInformationNode(path, colon: null, line: null); } - var lineNumber = _lexer.TryScanLineNumber(); - if (!lineNumber.HasValue || lineNumber.Value.Kind == StackFrameKind.InvalidNumberToken) + (success, var lineNumber) = _lexer.TryScanLineNumber(); + if (!success) { - return ParseResult.Abort; + return Result.Abort; } - Debug.Assert(lineNumber.Value.Kind == StackFrameKind.NumberToken); + if (lineNumber.Kind == StackFrameKind.None) + { + return new StackFrameFileInformationNode(path, colon: null, line: null); + } - return new StackFrameFileInformationNode(path.Value, colonToken, lineNumber.Value); + return new StackFrameFileInformationNode(path, colonToken, lineNumber); } } } From 1fcea01717fc5ae1ec0f9b74a0f2e99ef692fe9c Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 17:48:22 -0700 Subject: [PATCH 53/59] Fix test. Line number is required if looked for by the lexer --- .../EmbeddedLanguages/StackFrame/StackFrameLexer.cs | 8 ++++---- .../EmbeddedLanguages/StackFrame/StackFrameParser.cs | 9 ++------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 2508d19927f00..a647cb8d25b03 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -222,21 +222,21 @@ public Result TryScanPath() /// /// Returns a number token with the and remainging - /// attached to it. If no numbers are found, returns a token with the trivia that was scanned. + /// attached to it. /// /// - public Result TryScanLineNumber() + public StackFrameToken? TryScanRequiredLineNumber() { var lineTrivia = TryScanLineTrivia(); if (!lineTrivia.HasValue) { - return Result.Empty; + return null; } var numberToken = TryScanNumbers(); if (!numberToken.HasValue) { - return Result.Abort; + return null; } var remainingTrivia = TryScanRemainingTrivia(); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index bd8f604e41381..fddd2d6445802 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -423,17 +423,12 @@ private Result TryParseFileInformation() return new StackFrameFileInformationNode(path, colon: null, line: null); } - (success, var lineNumber) = _lexer.TryScanLineNumber(); - if (!success) + var lineNumber = _lexer.TryScanRequiredLineNumber(); + if (!lineNumber.HasValue) { return Result.Abort; } - if (lineNumber.Kind == StackFrameKind.None) - { - return new StackFrameFileInformationNode(path, colon: null, line: null); - } - return new StackFrameFileInformationNode(path, colonToken, lineNumber); } } From c3582993eeb93cd6b244ae06450d89e18385d16b Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Tue, 2 Nov 2021 21:20:41 -0700 Subject: [PATCH 54/59] Add more tests with invalid spacing --- .../EmbeddedLanguages/StackFrame/StackFrameParserTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index ff1dc90255761..8652df5da2af3 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -388,8 +388,10 @@ public void TestFileInformation_InvalidDirectory() [InlineData("M.N[T.Y]()")] // Generic type arguments should not be qualified types [InlineData("M.N(X.Y x.y)")] // argument names should not be qualified [InlineData("M.N(params)")] // argument with type but no name - [InlineData("M.N [T]()")] // Space between identifier and bracket is not allowed - [InlineData("M.N(string [] s)")] // Space between type and array brackets not allowed + [InlineData("M.N [T]()")] // Space between identifier and bracket + [InlineData("M.N(string [] s)")] // Space between type and array brackets + [InlineData("M.N ()")] // Space between method declaration and parameters + [InlineData("M.N .O.P(string s)")] // Space in type qualified name public void TestInvalidInputs(string input) => Verify(input, expectFailure: true); From 9ca8a9cafa8b976fcc2055250e2a0f47dee0dd94 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 3 Nov 2021 17:04:36 -0700 Subject: [PATCH 55/59] Add extension method --- .../StackFrame/StackFrameParserTests.cs | 11 ++++--- .../StackFrame/StackFrameSyntaxFactory.cs | 9 ++---- .../StackFrame/StackFrameExtensions.cs | 31 +++++++++++++++++++ .../StackFrame/StackFrameLexer.cs | 6 ++-- .../StackFrame/StackFrameParser.cs | 2 +- 5 files changed, 45 insertions(+), 14 deletions(-) create mode 100644 src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameExtensions.cs diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 8652df5da2af3..3b70300122110 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -5,6 +5,7 @@ using Xunit; using System.Collections.Immutable; using static Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame.StackFrameSyntaxFactory; +using static Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame.StackFrameExtensions; namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame { @@ -174,10 +175,10 @@ public void TestMethodArrayParamWithSpace() Parameter( ArrayType(Identifier("string"), ArrayRankSpecifier( - OpenBracketToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia())), - CloseBracketToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia())), - CommaToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia())), - CommaToken.With(trailingTrivia: CreateTriviaArray(SpaceTrivia())))), + OpenBracketToken.With(trailingTrivia: SpaceTrivia().ToImmutableArray()), + CloseBracketToken.With(trailingTrivia: SpaceTrivia().ToImmutableArray()), + CommaToken.With(trailingTrivia: SpaceTrivia().ToImmutableArray()), + CommaToken.With(trailingTrivia: SpaceTrivia().ToImmutableArray()))), IdentifierToken("s") ) )) @@ -377,6 +378,8 @@ public void TestFileInformation_InvalidDirectory() [InlineData(@"at M)")] // MIssing open paren [InlineData(@"at M.M[T>(T t)")] // Mismatched generic opening/close [InlineData(@"at M.M>(T t)")] // Invalid nested generics + [InlineData("at M.M(T t)")] // Invalid generic in parameter [InlineData(@"at M.M(string[ s)")] // Opening array bracket no close [InlineData(@"at M.M(string] s)")] // Close only array bracket [InlineData(@"at M.M(string[][][ s)")] diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 59b154ae3fb27..41cced2af1aa3 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -30,9 +30,6 @@ public static StackFrameToken CreateToken(StackFrameKind kind, string s, Immutab public static StackFrameTrivia CreateTrivia(StackFrameKind kind, string text) => new(kind, CodeAnalysis.EmbeddedLanguages.VirtualChars.VirtualCharSequence.Create(0, text), ImmutableArray.Empty); - public static ImmutableArray CreateTriviaArray(StackFrameTrivia? trivia) - => trivia.HasValue ? ImmutableArray.Create(trivia.Value) : ImmutableArray.Empty; - public static ImmutableArray CreateTriviaArray(params string[] strings) => strings.Select(s => CreateTrivia(StackFrameKind.SkippedTextTrivia, s)).ToImmutableArray(); @@ -92,7 +89,7 @@ public static StackFrameMethodDeclarationNode MethodDeclaration( } public static StackFrameQualifiedNameNode QualifiedName(string s, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) - => QualifiedName(s, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); + => QualifiedName(s, leadingTrivia.ToImmutableArray(), trailingTrivia.ToImmutableArray()); public static StackFrameQualifiedNameNode QualifiedName(string s, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) { @@ -133,7 +130,7 @@ public static StackFrameToken IdentifierToken(string identifierName) => IdentifierToken(identifierName, leadingTrivia: null, trailingTrivia: null); public static StackFrameToken IdentifierToken(string identifierName, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) - => IdentifierToken(identifierName, CreateTriviaArray(leadingTrivia), CreateTriviaArray(trailingTrivia)); + => IdentifierToken(identifierName, leadingTrivia.ToImmutableArray(), trailingTrivia.ToImmutableArray()); public static StackFrameToken IdentifierToken(string identifierName, ImmutableArray leadingTrivia, ImmutableArray trailingTrivia) => CreateToken(StackFrameKind.IdentifierToken, identifierName, leadingTrivia: leadingTrivia, trailingTrivia: trailingTrivia); @@ -148,7 +145,7 @@ public static StackFrameIdentifierNameNode Identifier(string name, StackFrameTri => Identifier(IdentifierToken(name, leadingTrivia, trailingTrivia)); public static StackFrameArrayRankSpecifier ArrayRankSpecifier(int commaCount = 0, StackFrameTrivia? leadingTrivia = null, StackFrameTrivia? trailingTrivia = null) - => new(OpenBracketToken.With(leadingTrivia: CreateTriviaArray(leadingTrivia)), CloseBracketToken.With(trailingTrivia: CreateTriviaArray(trailingTrivia)), Enumerable.Repeat(CommaToken, commaCount).ToImmutableArray()); + => new(OpenBracketToken.With(leadingTrivia: leadingTrivia.ToImmutableArray()), CloseBracketToken.With(trailingTrivia: trailingTrivia.ToImmutableArray()), Enumerable.Repeat(CommaToken, commaCount).ToImmutableArray()); public static StackFrameArrayRankSpecifier ArrayRankSpecifier(StackFrameToken openToken, StackFrameToken closeToken, params StackFrameToken[] commaTokens) => new(openToken, closeToken, commaTokens.ToImmutableArray()); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameExtensions.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameExtensions.cs new file mode 100644 index 0000000000000..31bbc0cfc956b --- /dev/null +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameExtensions.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; + +namespace Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame +{ + using StackFrameTrivia = EmbeddedSyntaxTrivia; + + internal static class StackFrameExtensions + { + /// + /// Creates an with a single value or empty + /// if the has no value + /// + public static ImmutableArray ToImmutableArray(this StackFrameTrivia? trivia) + => trivia.HasValue ? ImmutableArray.Create(trivia.Value) : ImmutableArray.Empty; + + /// + /// Creates an with a single trivia item in it + /// + /// + /// This is created for convenience so callers don't have to have different patterns between nullable and + /// non nullable calues + /// + public static ImmutableArray ToImmutableArray(this StackFrameTrivia trivia) + => ImmutableArray.Create(trivia); + } +} diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index a647cb8d25b03..114f9faf87f86 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -217,7 +217,7 @@ public Result TryScanPath() return Result.Abort; } - return CreateToken(StackFrameKind.PathToken, ImmutableArray.Create(inTrivia.Value), GetSubSequenceToCurrentPos(startPosition)); + return CreateToken(StackFrameKind.PathToken, inTrivia.ToImmutableArray(), GetSubSequenceToCurrentPos(startPosition)); } /// @@ -242,8 +242,8 @@ public Result TryScanPath() var remainingTrivia = TryScanRemainingTrivia(); return numberToken.Value.With( - leadingTrivia: ImmutableArray.Create(lineTrivia.Value), - trailingTrivia: remainingTrivia.HasValue ? ImmutableArray.Create(remainingTrivia.Value) : ImmutableArray.Empty); + leadingTrivia: lineTrivia.ToImmutableArray(), + trailingTrivia: remainingTrivia.ToImmutableArray()); } public StackFrameToken? TryScanNumbers() diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index fddd2d6445802..373022fe5d689 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -79,7 +79,7 @@ private StackFrameParser(VirtualCharSequence text) var remainingTrivia = _lexer.TryScanRemainingTrivia(); - var eolToken = CurrentCharAsToken().With(leadingTrivia: remainingTrivia.HasValue ? ImmutableArray.Create(remainingTrivia.Value) : ImmutableArray.Empty); + var eolToken = CurrentCharAsToken().With(leadingTrivia: remainingTrivia.ToImmutableArray()); Contract.ThrowIfFalse(_lexer.Position == _lexer.Text.Length); Contract.ThrowIfFalse(eolToken.Kind == StackFrameKind.EndOfLine); From 750a12a1e47db674425287d46dbedeb1cf7343b8 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 3 Nov 2021 17:18:52 -0700 Subject: [PATCH 56/59] Add clarifying comment and more tests --- .../StackFrame/StackFrameParserTests.cs | 21 +++++++++++++++++++ .../StackFrame/StackFrameParser.cs | 9 ++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 3b70300122110..9131751790508 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -127,6 +127,26 @@ public void TestMethodOneParam() ) ); + [Fact] + public void TestMethodOneParamSpacing() + => Verify( + @"at ConsoleApp4.MyClass.M( string s )", + methodDeclaration: MethodDeclaration( + QualifiedName( + QualifiedName( + Identifier("ConsoleApp4", leadingTrivia: AtTrivia), + Identifier("MyClass")), + Identifier("M")), + + argumentList: ParameterList( + OpenParenToken.With(trailingTrivia: SpaceTrivia().ToImmutableArray()), + CloseParenToken, + Parameter( + Identifier("string"), + IdentifierToken("s", leadingTrivia: SpaceTrivia(), trailingTrivia: SpaceTrivia()))) + ) + ); + [Fact] public void TestMethodTwoParam() => Verify( @@ -384,6 +404,7 @@ public void TestFileInformation_InvalidDirectory() [InlineData(@"at M.M(string] s)")] // Close only array bracket [InlineData(@"at M.M(string[][][ s)")] [InlineData(@"at M.M(string[[]] s)")] + [InlineData("at M.M(string s, string t,")] // Trailing comma in parameters [InlineData(@"at M.N`.P()")] // Missing numeric for arity [InlineData(@"at M.N`9N.P()")] // Invalid character after arity [InlineData("M.N.P.()")] // Trailing . with no identifier before arguments diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 373022fe5d689..1b30249103446 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -99,6 +99,11 @@ private StackFrameParser(VirtualCharSequence text) { var identifierNode = TryParseRequiredNameNode(scanAtTrivia: true); + // + // TryParseRequiredNameNode does not necessarily return a qualified name even if + // it parses a name. For method declarations, a fully qualified name is required so + // we know both the class (and namespace) that the method is contained in. + // if (identifierNode is not StackFrameQualifiedNameNode memberAccessExpression) { return null; @@ -240,7 +245,7 @@ private Result TryParseTypeArguments() return Result.Empty; } - var closeBrackeKind = openToken.Kind is StackFrameKind.OpenBracketToken + var closeBracketKind = openToken.Kind is StackFrameKind.OpenBracketToken ? StackFrameKind.CloseBracketToken : StackFrameKind.GreaterThanToken; @@ -252,7 +257,7 @@ private Result TryParseTypeArguments() { builder.Add(new StackFrameIdentifierNameNode(currentIdentifier.Value)); - if (_lexer.ScanCurrentCharAsTokenIfMatch(closeBrackeKind, out closeToken)) + if (_lexer.ScanCurrentCharAsTokenIfMatch(closeBracketKind, out closeToken)) { break; } From eaeb334f9fc0c3a58cfa9577df9a70bf41a05bae Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 3 Nov 2021 17:50:13 -0700 Subject: [PATCH 57/59] Be strict about \r and \n characters. --- .../StackFrameParserTests.Utilities.cs | 2 +- .../StackFrame/StackFrameParserTests.cs | 6 ++++ .../StackFrame/StackFrameSyntaxFactory.cs | 2 +- .../StackFrame/StackFrameKind.cs | 2 +- .../StackFrame/StackFrameLexer.cs | 35 ++++++++++++++++--- .../StackFrame/StackFrameParser.cs | 14 +++++--- 6 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 8fa87009120db..3d46a631b3d15 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -59,7 +59,7 @@ private static void Verify( var eolToken = eolTokenOpt.HasValue ? eolTokenOpt.Value - : CreateToken(StackFrameKind.EndOfLine, ""); + : CreateToken(StackFrameKind.EndOfFrame, ""); AssertEqual(eolToken, tree.Root.EndOfLineToken); } diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs index 9131751790508..666f56a11c4b0 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.cs @@ -416,6 +416,12 @@ public void TestFileInformation_InvalidDirectory() [InlineData("M.N(string [] s)")] // Space between type and array brackets [InlineData("M.N ()")] // Space between method declaration and parameters [InlineData("M.N .O.P(string s)")] // Space in type qualified name + [InlineData("\r\nM.N()")] + [InlineData("\nM.N()")] + [InlineData("\rM.N()")] + [InlineData("M.N(\r\n)")] + [InlineData("M.N(\r)")] + [InlineData("M.N(\n)")] public void TestInvalidInputs(string input) => Verify(input, expectFailure: true); diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs index 41cced2af1aa3..8830e4ae412b4 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameSyntaxFactory.cs @@ -42,7 +42,7 @@ public static ImmutableArray CreateTriviaArray(params string[] public static readonly StackFrameToken LessThanToken = CreateToken(StackFrameKind.LessThanToken, "<"); public static readonly StackFrameToken GreaterThanToken = CreateToken(StackFrameKind.GreaterThanToken, ">"); public static readonly StackFrameToken GraveAccentToken = CreateToken(StackFrameKind.GraveAccentToken, "`"); - public static readonly StackFrameToken EOLToken = CreateToken(StackFrameKind.EndOfLine, ""); + public static readonly StackFrameToken EOLToken = CreateToken(StackFrameKind.EndOfFrame, ""); public static readonly StackFrameToken ColonToken = CreateToken(StackFrameKind.ColonToken, ":"); public static readonly StackFrameTrivia AtTrivia = CreateTrivia(StackFrameKind.AtTrivia, "at "); diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs index 70883d434b844..262a4425e9962 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameKind.cs @@ -22,7 +22,7 @@ internal enum StackFrameKind FileInformation, // Tokens - EndOfLine, + EndOfFrame, AmpersandToken, OpenBracketToken, CloseBracketToken, diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs index 114f9faf87f86..5d329d25cff24 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameLexer.cs @@ -23,14 +23,40 @@ internal struct StackFrameLexer public readonly VirtualCharSequence Text; public int Position { get; private set; } - public StackFrameLexer(string text) + private StackFrameLexer(string text) : this(VirtualCharSequence.Create(0, text)) { } - public StackFrameLexer(VirtualCharSequence text) : this() + private StackFrameLexer(VirtualCharSequence text) : this() => Text = text; + public static StackFrameLexer? TryCreate(string text) + { + foreach (var c in text) + { + if (c == '\r' || c == '\n') + { + return null; + } + } + + return new(text); + } + + public static StackFrameLexer? TryCreate(VirtualCharSequence text) + { + foreach (var c in text) + { + if (c.Value == '\r' || c.Value == '\n') + { + return null; + } + } + + return new(text); + } + public VirtualChar CurrentChar => Position < Text.Length ? Text[Position] : default; public VirtualCharSequence GetSubSequenceToCurrentPos(int start) @@ -92,7 +118,7 @@ public StackFrameToken CurrentCharAsToken() { if (Position == Text.Length) { - return CreateToken(StackFrameKind.EndOfLine, VirtualCharSequence.Empty); + return CreateToken(StackFrameKind.EndOfFrame, VirtualCharSequence.Empty); } var ch = Text[Position]; @@ -361,7 +387,8 @@ private bool IsAtStartOfText(int position, string val) private static StackFrameKind GetKind(VirtualChar ch) => ch.Value switch { - '\n' => StackFrameKind.EndOfLine, + '\n' => throw new InvalidOperationException(), + '\r' => throw new InvalidOperationException(), '&' => StackFrameKind.AmpersandToken, '[' => StackFrameKind.OpenBracketToken, ']' => StackFrameKind.CloseBracketToken, diff --git a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs index 1b30249103446..5ce37d2e5e994 100644 --- a/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs +++ b/src/Features/Core/Portable/EmbeddedLanguages/StackFrame/StackFrameParser.cs @@ -25,9 +25,9 @@ internal struct StackFrameParser { private StackFrameLexer _lexer; - private StackFrameParser(VirtualCharSequence text) + private StackFrameParser(StackFrameLexer lexer) { - _lexer = new(text); + _lexer = lexer; } private StackFrameToken CurrentCharAsToken() => _lexer.CurrentCharAsToken(); @@ -46,7 +46,13 @@ private StackFrameParser(VirtualCharSequence text) try { - return new StackFrameParser(text).TryParseTree(); + var lexer = StackFrameLexer.TryCreate(text); + if (!lexer.HasValue) + { + return null; + } + + return new StackFrameParser(lexer.Value).TryParseTree(); } catch (InsufficientExecutionStackException) { @@ -82,7 +88,7 @@ private StackFrameParser(VirtualCharSequence text) var eolToken = CurrentCharAsToken().With(leadingTrivia: remainingTrivia.ToImmutableArray()); Contract.ThrowIfFalse(_lexer.Position == _lexer.Text.Length); - Contract.ThrowIfFalse(eolToken.Kind == StackFrameKind.EndOfLine); + Contract.ThrowIfFalse(eolToken.Kind == StackFrameKind.EndOfFrame); var root = new StackFrameCompilationUnit(methodDeclaration, fileInformationResult.Value, eolToken); From 62d4bad2ec48283840049e88cf3143b29c29d2f1 Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 3 Nov 2021 18:00:39 -0700 Subject: [PATCH 58/59] Add fuzzy testing for all inputs to ensure no crash --- .../StackFrameParserTests.Utilities.cs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 3d46a631b3d15..7fc22c323089b 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -29,6 +29,8 @@ private static void Verify( StackFrameFileInformationNode? fileInformation = null, StackFrameToken? eolTokenOpt = null) { + FuzzyTest(input); + var tree = StackFrameParser.TryParse(input); if (expectFailure) { @@ -64,6 +66,26 @@ private static void Verify( AssertEqual(eolToken, tree.Root.EndOfLineToken); } + /// + /// Tests that with a given input, no crashes are found + /// with multiple substrings of the input + /// + private static void FuzzyTest(string input) + { + for (var i = 0; i < input.Length - 1; i++) + { + StackFrameParser.TryParse(input[i..]); + StackFrameParser.TryParse(input[..^i]); + + for (var j = 0; j + i < input.Length; j++) + { + var start = input[..j]; + var end = input[(j + i)..]; + StackFrameParser.TryParse(start + end); + } + } + } + private static void AssertEqual(StackFrameNodeOrToken expected, StackFrameNodeOrToken actual) { Assert.Equal(expected.IsNode, actual.IsNode); From a4013bbb7607175f7387672322ed223e6645a9af Mon Sep 17 00:00:00 2001 From: Andrew Hall Date: Wed, 3 Nov 2021 18:45:00 -0700 Subject: [PATCH 59/59] Sort usings --- .../StackFrame/StackFrameParserTests.Utilities.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs index 7fc22c323089b..c29d0622b5c08 100644 --- a/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs +++ b/src/EditorFeatures/Test/EmbeddedLanguages/StackFrame/StackFrameParserTests.Utilities.cs @@ -2,23 +2,23 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Text; -using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Microsoft.CodeAnalysis.EmbeddedLanguages.Common; using Microsoft.CodeAnalysis.EmbeddedLanguages.StackFrame; +using Microsoft.CodeAnalysis.EmbeddedLanguages.VirtualChars; using Roslyn.Test.Utilities; using Xunit; -using System; using static Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame.StackFrameSyntaxFactory; namespace Microsoft.CodeAnalysis.Editor.UnitTests.EmbeddedLanguages.StackFrame { + using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; using StackFrameToken = EmbeddedSyntaxToken; using StackFrameTrivia = EmbeddedSyntaxTrivia; - using StackFrameNodeOrToken = EmbeddedSyntaxNodeOrToken; public partial class StackFrameParserTests {