diff --git a/src/Microsoft.DotNet.Interactive.Documents/CodeSubmission.cs b/src/Microsoft.DotNet.Interactive.Documents/CodeSubmission.cs index f26651a30b..5a8e08b280 100644 --- a/src/Microsoft.DotNet.Interactive.Documents/CodeSubmission.cs +++ b/src/Microsoft.DotNet.Interactive.Documents/CodeSubmission.cs @@ -53,7 +53,7 @@ public static InteractiveDocument Parse( metadata = JsonSerializer.Deserialize>(metadataString, InteractiveDocument.JsonSerializerOptions); - if (InteractiveDocument.TryGetKernelInfoFromMetadata(metadata, out var kernelInfoFromMetadata)) + if (InteractiveDocument.TryGetKernelInfosFromMetadata(metadata, out var kernelInfoFromMetadata)) { InteractiveDocument.MergeKernelInfos(kernelInfos, kernelInfoFromMetadata); document.Metadata["kernelInfo"] = kernelInfoFromMetadata; diff --git a/src/Microsoft.DotNet.Interactive.Documents/InteractiveDocument.cs b/src/Microsoft.DotNet.Interactive.Documents/InteractiveDocument.cs index 68e60d3637..839ed2dcc6 100644 --- a/src/Microsoft.DotNet.Interactive.Documents/InteractiveDocument.cs +++ b/src/Microsoft.DotNet.Interactive.Documents/InteractiveDocument.cs @@ -65,7 +65,7 @@ public async IAsyncEnumerable GetImportsAsync(bool recursiv { EnsureImportFieldParserIsInitialized(); - if (!TryGetKernelInfoFromMetadata(Metadata, out var kernelInfos)) + if (!TryGetKernelInfosFromMetadata(Metadata, out var kernelInfos)) { kernelInfos = new(); } @@ -198,7 +198,8 @@ public static async Task LoadAsync( public string? GetDefaultKernelName() { - if (TryGetKernelInfoFromMetadata(Metadata, out var kernelInfo)) + // FIX: (GetDefaultKernelName) remove from public API + if (TryGetKernelInfosFromMetadata(Metadata, out var kernelInfo)) { return kernelInfo.DefaultKernelName; } @@ -208,7 +209,7 @@ public static async Task LoadAsync( internal string? GetDefaultKernelName(KernelInfoCollection kernelInfos) { - if (TryGetKernelInfoFromMetadata(Metadata, out var kernelInfoCollection)) + if (TryGetKernelInfosFromMetadata(Metadata, out var kernelInfoCollection)) { return kernelInfoCollection.DefaultKernelName; } @@ -243,7 +244,7 @@ public static async Task LoadAsync( internal static void MergeKernelInfos(InteractiveDocument document, KernelInfoCollection kernelInfos) { - if (TryGetKernelInfoFromMetadata(document.Metadata, out var kernelInfoCollection)) + if (TryGetKernelInfosFromMetadata(document.Metadata, out var kernelInfoCollection)) { MergeKernelInfos(kernelInfoCollection, kernelInfos); } @@ -264,7 +265,7 @@ internal static void MergeKernelInfos(KernelInfoCollection destination, KernelIn destination.AddRange(source.Where(ki => added.Add(ki.Name))); } - internal static bool TryGetKernelInfoFromMetadata( + internal static bool TryGetKernelInfosFromMetadata( IDictionary? metadata, [NotNullWhen(true)] out KernelInfoCollection? kernelInfo) { diff --git a/src/Microsoft.DotNet.Interactive.Documents/InteractiveDocumentElement.cs b/src/Microsoft.DotNet.Interactive.Documents/InteractiveDocumentElement.cs index 24a8a6fd2d..fb7f47fba7 100644 --- a/src/Microsoft.DotNet.Interactive.Documents/InteractiveDocumentElement.cs +++ b/src/Microsoft.DotNet.Interactive.Documents/InteractiveDocumentElement.cs @@ -6,7 +6,7 @@ namespace Microsoft.DotNet.Interactive.Documents; -public class InteractiveDocumentElement +public sealed class InteractiveDocumentElement { [JsonConstructor] public InteractiveDocumentElement() diff --git a/src/Microsoft.DotNet.Interactive.Formatting/PlainTextSummaryFormatter.cs b/src/Microsoft.DotNet.Interactive.Formatting/PlainTextSummaryFormatter.cs index f874c2132e..7fbc8a32b6 100644 --- a/src/Microsoft.DotNet.Interactive.Formatting/PlainTextSummaryFormatter.cs +++ b/src/Microsoft.DotNet.Interactive.Formatting/PlainTextSummaryFormatter.cs @@ -8,7 +8,6 @@ namespace Microsoft.DotNet.Interactive.Formatting; public static class PlainTextSummaryFormatter { - // FIX: (PlainTextSummaryFormatter) rename this public const string MimeType = "text/plain+summary"; public static ITypeFormatter GetPreferredFormatterFor(Type type) diff --git a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Body.cs b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Body.cs index 901a0523d4..cf03bf63bf 100644 --- a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Body.cs +++ b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Body.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Linq; +using System; using FluentAssertions; using Microsoft.DotNet.Interactive.HttpRequest.Tests.Utility; using Xunit; @@ -12,27 +12,6 @@ public partial class ParserTests { public class Body { - [Fact] - public void body_separator_is_present() - { - var result = Parse( - """ - POST https://example.com/comments HTTP/1.1 - Content-Type: application/xml - Authorization: token xxx - - - sample - - - """); - - var requestNode = result.SyntaxTree.RootNode - .ChildNodes.Should().ContainSingle().Which; - - requestNode.BodySeparatorNode.ChildTokens.First().Kind.Should().Be(HttpTokenKind.NewLine); - } - [Fact] public void body_is_parsed_correctly_when_headers_are_not_present() { @@ -87,5 +66,23 @@ public void multiple_new_lines_before_body_are_parsed_correctly() """); } + + [Fact] + public void Whitespace_after_headers_is_not_parsed_as_body() + { + var code = """ + + hptps://example.com + Accept: */* + + + + + """; + + var result = Parse(code); + + result.SyntaxTree.RootNode.DescendantNodesAndTokens().Should().NotContain(n => n is HttpBodyNode); + } } } \ No newline at end of file diff --git a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Combinatorial.cs b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Combinatorial.cs index 5db22c9c4e..3b74cf5a9e 100644 --- a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Combinatorial.cs +++ b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Combinatorial.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using FluentAssertions; +using Microsoft.DotNet.Interactive.Formatting; using Microsoft.DotNet.Interactive.HttpRequest.Tests.Utility; using Xunit; using Xunit.Abstractions; @@ -13,6 +14,8 @@ namespace Microsoft.DotNet.Interactive.HttpRequest.Tests; public partial class ParserTests { + private readonly ITestOutputHelper _output; + public class Combinatorial { private readonly ITestOutputHelper _output; diff --git a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Headers.cs b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Headers.cs index b2a62f3d82..8f8b91e73f 100644 --- a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Headers.cs +++ b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Headers.cs @@ -1,6 +1,7 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; using System.Linq; using FluentAssertions; using Microsoft.DotNet.Interactive.HttpRequest.Tests.Utility; @@ -74,6 +75,25 @@ public void header_separator_is_parsed() .Which.Text.Should().Be(":"); } + [Fact] + public void Comments_can_precede_headers() + { + var code = """ + POST https://example.com + # this is a comment + Accept: */* + Accept-Encoding: gzip, deflate, br + """; + + var result = Parse(code); + + var requestNode = result.SyntaxTree.RootNode.DescendantNodesAndTokens() + .Should().ContainSingle().Which; + + requestNode.DescendantNodesAndTokens().Should().ContainSingle().Which.Text.Should().Be("# this is a comment"); + requestNode.ChildNodes.Should().ContainSingle(); + } + [Fact] public void headers_are_parsed_correctly() { diff --git a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Url.cs b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Url.cs index 32765fc395..cbe914ef34 100644 --- a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Url.cs +++ b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.Url.cs @@ -32,9 +32,9 @@ public void newline_is_legal_at_the_after_url() """); - result.SyntaxTree.RootNode - .ChildNodes.Should().ContainSingle().Which - .ChildTokens.Last().Kind.Should().Be(HttpTokenKind.NewLine); + result.SyntaxTree.RootNode.ChildNodes.Should().ContainSingle(); + + result.GetDiagnostics().Should().BeEmpty(); } [Theory] diff --git a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.cs b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.cs index ad92f444f5..e6231093c3 100644 --- a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.cs +++ b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/ParserTests.cs @@ -2,8 +2,10 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Linq; using FluentAssertions; using FluentAssertions.Execution; +using Xunit.Abstractions; namespace Microsoft.DotNet.Interactive.HttpRequest.Tests; @@ -11,8 +13,9 @@ public partial class ParserTests : IDisposable { private readonly AssertionScope _assertionScope; - public ParserTests() + public ParserTests(ITestOutputHelper output) { + _output = output; _assertionScope = new AssertionScope(); } @@ -24,9 +27,9 @@ public void Dispose() private static HttpRequestParseResult Parse(string code) { var result = HttpRequestParser.Parse(code); - - result.SyntaxTree.RootNode.FullText.Should().Be(code); + result.SyntaxTree.RootNode.FullText.Should().Be(code); + return result; } } diff --git a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/Utility/HttpBodyNodeSyntaxSpec.cs b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/Utility/HttpBodyNodeSyntaxSpec.cs index e053e6156c..bd18736182 100644 --- a/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/Utility/HttpBodyNodeSyntaxSpec.cs +++ b/src/Microsoft.DotNet.Interactive.HttpRequest.Tests/Utility/HttpBodyNodeSyntaxSpec.cs @@ -184,7 +184,6 @@ public override string ToString() sb.Append(MaybeWhitespace()); sb.AppendLine(); sb.Append(MaybeLineComment()); - sb.Append(MaybeNewLines()); if (HeadersSection is not null) { @@ -201,8 +200,6 @@ public override string ToString() sb.Append(MaybeNewLines()); } - sb.Append(MaybeLineComment()); - return sb.ToString(); } @@ -227,8 +224,8 @@ private string MaybeNewLines() => private string MaybeLineComment() => ExtraTriviaRandomizer?.NextDouble() switch { - // FIX: (MaybeLineComment) - < 0 => "# random line comment", + < .2 => "# random line comment followed by a LF\n", + > .2 and < .4 => "# random line comment followed by a CRLF\r\n", _ => "" }; } diff --git a/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpBodySeparatorNode.cs b/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpBodySeparatorNode.cs deleted file mode 100644 index 7fc81106a5..0000000000 --- a/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpBodySeparatorNode.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -#nullable enable - -using Microsoft.CodeAnalysis.Text; - -namespace Microsoft.DotNet.Interactive.HttpRequest; - -internal class HttpBodySeparatorNode : HttpSyntaxNode -{ - internal HttpBodySeparatorNode(SourceText sourceText, HttpSyntaxTree? syntaxTree) : base(sourceText, syntaxTree) - { - } -} diff --git a/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpRequestNode.cs b/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpRequestNode.cs index 9e005e579c..5a0604ad44 100644 --- a/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpRequestNode.cs +++ b/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpRequestNode.cs @@ -27,8 +27,6 @@ internal HttpRequestNode(SourceText sourceText, HttpSyntaxTree? syntaxTree) : ba public HttpHeadersNode? HeadersNode { get; private set; } - public HttpBodySeparatorNode? BodySeparatorNode { get; private set; } - public HttpBodyNode? BodyNode { get; private set; } public void Add(HttpMethodNode node) @@ -71,16 +69,6 @@ public void Add(HttpHeadersNode node) AddInternal(node); } - public void Add(HttpBodySeparatorNode node) - { - if (BodySeparatorNode is not null) - { - throw new InvalidOperationException($"{nameof(BodySeparatorNode)} was already added."); - } - BodySeparatorNode = node; - AddInternal(node); - } - public void Add(HttpBodyNode node) { if (BodyNode is not null) diff --git a/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpRequestParser.cs b/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpRequestParser.cs index 0c91b02932..1698a56d93 100644 --- a/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpRequestParser.cs +++ b/src/Microsoft.DotNet.Interactive.HttpRequestParser/Parser/HttpRequestParser.cs @@ -3,7 +3,6 @@ #nullable enable -using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.Text; using System.Collections.Generic; using System.Diagnostics; @@ -74,17 +73,36 @@ public HttpSyntaxTree Parse() if (ParseRequestSeparator() is { } separatorNode) { + // FIX: (Parse) uncovered _syntaxTree.RootNode.Add(separatorNode); } - } return _syntaxTree; } - private IEnumerable? ParseVariableDeclarations() + private static void AppendCommentsIfAny(List commentsToAppend, HttpSyntaxNode prependToNode) { + foreach (var comment in commentsToAppend) + { + prependToNode.Add(comment, addBefore: false); + } + + commentsToAppend.Clear(); + } + + private static void PrependCommentsIfAny(List commentsToPrepend, HttpSyntaxNode prependToNode) + { + foreach (var comment in commentsToPrepend) + { + prependToNode.Add(comment, addBefore: true); + } + + commentsToPrepend.Clear(); + } + private IEnumerable? ParseVariableDeclarations() + { while (MoreTokens()) { if (GetNextSignificantToken() is { Kind: HttpTokenKind.Punctuation } and { Text: "@" }) @@ -104,7 +122,6 @@ public HttpSyntaxTree Parse() break; } } - } private HttpVariableValueNode? ParseVariableValue() @@ -120,7 +137,7 @@ public HttpSyntaxTree Parse() { node = new HttpVariableValueNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); } else { @@ -139,7 +156,7 @@ public HttpSyntaxTree Parse() } return node is not null - ? ParseTrailingTrivia(node, stopAfterNewLine: true) + ? ParseTrailingWhitespace(node, stopAfterNewLine: true) : null; } @@ -147,14 +164,14 @@ private HttpVariableAssignmentNode ParserVariableAssignment() { var node = new HttpVariableAssignmentNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); - if (MoreTokens() && CurrentToken is { Kind: HttpTokenKind.Word } and { Text: "=" }) + if (MoreTokens() && CurrentToken is { Text: "=" }) { ConsumeCurrentTokenInto(node); } - return ParseTrailingTrivia(node); + return ParseTrailingWhitespace(node); } private HttpVariableDeclarationNode ParseVariableDeclaration() @@ -163,11 +180,11 @@ private HttpVariableDeclarationNode ParseVariableDeclaration() if (MoreTokens()) { - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); while (MoreTokens()) { - if (CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "@" }) + if (CurrentToken is { Text: "@" }) { ConsumeCurrentTokenInto(node); } @@ -182,7 +199,7 @@ private HttpVariableDeclarationNode ParseVariableDeclaration() } - return ParseTrailingTrivia(node); + return ParseTrailingWhitespace(node); } private HttpSyntaxToken CurrentToken => _tokens![_currentTokenIndex]; @@ -204,15 +221,17 @@ private HttpVariableDeclarationNode ParseVariableDeclaration() [DebuggerStepThrough] private bool MoreTokens() => _tokens!.Count > _currentTokenIndex; + [DebuggerStepThrough] private void AdvanceToNextToken() => _currentTokenIndex++; + [DebuggerStepThrough] private void ConsumeCurrentTokenInto(HttpSyntaxNode node) { node.Add(CurrentToken); AdvanceToNextToken(); } - private T ParseLeadingTrivia(T node) where T : HttpSyntaxNode + private T ParseLeadingWhitespaceAndComments(T node) where T : HttpSyntaxNode { while (MoreTokens()) { @@ -224,22 +243,9 @@ private T ParseLeadingTrivia(T node) where T : HttpSyntaxNode { ConsumeCurrentTokenInto(node); } - else if (CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "#" } && - !(CurrentTokenPlus(1) is { Kind: HttpTokenKind.Punctuation } and { Text: "#" } && - CurrentTokenPlus(2) is { Kind: HttpTokenKind.Punctuation } and { Text: "#" })) + else if (ParseComment() is { } commentNode) { - if (ParseComment() is { } commentNode) - { - node.Add(commentNode, addBefore: true); - } - } - else if (CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "/" } && - CurrentTokenPlus(1) is { Kind: HttpTokenKind.Punctuation } and { Text: "/" }) - { - if (ParseComment() is { } commentNode) - { - node.Add(commentNode, addBefore: true); - } + node.Add(commentNode, addBefore: true); } else { @@ -250,13 +256,13 @@ private T ParseLeadingTrivia(T node) where T : HttpSyntaxNode return node; } - private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool stopBeforeNewline = false) where T : HttpSyntaxNode + private T ParseTrailingWhitespace(T node, bool stopAfterNewLine = false, bool stopBeforeNewLine = false) where T : HttpSyntaxNode { while (MoreTokens()) { if (CurrentToken.Kind is HttpTokenKind.NewLine) { - if (stopBeforeNewline) + if (stopBeforeNewLine) { break; } @@ -281,16 +287,17 @@ private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool sto private HttpRequestNode? ParseRequest() { - if (IsComment()) { + // FIX: (ParseRequest) this looks suspect... test with a comment at the start of the request node? return null; } + var requestNode = new HttpRequestNode( _sourceText, _syntaxTree); - ParseLeadingTrivia(requestNode); + ParseLeadingWhitespaceAndComments(requestNode); var methodNode = ParseMethod(); if (methodNode is not null) @@ -318,30 +325,37 @@ private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool sto { requestNode.Add(versionNode); } - else + else { - ParseTrailingTrivia(requestNode, stopAfterNewLine: true); + ParseTrailingWhitespace(requestNode, stopAfterNewLine: true); + } + + var commentsToPrepend = new List(); + if (ParseComment() is { } comment) + { + commentsToPrepend.Add(comment); } var headersNode = ParseHeaders(); if (headersNode is not null) { + PrependCommentsIfAny(commentsToPrepend, headersNode); requestNode.Add(headersNode); } - var bodySeparatorNode = ParseBodySeparator(); - if (bodySeparatorNode is not null) - { - requestNode.Add(bodySeparatorNode); - } + ParseTrailingWhitespace(requestNode); var bodyNode = ParseBody(); if (bodyNode is not null) { requestNode.Add(bodyNode); } + else + { + PrependCommentsIfAny(commentsToPrepend, requestNode); + } - ParseTrailingTrivia(requestNode); + ParseTrailingWhitespace(requestNode); return requestNode; } @@ -351,18 +365,13 @@ private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool sto if (MoreTokens() && IsRequestSeparator()) { var node = new HttpRequestSeparatorNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); ConsumeCurrentTokenInto(node); ConsumeCurrentTokenInto(node); ConsumeCurrentTokenInto(node); - - while (MoreTokens() && CurrentToken.Kind is not (HttpTokenKind.NewLine or HttpTokenKind.Whitespace)) - { - ConsumeCurrentTokenInto(node); - } - - ParseTrailingTrivia(node); + + ParseTrailingWhitespace(node); } return null; @@ -379,7 +388,7 @@ private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool sto { node = new HttpMethodNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); if (CurrentToken.Text.ToLower() is not ("get" or "post" or "patch" or "put" or "delete" or "head" or "options" or "trace")) { @@ -392,7 +401,7 @@ private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool sto ConsumeCurrentTokenInto(node); - ParseTrailingTrivia(node, stopBeforeNewline: true); + ParseTrailingWhitespace(node, stopBeforeNewLine: true); } } @@ -412,7 +421,7 @@ private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool sto { node = new HttpUrlNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); } else { @@ -431,7 +440,7 @@ private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool sto } return node is not null - ? ParseTrailingTrivia(node, stopBeforeNewline: true) + ? ParseTrailingWhitespace(node, stopBeforeNewLine: true) : null; } @@ -465,11 +474,9 @@ private T ParseTrailingTrivia(T node, bool stopAfterNewLine = false, bool sto return null; } - private bool IsAtStartOfEmbeddedExpression() - { - return CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "{" } && - CurrentTokenPlus(1) is { Kind: HttpTokenKind.Punctuation } and { Text: "{" }; - } + private bool IsAtStartOfEmbeddedExpression() => + CurrentToken is { Text: "{" } && + CurrentTokenPlus(1) is { Text: "{" }; private HttpEmbeddedExpressionNode ParseEmbeddedExpression() { @@ -493,13 +500,13 @@ private HttpExpressionStartNode ParseExpressionStart() ConsumeCurrentTokenInto(node); // parse the first { ConsumeCurrentTokenInto(node); // parse the second { - return ParseTrailingTrivia(node); + return ParseTrailingWhitespace(node); } private HttpExpressionNode ParseExpression() { var node = new HttpExpressionNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); while (MoreTokens() && !(CurrentToken is { Text: "}" } && @@ -508,7 +515,7 @@ private HttpExpressionNode ParseExpression() ConsumeCurrentTokenInto(node); } - return ParseTrailingTrivia(node); + return ParseTrailingWhitespace(node); } private HttpExpressionEndNode? ParseExpressionEnd() @@ -537,7 +544,7 @@ CurrentToken.Text is "}" && { var node = new HttpVersionNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); ConsumeCurrentTokenInto(node); while (MoreTokens() && CurrentToken.Kind is not HttpTokenKind.NewLine && @@ -546,7 +553,7 @@ CurrentToken.Text is "}" && ConsumeCurrentTokenInto(node); } - return ParseTrailingTrivia(node, stopAfterNewLine: true); + return ParseTrailingWhitespace(node, stopAfterNewLine: true); } return null; @@ -557,7 +564,7 @@ CurrentToken.Text is "}" && HttpHeadersNode? headersNode = null; while (MoreTokens() && - (CurrentToken is { Kind: HttpTokenKind.Word } || CurrentToken is { Text: ":" })) + CurrentToken is { Kind: HttpTokenKind.Word } or { Text: ":" }) { headersNode ??= new HttpHeadersNode(_sourceText, _syntaxTree); @@ -585,7 +592,7 @@ private HttpHeaderNameNode ParseHeaderName() if (MoreTokens() && CurrentToken is not { Kind: HttpTokenKind.NewLine } and not { Kind: HttpTokenKind.Punctuation, Text: ":" }) { - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); if (MoreTokens()) { @@ -603,28 +610,28 @@ private HttpHeaderNameNode ParseHeaderName() } } - return ParseTrailingTrivia(node); + return ParseTrailingWhitespace(node); } private HttpHeaderSeparatorNode ParserHeaderSeparator() { var node = new HttpHeaderSeparatorNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); if (MoreTokens() && CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: ":" }) { ConsumeCurrentTokenInto(node); } - return ParseTrailingTrivia(node); + return ParseTrailingWhitespace(node); } private HttpHeaderValueNode ParseHeaderValue() { var node = new HttpHeaderValueNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); while (MoreTokens() && CurrentToken.Kind is not HttpTokenKind.NewLine) { @@ -639,34 +646,23 @@ private HttpHeaderValueNode ParseHeaderValue() } } - return ParseTrailingTrivia(node, stopAfterNewLine: true); + return ParseTrailingWhitespace(node, stopAfterNewLine: true); } - private HttpBodySeparatorNode? ParseBodySeparator() + private HttpBodyNode? ParseBody() { - if (!MoreTokens() || IsRequestSeparator()) + if (!MoreTokens()) { return null; } - var node = new HttpBodySeparatorNode(_sourceText, _syntaxTree); - - ParseLeadingTrivia(node); - - return ParseTrailingTrivia(node); - } - - private HttpBodyNode? ParseBody() - { - if (!MoreTokens() || IsRequestSeparator()) + if (IsRequestSeparator()) { return null; } var node = new HttpBodyNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); - if (MoreTokens() && CurrentToken.Kind is not (HttpTokenKind.Whitespace or HttpTokenKind.NewLine) && !IsRequestSeparator()) @@ -686,29 +682,34 @@ CurrentToken.Kind is not (HttpTokenKind.Whitespace or HttpTokenKind.NewLine) && } } - return ParseTrailingTrivia(node); + ParseTrailingWhitespace(node); + + return node; } private HttpCommentNode? ParseComment() { - var commentNode = new HttpCommentNode(_sourceText, _syntaxTree); + if (MoreTokens() && IsComment()) + { + var commentNode = new HttpCommentNode(_sourceText, _syntaxTree); - var commentStartNode = ParseCommentStart(); + var commentStartNode = ParseCommentStart(); - if (commentStartNode is null) - { - return null; - } + if (commentStartNode is not null) + { + commentNode.Add(commentStartNode); + } - commentNode.Add(commentStartNode); + var commentBodyNode = ParseCommentBody(); + if (commentBodyNode is not null) + { + commentNode.Add(commentBodyNode); + } - var commentBodyNode = ParseCommentBody(); - if (commentBodyNode is not null) - { - commentNode.Add(commentBodyNode); + return commentNode; } - return commentNode; + return null; } private HttpCommentBodyNode? ParseCommentBody() @@ -719,13 +720,15 @@ CurrentToken.Kind is not (HttpTokenKind.Whitespace or HttpTokenKind.NewLine) && } var node = new HttpCommentBodyNode(_sourceText, _syntaxTree); - ParseLeadingTrivia(node); + ParseLeadingWhitespaceAndComments(node); while (MoreTokens() && CurrentToken.Kind is not HttpTokenKind.NewLine) { ConsumeCurrentTokenInto(node); } + ParseTrailingWhitespace(node); + return node; } @@ -735,7 +738,7 @@ CurrentToken.Kind is not (HttpTokenKind.Whitespace or HttpTokenKind.NewLine) && { var node = new HttpCommentStartNode(_sourceText, _syntaxTree); ConsumeCurrentTokenInto(node); - return ParseTrailingTrivia(node); + return ParseTrailingWhitespace(node); } if (MoreTokens() && CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "/" } && @@ -745,7 +748,7 @@ CurrentToken.Kind is not (HttpTokenKind.Whitespace or HttpTokenKind.NewLine) && ConsumeCurrentTokenInto(node); ConsumeCurrentTokenInto(node); - return ParseTrailingTrivia(node); + return ParseTrailingWhitespace(node); } return null; @@ -753,14 +756,14 @@ CurrentToken.Kind is not (HttpTokenKind.Whitespace or HttpTokenKind.NewLine) && private bool IsComment() { - if (MoreTokens()) + if (MoreTokens() && !IsRequestSeparator()) { - if (CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "#" }) + if (CurrentToken is { Text: "#" }) { return true; } - else if (CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "/" } && - CurrentTokenPlus(1) is { Kind: HttpTokenKind.Punctuation } and { Text: "/" }) + else if (CurrentToken is { Text: "/" } && + CurrentTokenPlus(1) is { Text: "/" }) { return true; } @@ -769,15 +772,14 @@ private bool IsComment() return false; } } + return false; } - private bool IsRequestSeparator() - { - return CurrentToken is { Kind: HttpTokenKind.Punctuation } and { Text: "#" } && - (CurrentTokenPlus(1) is { Kind: HttpTokenKind.Punctuation } and { Text: "#" } && - CurrentTokenPlus(2) is { Kind: HttpTokenKind.Punctuation } and { Text: "#" }); - } + private bool IsRequestSeparator() => + CurrentToken is { Text: "#" } && + CurrentTokenPlus(1) is { Text: "#" } && + CurrentTokenPlus(2) is { Text: "#" }; private static LinePosition GetLinePositionFromCursorOffset(SourceText code, int cursorOffset) { @@ -837,17 +839,21 @@ public IReadOnlyList Lex() { ' ' or '\t' => HttpTokenKind.Whitespace, '\n' or '\r' or '\v' => HttpTokenKind.NewLine, - _ => char.IsPunctuation(currentCharacter) - ? HttpTokenKind.Punctuation - : HttpTokenKind.Word, + _ => char.IsLetterOrDigit(currentCharacter) + ? HttpTokenKind.Word + : HttpTokenKind.Punctuation, }; if (previousTokenKind is { } previousTokenKindValue) { - if (!IsCurrentTokenANewLinePrecededByACarriageReturn(previousTokenKindValue, previousCharacter, - currentTokenKind, currentCharacter) && (previousTokenKind != currentTokenKind || currentTokenKind - is HttpTokenKind.NewLine || - currentTokenKind is HttpTokenKind.Punctuation)) + if (!IsCurrentTokenANewLinePrecededByACarriageReturn( + previousTokenKindValue, + previousCharacter, + currentTokenKind, + currentCharacter) && + (previousTokenKind != currentTokenKind || currentTokenKind + is HttpTokenKind.NewLine || + currentTokenKind is HttpTokenKind.Punctuation)) { FlushToken(previousTokenKindValue); } @@ -894,11 +900,11 @@ private bool IsCurrentTokenANewLinePrecededByACarriageReturn( HttpTokenKind previousTokenKindValue, char previousCharacter, HttpTokenKind currentTokenKind, - char currentCharacter) - { - return (currentTokenKind is HttpTokenKind.NewLine && previousTokenKindValue is HttpTokenKind.NewLine - && previousCharacter is '\r' && currentCharacter is '\n'); - } + char currentCharacter) => + currentTokenKind is HttpTokenKind.NewLine && + previousTokenKindValue is HttpTokenKind.NewLine + && + previousCharacter is '\r' && currentCharacter is '\n'; } private class TextWindow @@ -936,5 +942,4 @@ public void Advance() public override string ToString() => $"[{Start}..{End}]"; } - }