diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md index bbca4b74ad9d7..5f181658a03ea 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/README.md @@ -81,7 +81,7 @@ Ensure that the `PLAYWRIGHT_SERVICE_URL` that you obtained in previous step is a Run Playwright tests against browsers managed by the service using the configuration you created above. ```dotnetcli -dotnet test --logger "ms-playwright-service" +dotnet test --logger "microsoft-playwright-testing" ``` ## Key concepts diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample1_CustomisingServiceParameters.md b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample1_CustomisingServiceParameters.md index ef62a3929fe6a..a2eeefe04b61b 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample1_CustomisingServiceParameters.md +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample1_CustomisingServiceParameters.md @@ -16,7 +16,7 @@ This guide explains the different options available to you in the Azure.Develope - + @@ -79,7 +79,7 @@ public class PlaywrightServiceSetup : PlaywrightServiceNUnit 3. **`ExposeNetwork`**: - **Description**: This settings exposes network available on the connecting client to the browser being connected to. -4. **`ServiceAuth`** +4. **`ServiceAuthType`** - **Description**: This setting allows you to specify the default authentication mechanism to be used for sending requests to the service. - **Available Options**: - `ServiceAuthType.EntraId` for Microsoft Entra ID authentication. diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample2_SetDefaultAuthenticationMechanism.md b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample2_SetDefaultAuthenticationMechanism.md index 13c3ba68314c6..d8fd28f8d7ebe 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample2_SetDefaultAuthenticationMechanism.md +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.NUnit/samples/Sample2_SetDefaultAuthenticationMechanism.md @@ -24,7 +24,7 @@ public class PlaywrightServiceSetup : PlaywrightServiceNUnit {}; - + ``` diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs index 0880153e7dc93..b961eccd0a9bb 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/api/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.netstandard2.0.cs @@ -55,10 +55,10 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client { public partial class TestReportingClientOptions : Azure.Core.ClientOptions { - public TestReportingClientOptions(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client.TestReportingClientOptions.ServiceVersion version = Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client.TestReportingClientOptions.ServiceVersion.V2024_05_20_Preview) { } + public TestReportingClientOptions(Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client.TestReportingClientOptions.ServiceVersion version = Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client.TestReportingClientOptions.ServiceVersion.V2024_09_01_Preview) { } public enum ServiceVersion { - V2024_05_20_Preview = 1, + V2024_09_01_Preview = 1, } } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestResultsClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestResultsClient.cs deleted file mode 100644 index ed2f8c76c3fe1..0000000000000 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestResultsClient.cs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// - -#nullable disable - -using System; -using System.Threading.Tasks; -using Azure.Core; -using Azure.Core.Pipeline; - -namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client -{ - // Data plane generated client. - /// The TestResults service client. - internal partial class ReportingTestResultsClient - { - private readonly HttpPipeline _pipeline; - private readonly Uri _endpoint; - private readonly string _apiVersion; - - /// The ClientDiagnostics is used to provide tracing support for the client library. - internal ClientDiagnostics ClientDiagnostics { get; } - - /// The HTTP pipeline for sending and receiving REST requests and responses. - public virtual HttpPipeline Pipeline => _pipeline; - - /// Initializes a new instance of ReportingTestResultsClient for mocking. - protected ReportingTestResultsClient() - { - } - - /// Initializes a new instance of ReportingTestResultsClient. - /// server parameter. - /// is null. - public ReportingTestResultsClient(Uri endpoint) : this(endpoint, new TestReportingClientOptions()) - { - } - - /// Initializes a new instance of ReportingTestResultsClient. - /// server parameter. - /// The options for configuring the client. - /// is null. - public ReportingTestResultsClient(Uri endpoint, TestReportingClientOptions options) - { - Argument.AssertNotNull(endpoint, nameof(endpoint)); - options ??= new TestReportingClientOptions(); - - ClientDiagnostics = new ClientDiagnostics(options, true); - _pipeline = HttpPipelineBuilder.Build(options, Array.Empty(), Array.Empty(), new ResponseClassifier()); - _endpoint = endpoint; - _apiVersion = options.Version; - } - - /// - /// [Protocol Method] Uploads a batch of test results to the test run - /// - /// - /// - /// This protocol method allows explicit creation of the request and processing of the response for advanced scenarios. - /// - /// - /// - /// - /// The to use. - /// The content to send as the body of the request. - /// access token. - /// Correlation-id used for tracing and debugging. - /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// is null. - /// is an empty string, and was expected to be non-empty. - /// Service returned a non-success status code. - /// The response returned from the service. - public virtual async Task UploadBatchTestResultsAsync(string workspaceId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) - { - Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); - - using var scope = ClientDiagnostics.CreateScope("ReportingTestResultsClient.UploadBatchTestResults"); - scope.Start(); - try - { - using HttpMessage message = CreateUploadBatchTestResultsRequest(workspaceId, content, authorization, xCorrelationId, context); - return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); - } - catch (Exception e) - { - scope.Failed(e); - throw; - } - } - - /// - /// [Protocol Method] Uploads a batch of test results to the test run - /// - /// - /// - /// This protocol method allows explicit creation of the request and processing of the response for advanced scenarios. - /// - /// - /// - /// - /// The to use. - /// The content to send as the body of the request. - /// access token. - /// Correlation-id used for tracing and debugging. - /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// is null. - /// is an empty string, and was expected to be non-empty. - /// Service returned a non-success status code. - /// The response returned from the service. - public virtual Response UploadBatchTestResults(string workspaceId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) - { - Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); - - using var scope = ClientDiagnostics.CreateScope("ReportingTestResultsClient.UploadBatchTestResults"); - scope.Start(); - try - { - using HttpMessage message = CreateUploadBatchTestResultsRequest(workspaceId, content, authorization, xCorrelationId, context); - return _pipeline.ProcessMessage(message, context); - } - catch (Exception e) - { - scope.Failed(e); - throw; - } - } - - internal HttpMessage CreateUploadBatchTestResultsRequest(string workspaceId, RequestContent content, string authorization, string xCorrelationId, RequestContext context) - { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400500); - var request = message.Request; - request.Method = RequestMethod.Post; - var uri = new RawRequestUriBuilder(); - uri.Reset(_endpoint); - uri.AppendPath("/workspaces/", false); - uri.AppendPath(workspaceId, true); - uri.AppendPath("/test-results/upload-batch", false); - uri.AppendQuery("api-version", _apiVersion, true); - request.Uri = uri; - request.Headers.Add("Accept", "application/json"); - if (authorization != null) - { - request.Headers.Add("Authorization", authorization); - } - if (xCorrelationId != null) - { - request.Headers.Add("x-correlation-id", xCorrelationId); - } - request.Headers.Add("Content-Type", "application/json"); - request.Content = content; - return message; - } - - private static ResponseClassifier _responseClassifier200400500; - private static ResponseClassifier ResponseClassifier200400500 => _responseClassifier200400500 ??= new StatusCodeClassifier(stackalloc ushort[] { 200, 400, 500 }); - } -} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestRunsClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClient.cs similarity index 74% rename from sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestRunsClient.cs rename to sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClient.cs index 4bdb0cee6e0a9..12944db1f0f63 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/ReportingTestRunsClient.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClient.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // @@ -13,8 +13,8 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client { // Data plane generated client. - /// The TestRuns service client. - internal partial class ReportingTestRunsClient + /// The TestReporting service client. + internal partial class TestReportingClient { private readonly HttpPipeline _pipeline; private readonly Uri _endpoint; @@ -26,23 +26,23 @@ internal partial class ReportingTestRunsClient /// The HTTP pipeline for sending and receiving REST requests and responses. public virtual HttpPipeline Pipeline => _pipeline; - /// Initializes a new instance of ReportingTestRunsClient for mocking. - protected ReportingTestRunsClient() + /// Initializes a new instance of TestReportingClient for mocking. + protected TestReportingClient() { } - /// Initializes a new instance of ReportingTestRunsClient. + /// Initializes a new instance of TestReportingClient. /// server parameter. /// is null. - public ReportingTestRunsClient(Uri endpoint) : this(endpoint, new TestReportingClientOptions()) + public TestReportingClient(Uri endpoint) : this(endpoint, new TestReportingClientOptions()) { } - /// Initializes a new instance of ReportingTestRunsClient. + /// Initializes a new instance of TestReportingClient. /// server parameter. /// The options for configuring the client. /// is null. - public ReportingTestRunsClient(Uri endpoint, TestReportingClientOptions options) + public TestReportingClient(Uri endpoint, TestReportingClientOptions options) { Argument.AssertNotNull(endpoint, nameof(endpoint)); options ??= new TestReportingClientOptions(); @@ -54,7 +54,7 @@ public ReportingTestRunsClient(Uri endpoint, TestReportingClientOptions options) } /// - /// [Protocol Method] Patch Test Run Info + /// [Protocol Method] /// /// /// @@ -64,26 +64,23 @@ public ReportingTestRunsClient(Uri endpoint, TestReportingClientOptions options) /// /// /// The to use. - /// The to use. /// The content to send as the body of the request. - /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// or is null. - /// or is an empty string, and was expected to be non-empty. + /// is null. + /// is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual async Task PatchTestRunInfoAsync(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) + public virtual async Task UploadBatchTestResultsAsync(string workspaceId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); - Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.PatchTestRunInfoAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.UploadBatchTestResults"); scope.Start(); try { - using HttpMessage message = CreatePatchTestRunInfoAsyncRequest(workspaceId, testRunId, content, contentType, authorization, xCorrelationId, context); + using HttpMessage message = CreateUploadBatchTestResultsRequest(workspaceId, content, authorization, xCorrelationId, context); return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); } catch (Exception e) @@ -94,7 +91,7 @@ public virtual async Task PatchTestRunInfoAsync(string workspaceId, st } /// - /// [Protocol Method] Patch Test Run Info + /// [Protocol Method] /// /// /// @@ -104,26 +101,23 @@ public virtual async Task PatchTestRunInfoAsync(string workspaceId, st /// /// /// The to use. - /// The to use. /// The content to send as the body of the request. - /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// or is null. - /// or is an empty string, and was expected to be non-empty. + /// is null. + /// is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) + public virtual Response UploadBatchTestResults(string workspaceId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); - Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.PatchTestRunInfoAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.UploadBatchTestResults"); scope.Start(); try { - using HttpMessage message = CreatePatchTestRunInfoAsyncRequest(workspaceId, testRunId, content, contentType, authorization, xCorrelationId, context); + using HttpMessage message = CreateUploadBatchTestResultsRequest(workspaceId, content, authorization, xCorrelationId, context); return _pipeline.ProcessMessage(message, context); } catch (Exception e) @@ -134,7 +128,7 @@ public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, R } /// - /// [Protocol Method] Get Test Run Info + /// [Protocol Method] /// /// /// @@ -145,6 +139,7 @@ public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, R /// /// The to use. /// The to use. + /// The content to send as the body of the request. /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. @@ -152,16 +147,16 @@ public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, R /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual async Task GetTestRunAsync(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + public virtual async Task PatchTestRunInfoAsync(string workspaceId, string testRunId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.GetTestRunAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.PatchTestRunInfo"); scope.Start(); try { - using HttpMessage message = CreateGetTestRunAsyncRequest(workspaceId, testRunId, authorization, xCorrelationId, context); + using HttpMessage message = CreatePatchTestRunInfoRequest(workspaceId, testRunId, content, authorization, xCorrelationId, context); return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); } catch (Exception e) @@ -172,7 +167,7 @@ public virtual async Task GetTestRunAsync(string workspaceId, string t } /// - /// [Protocol Method] Get Test Run Info + /// [Protocol Method] /// /// /// @@ -183,6 +178,7 @@ public virtual async Task GetTestRunAsync(string workspaceId, string t /// /// The to use. /// The to use. + /// The content to send as the body of the request. /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. @@ -190,16 +186,16 @@ public virtual async Task GetTestRunAsync(string workspaceId, string t /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual Response GetTestRun(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + public virtual Response PatchTestRunInfo(string workspaceId, string testRunId, RequestContent content, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.GetTestRunAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.PatchTestRunInfo"); scope.Start(); try { - using HttpMessage message = CreateGetTestRunAsyncRequest(workspaceId, testRunId, authorization, xCorrelationId, context); + using HttpMessage message = CreatePatchTestRunInfoRequest(workspaceId, testRunId, content, authorization, xCorrelationId, context); return _pipeline.ProcessMessage(message, context); } catch (Exception e) @@ -210,7 +206,7 @@ public virtual Response GetTestRun(string workspaceId, string testRunId, string } /// - /// [Protocol Method] Patch Test Run Shard Info + /// [Protocol Method] /// /// /// @@ -221,27 +217,23 @@ public virtual Response GetTestRun(string workspaceId, string testRunId, string /// /// The to use. /// The to use. - /// The to use. - /// The content to send as the body of the request. - /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// , or is null. - /// , or is an empty string, and was expected to be non-empty. + /// or is null. + /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual async Task PatchTestRunShardInfoAsync(string workspaceId, string testRunId, string shardId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) + public virtual async Task GetTestRunResultsUriAsync(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - Argument.AssertNotNullOrEmpty(shardId, nameof(shardId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.PatchTestRunShardInfoAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.GetTestRunResultsUri"); scope.Start(); try { - using HttpMessage message = CreatePatchTestRunShardInfoAsyncRequest(workspaceId, testRunId, shardId, content, contentType, authorization, xCorrelationId, context); + using HttpMessage message = CreateGetTestRunResultsUriRequest(workspaceId, testRunId, authorization, xCorrelationId, context); return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); } catch (Exception e) @@ -252,7 +244,7 @@ public virtual async Task PatchTestRunShardInfoAsync(string workspaceI } /// - /// [Protocol Method] Patch Test Run Shard Info + /// [Protocol Method] /// /// /// @@ -263,27 +255,23 @@ public virtual async Task PatchTestRunShardInfoAsync(string workspaceI /// /// The to use. /// The to use. - /// The to use. - /// The content to send as the body of the request. - /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. - /// , or is null. - /// , or is an empty string, and was expected to be non-empty. + /// or is null. + /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual Response PatchTestRunShardInfo(string workspaceId, string testRunId, string shardId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) + public virtual Response GetTestRunResultsUri(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - Argument.AssertNotNullOrEmpty(shardId, nameof(shardId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.PatchTestRunShardInfoAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.GetTestRunResultsUri"); scope.Start(); try { - using HttpMessage message = CreatePatchTestRunShardInfoAsyncRequest(workspaceId, testRunId, shardId, content, contentType, authorization, xCorrelationId, context); + using HttpMessage message = CreateGetTestRunResultsUriRequest(workspaceId, testRunId, authorization, xCorrelationId, context); return _pipeline.ProcessMessage(message, context); } catch (Exception e) @@ -294,7 +282,7 @@ public virtual Response PatchTestRunShardInfo(string workspaceId, string testRun } /// - /// [Protocol Method] Get Test Run Results Uri + /// [Protocol Method] /// /// /// @@ -305,6 +293,8 @@ public virtual Response PatchTestRunShardInfo(string workspaceId, string testRun /// /// The to use. /// The to use. + /// The content to send as the body of the request. + /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. @@ -312,16 +302,16 @@ public virtual Response PatchTestRunShardInfo(string workspaceId, string testRun /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual async Task GetTestRunResultsUriAsync(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + public virtual async Task PostTestRunShardInfoAsync(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.GetTestRunResultsUriAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.PostTestRunShardInfo"); scope.Start(); try { - using HttpMessage message = CreateGetTestRunResultsUriAsyncRequest(workspaceId, testRunId, authorization, xCorrelationId, context); + using HttpMessage message = CreatePostTestRunShardInfoRequest(workspaceId, testRunId, content, contentType, authorization, xCorrelationId, context); return await _pipeline.ProcessMessageAsync(message, context).ConfigureAwait(false); } catch (Exception e) @@ -332,7 +322,7 @@ public virtual async Task GetTestRunResultsUriAsync(string workspaceId } /// - /// [Protocol Method] Get Test Run Results Uri + /// [Protocol Method] /// /// /// @@ -343,6 +333,8 @@ public virtual async Task GetTestRunResultsUriAsync(string workspaceId /// /// The to use. /// The to use. + /// The content to send as the body of the request. + /// Body Parameter content-type. Allowed values: "application/*+json" | "application/json" | "application/json-patch+json" | "text/json". /// access token. /// Correlation-id used for tracing and debugging. /// The request context, which can override default behaviors of the client pipeline on a per-call basis. @@ -350,16 +342,16 @@ public virtual async Task GetTestRunResultsUriAsync(string workspaceId /// or is an empty string, and was expected to be non-empty. /// Service returned a non-success status code. /// The response returned from the service. - public virtual Response GetTestRunResultsUri(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + public virtual Response PostTestRunShardInfo(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization = null, string xCorrelationId = null, RequestContext context = null) { Argument.AssertNotNullOrEmpty(workspaceId, nameof(workspaceId)); Argument.AssertNotNullOrEmpty(testRunId, nameof(testRunId)); - using var scope = ClientDiagnostics.CreateScope("ReportingTestRunsClient.GetTestRunResultsUriAsync"); + using var scope = ClientDiagnostics.CreateScope("TestReportingClient.PostTestRunShardInfo"); scope.Start(); try { - using HttpMessage message = CreateGetTestRunResultsUriAsyncRequest(workspaceId, testRunId, authorization, xCorrelationId, context); + using HttpMessage message = CreatePostTestRunShardInfoRequest(workspaceId, testRunId, content, contentType, authorization, xCorrelationId, context); return _pipeline.ProcessMessage(message, context); } catch (Exception e) @@ -369,20 +361,18 @@ public virtual Response GetTestRunResultsUri(string workspaceId, string testRunI } } - internal HttpMessage CreatePatchTestRunInfoAsyncRequest(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization, string xCorrelationId, RequestContext context) + internal HttpMessage CreateUploadBatchTestResultsRequest(string workspaceId, RequestContent content, string authorization, string xCorrelationId, RequestContext context) { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400401500); + var message = _pipeline.CreateMessage(context, ResponseClassifier200); var request = message.Request; - request.Method = RequestMethod.Patch; + request.Method = RequestMethod.Post; var uri = new RawRequestUriBuilder(); uri.Reset(_endpoint); uri.AppendPath("/workspaces/", false); uri.AppendPath(workspaceId, true); - uri.AppendPath("/test-runs/", false); - uri.AppendPath(testRunId, true); + uri.AppendPath("/test-results/upload-batch", false); uri.AppendQuery("api-version", _apiVersion, true); request.Uri = uri; - request.Headers.Add("Accept", "application/json"); if (authorization != null) { request.Headers.Add("Authorization", authorization); @@ -391,16 +381,16 @@ internal HttpMessage CreatePatchTestRunInfoAsyncRequest(string workspaceId, stri { request.Headers.Add("x-correlation-id", xCorrelationId); } - request.Headers.Add("Content-Type", contentType.ToString()); + request.Headers.Add("Content-Type", "application/json"); request.Content = content; return message; } - internal HttpMessage CreateGetTestRunAsyncRequest(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + internal HttpMessage CreatePatchTestRunInfoRequest(string workspaceId, string testRunId, RequestContent content, string authorization, string xCorrelationId, RequestContext context) { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400401500); + var message = _pipeline.CreateMessage(context, ResponseClassifier200); var request = message.Request; - request.Method = RequestMethod.Get; + request.Method = RequestMethod.Patch; var uri = new RawRequestUriBuilder(); uri.Reset(_endpoint); uri.AppendPath("/workspaces/", false); @@ -418,22 +408,23 @@ internal HttpMessage CreateGetTestRunAsyncRequest(string workspaceId, string tes { request.Headers.Add("x-correlation-id", xCorrelationId); } + request.Headers.Add("Content-Type", "application/merge-patch+json"); + request.Content = content; return message; } - internal HttpMessage CreatePatchTestRunShardInfoAsyncRequest(string workspaceId, string testRunId, string shardId, RequestContent content, ContentType contentType, string authorization, string xCorrelationId, RequestContext context) + internal HttpMessage CreateGetTestRunResultsUriRequest(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400401500); + var message = _pipeline.CreateMessage(context, ResponseClassifier200); var request = message.Request; - request.Method = RequestMethod.Patch; + request.Method = RequestMethod.Post; var uri = new RawRequestUriBuilder(); uri.Reset(_endpoint); uri.AppendPath("/workspaces/", false); uri.AppendPath(workspaceId, true); uri.AppendPath("/test-runs/", false); uri.AppendPath(testRunId, true); - uri.AppendPath("/shards/", false); - uri.AppendPath(shardId, true); + uri.AppendPath(":createartifactsuploadbaseuri", false); uri.AppendQuery("api-version", _apiVersion, true); request.Uri = uri; request.Headers.Add("Accept", "application/json"); @@ -445,23 +436,21 @@ internal HttpMessage CreatePatchTestRunShardInfoAsyncRequest(string workspaceId, { request.Headers.Add("x-correlation-id", xCorrelationId); } - request.Headers.Add("Content-Type", contentType.ToString()); - request.Content = content; return message; } - internal HttpMessage CreateGetTestRunResultsUriAsyncRequest(string workspaceId, string testRunId, string authorization, string xCorrelationId, RequestContext context) + internal HttpMessage CreatePostTestRunShardInfoRequest(string workspaceId, string testRunId, RequestContent content, ContentType contentType, string authorization, string xCorrelationId, RequestContext context) { - var message = _pipeline.CreateMessage(context, ResponseClassifier200400401500); + var message = _pipeline.CreateMessage(context, ResponseClassifier200); var request = message.Request; - request.Method = RequestMethod.Get; + request.Method = RequestMethod.Post; var uri = new RawRequestUriBuilder(); uri.Reset(_endpoint); uri.AppendPath("/workspaces/", false); uri.AppendPath(workspaceId, true); uri.AppendPath("/test-runs/", false); uri.AppendPath(testRunId, true); - uri.AppendPath("/resulturi", false); + uri.AppendPath(":updateshardexecutionstatus", false); uri.AppendQuery("api-version", _apiVersion, true); request.Uri = uri; request.Headers.Add("Accept", "application/json"); @@ -473,10 +462,12 @@ internal HttpMessage CreateGetTestRunResultsUriAsyncRequest(string workspaceId, { request.Headers.Add("x-correlation-id", xCorrelationId); } + request.Headers.Add("Content-Type", contentType.ToString()); + request.Content = content; return message; } - private static ResponseClassifier _responseClassifier200400401500; - private static ResponseClassifier ResponseClassifier200400401500 => _responseClassifier200400401500 ??= new StatusCodeClassifier(stackalloc ushort[] { 200, 400, 401, 500 }); + private static ResponseClassifier _responseClassifier200; + private static ResponseClassifier ResponseClassifier200 => _responseClassifier200 ??= new StatusCodeClassifier(stackalloc ushort[] { 200 }); } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClientOptions.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClientOptions.cs index cf4492767fe75..8416fd53cc7b6 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClientOptions.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Client/TestReportingClientOptions.cs @@ -10,16 +10,16 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client { - /// Client options for TestReporting library clients. + /// Client options for TestReportingClient. public partial class TestReportingClientOptions : ClientOptions { - private const ServiceVersion LatestVersion = ServiceVersion.V2024_05_20_Preview; + private const ServiceVersion LatestVersion = ServiceVersion.V2024_09_01_Preview; /// The version of the service to use. public enum ServiceVersion { - /// Service version "2024-05-20-preview". - V2024_05_20_Preview = 1, + /// Service version "2024-09-01-preview". + V2024_09_01_Preview = 1, } internal string Version { get; } @@ -29,7 +29,7 @@ public TestReportingClientOptions(ServiceVersion version = LatestVersion) { Version = version switch { - ServiceVersion.V2024_05_20_Preview => "2024-05-20-preview", + ServiceVersion.V2024_09_01_Preview => "2024-09-01-preview", _ => throw new NotSupportedException() }; } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs index 73789ac678eb0..6b2559845d2f5 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Constants.cs @@ -1,6 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using System.Collections.Generic; +using System.Text.RegularExpressions; + namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger; /// @@ -183,6 +187,7 @@ internal class Constants // Default constants internal static readonly string s_default_os = ServiceOs.Linux; internal static readonly string s_default_expose_network = ""; + internal static readonly string s_pLAYWRIGHT_SERVICE_DEBUG = "Logging__LogLevel__MicrosoftPlaywrightTesting"; // Entra id access token constants internal static readonly int s_entra_access_token_lifetime_left_threshold_in_minutes_for_rotation = 15; @@ -206,3 +211,196 @@ internal class Constants internal static readonly string s_playwright_service_reporting_url_environment_variable = "PLAYWRIGHT_SERVICE_REPORTING_URL"; internal static readonly string s_playwright_service_workspace_id_environment_variable = "PLAYWRIGHT_SERVICE_WORKSPACE_ID"; } + +internal class OSConstants +{ + internal static readonly string s_lINUX = "LINUX"; + internal static readonly string s_wINDOWS = "WINDOWS"; + internal static readonly string s_mACOS = "MACOS"; +} + +internal class ReporterConstants +{ + internal static readonly string s_executionIdPropertyIdentifier = "ExecutionId"; + internal static readonly string s_parentExecutionIdPropertyIdentifier = "ParentExecId"; + internal static readonly string s_testTypePropertyIdentifier = "TestType"; + internal static readonly string s_sASUriSeparator = "?"; + internal static readonly string s_portalBaseUrl = "https://playwright.microsoft.com/workspaces/"; + internal static readonly string s_reportingRoute = "/runs/"; + internal static readonly string s_reportingAPIVersion_2024_04_30_preview = "2024-04-30-preview"; + internal static readonly string s_reportingAPIVersion_2024_05_20_preview = "2024-05-20-preview"; + internal static readonly string s_pLAYWRIGHT_SERVICE_REPORTING_URL = "PLAYWRIGHT_SERVICE_REPORTING_URL"; + internal static readonly string s_pLAYWRIGHT_SERVICE_WORKSPACE_ID = "PLAYWRIGHT_SERVICE_WORKSPACE_ID"; + internal static readonly string s_aPPLICATION_JSON = "application/json"; + internal static readonly string s_cONFLICT_409_ERROR_MESSAGE = "Test run with id {runId} already exists. Provide a unique run id."; + internal static readonly string s_cONFLICT_409_ERROR_MESSAGE_KEY = "DuplicateRunId"; + + internal static readonly string s_fORBIDDEN_403_ERROR_MESSAGE = "Reporting is not enabled for your workspace {workspaceId}. Enable the Reporting feature under Feature management settings using the Playwright portal: https://playwright.microsoft.com/workspaces/{workspaceId}/settings/general"; + internal static readonly string s_fORBIDDEN_403_ERROR_MESSAGE_KEY = "ReportingNotEnabled"; + internal static readonly string s_uNKNOWN_ERROR_MESSAGE = "Unknown error occured."; +} + +internal class CIConstants +{ + internal static readonly string s_gITHUB_ACTIONS = "GITHUB"; + internal static readonly string s_aZURE_DEVOPS = "ADO"; + internal static readonly string s_dEFAULT = "DEFAULT"; +} + +internal class TestCaseResultStatus +{ + internal static readonly string s_pASSED = "PASSED"; + internal static readonly string s_fAILED = "FAILED"; + internal static readonly string s_sKIPPED = "SKIPPED"; + internal static readonly string s_iNCONCLUSIVE = "INCONCLUSIVE"; +} + +internal class TestResultError +{ + internal string? Key { get; set; } = string.Empty; + internal string? Message { get; set; } = string.Empty; + internal Regex Pattern { get; set; } = new Regex(string.Empty); + internal TestErrorType Type { get; set; } +} + +internal enum TestErrorType +{ + Scalable +} + +internal class ServiceClientConstants +{ + internal static readonly int s_mAX_RETRIES = 3; + internal static readonly int s_mAX_RETRY_DELAY_IN_SECONDS = 2000; +} + +internal static class TestResultErrorConstants +{ + public static List ErrorConstants = new() + { + new TestResultError + { + Key = "401", + Message = "The authentication token provided is invalid. Please check the token and try again.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*401 Unauthorized)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "NoPermissionOnWorkspace_Scalable", + Message = @"You do not have the required permissions to run tests. This could be because: + + a. You do not have the required roles on the workspace. Only Owner and Contributor roles can run tests. Contact the service administrator. + b. The workspace you are trying to run the tests on is in a different Azure tenant than what you are signed into. Check the tenant id from Azure portal and login using the command 'az login --tenant '.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*CheckAccess API call with non successful response)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "InvalidWorkspace_Scalable", + Message = "The specified workspace does not exist. Please verify your workspace settings.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*InvalidAccountOrSubscriptionState)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "InvalidAccessToken", + Message = "The provided access token does not match the specified workspace URL. Please verify that both values are correct.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*InvalidAccessToken)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "AccessTokenOrUserOrWorkspaceNotFound_Scalable", + Message = "The data for the user, workspace or access token was not found. Please check the request or create new token and try again.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*404 Not Found)(?=[\s\S]*NotFound)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "AccessKeyBasedAuthNotSupported_Scalable", + Message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*AccessKeyBasedAuthNotSupported)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "ServiceUnavailable_Scalable", + Message = "The service is currently unavailable. Please check the service status and try again.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*503 Service Unavailable)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "GatewayTimeout_Scalable", + Message = "The request to the service timed out. Please try again later.", + Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*504 Gateway Timeout)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "QuotaLimitError_Scalable", + Message = "It is possible that the maximum number of concurrent sessions allowed for your workspace has been exceeded.", + Pattern = new Regex(@"(Timeout .* exceeded)(?=[\s\S]*ws connecting)", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + }, + new TestResultError + { + Key = "BrowserConnectionError_Scalable", + Message = "The service is currently unavailable. Please try again after some time.", + Pattern = new Regex(@"Target page, context or browser has been closed", RegexOptions.IgnoreCase), + Type = TestErrorType.Scalable + } + }; +} + +internal static class ApiErrorConstants +{ + private static Dictionary PatchTestRun { get; set; } = new Dictionary() { + { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, + { 401, "The authentication token provided is invalid. Please check the token and try again." }, + { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." }, + { 429, "You have exceeded the rate limit for the API. Please wait and try again later." }, + { 504, "The request to the service timed out. Please try again later." }, + { 503, "The service is currently unavailable. Please check the service status and try again." } + }; + + private static Dictionary UploadBatchTestResults { get; set; } = new Dictionary() + { + { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, + { 401, "The authentication token provided is invalid. Please check the token and try again." }, + { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." }, + { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." }, + { 429, "You have exceeded the rate limit for the API. Please wait and try again later." }, + { 504, "The request to the service timed out. Please try again later." }, + { 503, "The service is currently unavailable. Please check the service status and try again." } + }; + private static Dictionary PostTestRunShardInfo { get; set; } = new Dictionary() + { + { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, + { 401, "The authentication token provided is invalid. Please check the token and try again." }, + { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." }, + { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." }, + { 429, "You have exceeded the rate limit for the API. Please wait and try again later." }, + { 504, "The request to the service timed out. Please try again later." }, + { 503, "The service is currently unavailable. Please check the service status and try again." } + }; + private static Dictionary GetTestRunResultsUri { get; set; } = new Dictionary() + { + { 400, "The request made to the server is invalid. Please check the request parameters and try again." }, + { 401, "The authentication token provided is invalid. Please check the token and try again." }, + { 403, "You do not have the required permissions to run tests. Please contact your workspace administrator." }, + { 500, "An unexpected error occurred on our server. Our team is working to resolve the issue. Please try again later, or contact support if the problem continues." }, + { 429, "You have exceeded the rate limit for the API. Please wait and try again later." }, + { 504, "The request to the service timed out. Please try again later." }, + { 503, "The service is currently unavailable. Please check the service status and try again." } + }; + + internal static readonly Dictionary> s_errorOperationPair = new() + { + { "PatchTestRun", PatchTestRun }, + { "UploadBatchTestResults", UploadBatchTestResults }, + { "PostTestRunShardInfo", PostTestRunShardInfo }, + { "GetTestRunResultsUri", GetTestRunResultsUri } + }; +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs new file mode 100644 index 0000000000000..2410754187a5e --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/CloudRunErrorParser.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation +{ + internal class CloudRunErrorParser : ICloudRunErrorParser + { + internal List InformationalMessages { get; private set; } = new(); + private List ProcessedErrorMessageKeys { get; set; } = new(); + private readonly ILogger _logger; + private readonly IConsoleWriter _consoleWriter; + public CloudRunErrorParser(ILogger? logger = null, IConsoleWriter? consoleWriter = null) + { + _logger = logger ?? new Logger(); + _consoleWriter = consoleWriter ?? new ConsoleWriter(); + } + + public bool TryPushMessageAndKey(string? message, string? key) + { + if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(message)) + { + return false; + } + if (ProcessedErrorMessageKeys.Contains(key!)) + { + return false; + } + _logger.Info($"Adding message with key: {key}"); + + ProcessedErrorMessageKeys.Add(key!); + InformationalMessages.Add(message!); + return true; + } + + public void PushMessage(string message) + { + InformationalMessages.Add(message); + } + + public void DisplayMessages() + { + if (InformationalMessages.Count > 0) + _consoleWriter.WriteLine(); + int index = 1; + foreach (string message in InformationalMessages) + { + _consoleWriter.WriteLine($"{index++}) {message}"); + } + } + + public void PrintErrorToConsole(string message) + { + _consoleWriter.WriteError(message); + } + + public void HandleScalableRunErrorMessage(string? message) + { + if (string.IsNullOrEmpty(message)) + { + return; + } + foreach (TestResultError testResultErrorObj in TestResultErrorConstants.ErrorConstants) + { + if (testResultErrorObj.Pattern.IsMatch(message)) + { + TryPushMessageAndKey(testResultErrorObj.Message, testResultErrorObj.Key); + } + } + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs new file mode 100644 index 0000000000000..c0092b80de03e --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ConsoleWriter.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation +{ + internal class ConsoleWriter : IConsoleWriter + { + public void WriteLine(string? message = null) + { + Console.WriteLine(message); + } + + public void WriteError(string? message = null) + { + Console.Error.WriteLine(message); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs new file mode 100644 index 0000000000000..3251ea8584d71 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/Logger.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; + +internal enum LogLevel +{ + Debug, + Info, + Warning, + Error +} + +internal class Logger : ILogger +{ + internal static string SdkLogLevel => Environment.GetEnvironmentVariable(Constants.s_pLAYWRIGHT_SERVICE_DEBUG); + +#pragma warning disable CA1822 // Mark members as static + private void Log(LogLevel level, string message) +#pragma warning restore CA1822 // Mark members as static + { + if (Enum.TryParse(SdkLogLevel, out LogLevel configuredLevel) && level >= configuredLevel) + { + System.IO.TextWriter writer = level == LogLevel.Error || level == LogLevel.Warning ? Console.Error : Console.Out; + writer.WriteLine($"{DateTime.Now} [{level}]: {message}"); + } + } + + public void Debug(string message) + { + Log(LogLevel.Debug, message); + } + + public void Error(string message) + { + Log(LogLevel.Error, message); + } + + public void Info(string message) + { + Log(LogLevel.Info, message); + } + + public void Warning(string message) + { + Log(LogLevel.Warning, message); + } +}; diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs new file mode 100644 index 0000000000000..1d4f423da5f4a --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Implementation/ServiceClient.cs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +using System; +using Azure.Core.Serialization; +using Azure.Core; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using System.Text.Json; +using Azure.Core.Diagnostics; +using System.Diagnostics.Tracing; +using System.Net; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation +{ + internal class ServiceClient : IServiceClient + { + private readonly TestReportingClient _testReportingClient; + private readonly CloudRunMetadata _cloudRunMetadata; + private readonly ICloudRunErrorParser _cloudRunErrorParser; + private readonly ILogger _logger; + private static string AccessToken { get => $"Bearer {Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceAccessToken)}"; set { } } + private static string CorrelationId { get => Guid.NewGuid().ToString(); set { } } + private static readonly JsonSerializerOptions _jsonSerializerOptions = new() { DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }; + + public ServiceClient(CloudRunMetadata cloudRunMetadata, ICloudRunErrorParser cloudRunErrorParser, TestReportingClient? testReportingClient = null, ILogger? logger = null) + { + _cloudRunMetadata = cloudRunMetadata; + _cloudRunErrorParser = cloudRunErrorParser; + _logger = logger ?? new Logger(); + AzureEventSourceListener listener = new(delegate (EventWrittenEventArgs eventData, string text) + { + _logger.Info($"[{eventData.Level}] {eventData.EventSource.Name}: {text}"); + }, EventLevel.Informational); + var clientOptions = new TestReportingClientOptions(); + clientOptions.Diagnostics.IsLoggingEnabled = true; + clientOptions.Diagnostics.IsTelemetryEnabled = true; + clientOptions.Retry.MaxRetries = ServiceClientConstants.s_mAX_RETRIES; + clientOptions.Retry.MaxDelay = TimeSpan.FromSeconds(ServiceClientConstants.s_mAX_RETRY_DELAY_IN_SECONDS); + _testReportingClient = testReportingClient ?? new TestReportingClient(_cloudRunMetadata.BaseUri, clientOptions); + } + + public TestRunDto? PatchTestRunInfo(TestRunDto run) + { + int statusCode; + try + { + Response? apiResponse = _testReportingClient.PatchTestRunInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, RequestContent.Create(JsonSerializer.Serialize(run)), AccessToken, CorrelationId); + if (apiResponse.Status == (int)HttpStatusCode.OK) + { + return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + } + statusCode = apiResponse.Status; + } + catch (RequestFailedException ex) + { + if (ex.Status == (int)HttpStatusCode.Conflict) + { + var errorMessage = ReporterConstants.s_cONFLICT_409_ERROR_MESSAGE.Replace("{runId}", _cloudRunMetadata.RunId!); + _cloudRunErrorParser.PrintErrorToConsole(errorMessage); + _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, ReporterConstants.s_cONFLICT_409_ERROR_MESSAGE_KEY); + throw new Exception(errorMessage); + } + else if (ex.Status == (int)HttpStatusCode.Forbidden) + { + var errorMessage = ReporterConstants.s_fORBIDDEN_403_ERROR_MESSAGE.Replace("{workspaceId}", _cloudRunMetadata.WorkspaceId!); + _cloudRunErrorParser.PrintErrorToConsole(errorMessage); + _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, ReporterConstants.s_fORBIDDEN_403_ERROR_MESSAGE_KEY); + throw new Exception(errorMessage); + } + statusCode = ex.Status; + } + HandleAPIFailure(statusCode, "PatchTestRun"); + return null; + } + + public TestRunShardDto? PostTestRunShardInfo(TestRunShardDto runShard) + { + int statusCode; + try + { + Response apiResponse = _testReportingClient.PostTestRunShardInfo(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, RequestContent.Create(runShard), ReporterConstants.s_aPPLICATION_JSON, AccessToken, CorrelationId); + if (apiResponse.Status == (int)HttpStatusCode.OK) + { + return apiResponse.Content!.ToObject(new JsonObjectSerializer()); + } + statusCode = apiResponse.Status; + } + catch (RequestFailedException ex) + { + statusCode = ex.Status; + } + HandleAPIFailure(statusCode, "PostTestRunShardInfo"); + return null; + } + + public void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsRequest) + { + int statusCode; + try + { + Response apiResponse = _testReportingClient.UploadBatchTestResults(_cloudRunMetadata.WorkspaceId!, RequestContent.Create(JsonSerializer.Serialize(uploadTestResultsRequest)), AccessToken, CorrelationId, null); + if (apiResponse.Status == (int)HttpStatusCode.OK) + { + return; + } + statusCode = apiResponse.Status; + } + catch (RequestFailedException ex) + { + statusCode = ex.Status; + } + HandleAPIFailure(statusCode, "UploadBatchTestResults"); + } + + public TestResultsUri? GetTestRunResultsUri() + { + int statusCode; + try + { + Response response = _testReportingClient.GetTestRunResultsUri(_cloudRunMetadata.WorkspaceId!, _cloudRunMetadata.RunId!, AccessToken, CorrelationId, null); + if (response.Status == (int)HttpStatusCode.OK) + { + return response.Content!.ToObject(new JsonObjectSerializer()); + } + statusCode = response.Status; + } + catch (RequestFailedException ex) + { + statusCode = ex.Status; + } + HandleAPIFailure(statusCode, "GetTestRunResultsUri"); + return null; + } + + internal void HandleAPIFailure(int? statusCode, string operationName) + { + try + { + if (statusCode == null) + return; + ApiErrorConstants.s_errorOperationPair.TryGetValue(operationName, out System.Collections.Generic.Dictionary? errorObject); + if (errorObject == null) + return; + errorObject.TryGetValue((int)statusCode, out string? errorMessage); + errorMessage ??= ReporterConstants.s_uNKNOWN_ERROR_MESSAGE; + _cloudRunErrorParser.TryPushMessageAndKey(errorMessage, statusCode.ToString()); + } + catch (Exception ex) + { + _logger.Error(ex.Message); + } + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs new file mode 100644 index 0000000000000..7dbbecf68c149 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ICloudRunErrorParser.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface ICloudRunErrorParser + { + void HandleScalableRunErrorMessage(string? message); + bool TryPushMessageAndKey(string? message, string? key); + void PushMessage(string message); + void DisplayMessages(); + void PrintErrorToConsole(string message); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs new file mode 100644 index 0000000000000..41939a5d64296 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IConsoleWriter.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface IConsoleWriter + { + void WriteLine(string? message = null); + void WriteError(string? message = null); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs new file mode 100644 index 0000000000000..280a6fe08f846 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IDataProcessor.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface IDataProcessor + { + TestRunDto GetTestRun(); + TestRunShardDto GetTestRunShard(); + TestResults GetTestCaseResultData(TestResult? testResultSource); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ILogger.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ILogger.cs new file mode 100644 index 0000000000000..c1912bcfaa506 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ILogger.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface ILogger + { + void Info(string message); + void Debug(string message); + void Warning(string message); + void Error(string message); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs new file mode 100644 index 0000000000000..07edddb642303 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/IServiceClient.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface IServiceClient + { + TestRunDto? PatchTestRunInfo(TestRunDto run); + TestRunShardDto? PostTestRunShardInfo(TestRunShardDto runShard); + void UploadBatchTestResults(UploadTestResultsRequest uploadTestResultsRequest); + TestResultsUri? GetTestRunResultsUri(); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ITestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ITestProcessor.cs new file mode 100644 index 0000000000000..205332294b2e2 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Interface/ITestProcessor.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface +{ + internal interface ITestProcessor + { + void TestCaseResultHandler(object? sender, TestResultEventArgs e); + void TestRunStartHandler(object? sender, TestRunStartEventArgs e); + void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e); + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs new file mode 100644 index 0000000000000..124eed6ab0ce7 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/CloudRunMetadata.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model +{ + internal class CloudRunMetadata + { + internal string? WorkspaceId { get; set; } + internal string? RunId { get; set; } + internal Uri? BaseUri { get; set; } + internal string? PortalUrl + { + get { return ReporterConstants.s_portalBaseUrl + Uri.EscapeDataString(WorkspaceId) + ReporterConstants.s_reportingRoute + Uri.EscapeDataString(RunId); } + } + internal bool EnableResultPublish { get; set; } = true; + internal bool EnableGithubSummary { get; set; } = true; + internal DateTime TestRunStartTime { get; set; } + internal TokenDetails? AccessTokenDetails { get; set; } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/TestReporting.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/TestReporting.cs index 08cff08b222fa..4692883feaa0c 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/TestReporting.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Model/TestReporting.cs @@ -16,15 +16,15 @@ internal enum AccessLevel internal partial class CIConfig { - [JsonPropertyName("ciProviderName")] public string CiProviderName { get; set; } = ""; + [JsonPropertyName("ciProviderName")] public string? CiProviderName { get; set; } = ""; - [JsonPropertyName("branch")] public string Branch { get; set; } = ""; + [JsonPropertyName("branch")] public string? Branch { get; set; } - [JsonPropertyName("author")] public string Author { get; set; } = ""; + [JsonPropertyName("author")] public string? Author { get; set; } - [JsonPropertyName("commitId")] public string CommitId { get; set; } = ""; + [JsonPropertyName("commitId")] public string? CommitId { get; set; } - [JsonPropertyName("revisionUrl")] public string RevisionUrl { get; set; } = ""; + [JsonPropertyName("revisionUrl")] public string? RevisionUrl { get; set; } } internal partial class ClientConfig @@ -81,7 +81,7 @@ internal partial class Shard public int Total { get; set; } [JsonPropertyName("current")] - public int Current { get; set; } + public int? Current { get; set; } } internal partial class TestFramework @@ -103,6 +103,8 @@ internal partial class TestResults [JsonPropertyName("runId")] public string RunId { get; set; } = ""; + [JsonPropertyName("shardId")] public string ShardId { get; set; } = ""; + [JsonPropertyName("accountId")] public string AccountId { get; set; } = ""; [JsonPropertyName("suiteId")] public string SuiteId { get; set; } = ""; @@ -167,7 +169,7 @@ internal partial class TestResultsUri public AccessLevel? AccessLevel { get; set; } } -internal partial class TestRunDtoV2 +internal partial class TestRunDto { [JsonPropertyName("testRunId")] public string TestRunId { get; set; } = ""; @@ -199,9 +201,9 @@ internal partial class TestRunDtoV2 [JsonPropertyName("testResultsUri")] public TestResultsUri? TestResultsUri { get; set; } - [JsonPropertyName("cloudRunEnabled")] public string CloudRunEnabled { get; set; } = ""; + [JsonPropertyName("cloudRunEnabled")] public bool? CloudRunEnabled { get; set; } - [JsonPropertyName("cloudReportingEnabled")] public string CloudReportingEnabled { get; set; } = ""; + [JsonPropertyName("cloudReportingEnabled")] public bool? CloudReportingEnabled { get; set; } } internal partial class TestRunResultsSummary @@ -227,22 +229,18 @@ internal partial class TestRunResultsSummary internal partial class TestRunShardDto { - [JsonPropertyName("uploadCompleted")] public string UploadCompleted { get; set; } = ""; + [JsonPropertyName("shardId")] public string ShardId { get; set; } = ""; + [JsonPropertyName("uploadCompleted")] public bool UploadCompleted { get; set; } = false; [JsonPropertyName("summary")] public TestRunShardSummary? Summary { get; set; } - [JsonPropertyName("testRunConfig")] - public ClientConfig? TestRunConfig { get; set; } - - [JsonPropertyName("resultsSummary")] - public TestRunResultsSummary? ResultsSummary { get; set; } + [JsonPropertyName("workers")] public int? Workers { get; set; } } internal partial class TestRunShardSummary { [JsonPropertyName("status")] public string Status { get; set; } = ""; - [JsonPropertyName("startTime")] public string StartTime { get; set; } = ""; [JsonPropertyName("endTime")] public string EndTime { get; set; } = ""; @@ -260,14 +258,8 @@ internal partial class TestRunSummary { [JsonPropertyName("status")] public string Status { get; set; } = ""; - [JsonPropertyName("startTime")] public string StartTime { get; set; } = ""; - - [JsonPropertyName("endTime")] public string EndTime { get; set; } = ""; - [JsonPropertyName("billableTime")] public long BillableTime { get; set; } - [JsonPropertyName("totalTime")] public long TotalTime { get; set; } - [JsonPropertyName("numBrowserSessions")] public long NumBrowserSessions { get; set; } [JsonPropertyName("jobs")] @@ -278,12 +270,6 @@ internal partial class TestRunSummary [JsonPropertyName("tags")] public ICollection Tags { get; set; } = new List(); - - [JsonPropertyName("errorMessages")] - public ICollection ErrorMessages { get; set; } = new List(); - - [JsonPropertyName("uploadMetadata")] - public UploadMetadata? UploadMetadata { get; set; } } internal partial class UploadMetadata @@ -307,7 +293,7 @@ internal partial class WebTestConfig [JsonPropertyName("projectName")] public string ProjectName { get; set; } = ""; - [JsonPropertyName("browserName")] public string BrowserName { get; set; } = ""; + [JsonPropertyName("browserType")] public string BrowserName { get; set; } = ""; [JsonPropertyName("os")] public string Os { get; set; } = ""; } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs index bccfa06325172..bfb6d6fdccb3b 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/PlaywrightReporter.cs @@ -1,86 +1,34 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using Azure.Storage.Blobs; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; -using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client; -using Microsoft.IdentityModel.JsonWebTokens; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.IO; -using System.Text.Json; -using PlaywrightConstants = Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility.Constants; -using Azure.Core; -using Azure.Core.Serialization; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor; namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger; -[FriendlyName("ms-playwright-service")] -[ExtensionUri("logger://Microsoft/Playwright/ServiceLogger/v1")] +[FriendlyName("microsoft-playwright-testing")] +[ExtensionUri("logger://MicrosoftPlaywrightTesting/Logger/v1")] internal class PlaywrightReporter : ITestLoggerWithParameters { private Dictionary? _parametersDictionary; + private PlaywrightService? _playwrightService; + private readonly ILogger _logger; + private TestProcessor? _testProcessor; - private bool IsInitialized { get; set; } - - private HttpClient? _httpClient; - - private ReportingTestResultsClient? _reportingTestResultsClient; - private ReportingTestRunsClient? _reportingTestRunsClient; - - private static readonly JsonWebTokenHandler s_tokenHandler = new(); - - private readonly LogLevel _logLevel = LogLevel.Debug; - - internal static string EnableConsoleLog { get => Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_DEBUG) ?? "false"; set { } } - - internal string? PortalUrl { get; set; } - - internal static string? BaseUrl { get => Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_REPORTING_URL); private set { } } - - internal static string AccessToken { get => Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_ACCESS_TOKEN) ?? ""; set { } } - - internal string? WorkspaceId { get; set; } - - internal TokenDetails? TokenDetails { get; set; } - - internal CIInfo? CIInfo { get; set; } - - internal string? RunId { get; set; } - - internal DateTime TestRunStartTime { get; private set; } - - internal int TotalTestCount { get; private set; } - - internal int PassedTestCount { get; private set; } - - internal int FailedTestCount { get; private set; } - - internal int SkippedTestCount { get; private set; } - - internal TestRunDtoV2? TestRun { get; set; } - - internal TestRunShardDto? TestRunShard { get; set; } - - internal bool EnableGithubSummary { get; set; } = true; - internal bool EnableResultPublish { get; set; } = true; - - internal List TestResults = new(); - - internal ConcurrentDictionary RawTestResultsMap = new(); - - internal PlaywrightService? playwrightService; - private List informationalMessages = new(); - private List processedErrorMessageKeys = new(); + public PlaywrightReporter() : this(null) { } // no-op + public PlaywrightReporter(ILogger? logger) + { + _logger = logger ?? new Logger(); + } public void Initialize(TestLoggerEvents events, Dictionary parameters) { @@ -88,570 +36,48 @@ public void Initialize(TestLoggerEvents events, Dictionary para _parametersDictionary = parameters; Initialize(events, _parametersDictionary[DefaultLoggerParameterNames.TestRunDirectory]!); } - public void Initialize(TestLoggerEvents events, string testResultsDirPath) { ValidateArg.NotNull(events, nameof(events)); ValidateArg.NotNullOrEmpty(testResultsDirPath, nameof(testResultsDirPath)); // Register for the events. - events.TestRunMessage += TestMessageHandler; - events.TestResult += TestResultHandler; - events.TestRunComplete += TestRunCompleteHandler; - events.TestRunStart += TestRunStartHandler; + events.TestResult += TestResultHandler; // each test run end + events.TestRunComplete += TestRunCompleteHandler; // test suite end + events.TestRunStart += TestRunStartHandler; // test suite start } #region Event Handlers - internal void TestRunStartHandler(object? sender, TestRunStartEventArgs e) { InitializePlaywrightReporter(e.TestRunCriteria.TestRunSettings!); - LogMessage("Test Run start Handler"); - if (!EnableResultPublish) - { - return; - } - if (!IsInitialized || _reportingTestResultsClient == null || _reportingTestRunsClient == null) - { - LogErrorMessage("Test Run setup issue exiting handler"); - return; - } - - var startTime = TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); - LogMessage("Test Run start time: " + startTime); - var corelationId = Guid.NewGuid().ToString(); - var gitBasedRunName = ReporterUtils.GetRunName(CiInfoProvider.GetCIInfo()); - var runName = string.IsNullOrEmpty(gitBasedRunName) ? Guid.NewGuid().ToString() : gitBasedRunName; - var run = new TestRunDtoV2 - { - TestRunId = RunId!, - DisplayName = runName, - StartTime = startTime, - CreatorId = TokenDetails!.oid ?? "", - CreatorName = TokenDetails.userName ?? "", - //CloudRunEnabled = "false", - CloudReportingEnabled = "true", - Summary = new TestRunSummary - { - Status = "RUNNING", - StartTime = startTime, - //Projects = ["playwright-dotnet"], - //Tags = ["Nunit", "dotnet"], - //Jobs = ["playwright-dotnet"], - }, - CiConfig = new CIConfig // TODO fetch dynamically - { - Branch = CIInfo!.Branch ?? "", - Author = CIInfo.Author ?? "", - CommitId = CIInfo.CommitId ?? "", - RevisionUrl = CIInfo.RevisionUrl ?? "" - }, - TestRunConfig = new ClientConfig // TODO fetch some of these dynamically - { - Workers = 1, - PwVersion = "1.40", - Timeout = 60000, - TestType = "WebTest", - TestSdkLanguage = "Dotnet", - TestFramework = new TestFramework() { Name = "VSTest", RunnerName = "Nunit/MSTest", Version = "3.1" }, // TODO fetch runner name MSTest/Nunit - ReporterPackageVersion = "0.0.1-dotnet", - Shards = new Shard() { Current = 0, Total = 1 } - } - }; - var shard = new TestRunShardDto - { - UploadCompleted = "false", - Summary = new TestRunShardSummary - { - Status = "RUNNING", - StartTime = startTime, - }, - TestRunConfig = new ClientConfig // TODO fetch some of these dynamically - { - Workers = 1, - PwVersion = "1.40", - Timeout = 60000, - TestType = "Functional", - TestSdkLanguage = "dotnet", - TestFramework = new TestFramework() { Name = "VSTest", RunnerName = "Nunit", Version = "3.1" }, - ReporterPackageVersion = "0.0.1-dotnet", - Shards = new Shard() { Current = 0, Total = 1 }, - } - }; - var token = "Bearer " + AccessToken; - TestRunDtoV2? response = null; - try - { - Response apiResponse = _reportingTestRunsClient.PatchTestRunInfo(WorkspaceId, RunId, RequestContent.Create(run), "application/json", token, corelationId); - if (apiResponse.Content != null) - { - response = apiResponse.Content!.ToObject(new JsonObjectSerializer()); - } - } - catch (Exception ex) - { - Logger.Log(true, LogLevel.Error, ex.ToString()); - throw; - } - if (response != null) - { - TestRun = response; - - // Start shard - corelationId = Guid.NewGuid().ToString(); - TestRunShardDto? response1 = null; - try - { - Response apiResponse = _reportingTestRunsClient.PatchTestRunShardInfo(WorkspaceId, RunId, "1", RequestContent.Create(shard), "application/json", token, corelationId); - if (apiResponse.Content != null) - { - response1 = apiResponse.Content!.ToObject(new JsonObjectSerializer()); - } - } - catch (Exception ex) - { - Logger.Log(true, LogLevel.Error, ex.ToString()); - throw; - } - if (response1 != null) - { - TestRunShard = shard; // due to wrong response type TODO - } - else - { - Logger.Log(true, LogLevel.Error, "Run shard creation Failed"); - } - } - else - { - Logger.Log(true, LogLevel.Error, "Run creation Failed"); - } - LogMessage("Test Run start Handler completed"); - } - - internal void TestMessageHandler(object? sender, TestRunMessageEventArgs e) - { - LogMessage("Test Message Handler"); - ValidateArg.NotNull(sender, nameof(sender)); - ValidateArg.NotNull(e, nameof(e)); - LogMessage(e.Message); + _testProcessor?.TestRunStartHandler(sender, e); } internal void TestResultHandler(object? sender, TestResultEventArgs e) { - LogMessage("Test Result Handler"); - TestResults? testResult = GetTestCaseResultData(e.Result); - if (!EnableResultPublish) - { - return; - } - if (!IsInitialized || _reportingTestResultsClient == null || _reportingTestRunsClient == null) - { - LogErrorMessage("Test Run setup issue exiting handler"); - return; - } - // Set various counts (passed tests, failed tests, total tests) - if (testResult != null) - { - TotalTestCount++; - if (testResult.Status == "failed") - { - FailedTestCount++; - } - else if (testResult.Status == "passed") - { - PassedTestCount++; - } - else if (testResult.Status == "skipped") - { - SkippedTestCount++; - } - } - if (testResult != null) - { - TestResults.Add(testResult); - } + _testProcessor?.TestCaseResultHandler(sender, e); } internal void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) { - LogMessage("Test Run End Handler"); - if (!EnableResultPublish) - { - UpdateTestRun(e); // will not publish results, but will print informational messages - return; - } - if (!IsInitialized || _reportingTestResultsClient == null || _reportingTestRunsClient == null || TestRun == null) - { - LogErrorMessage("Test Run setup issue exiting handler"); - EnableResultPublish = false; - UpdateTestRun(e); // will not publish results, but will print informational messages - return; - } - // Upload TestResults - var corelationId = Guid.NewGuid().ToString(); - var token = "Bearer " + AccessToken; - - var body = new UploadTestResultsRequest() { Value = TestResults }; - try - { - _reportingTestResultsClient.UploadBatchTestResults(WorkspaceId, RequestContent.Create(JsonSerializer.Serialize(body)), token, corelationId, null); - LogMessage("Test Result Uploaded"); - } - catch (Exception ex) - { - LogErrorMessage(ex.Message); - } - - corelationId = Guid.NewGuid().ToString(); - TestResultsUri? sasUri = null; - Response response = _reportingTestRunsClient.GetTestRunResultsUri(WorkspaceId, RunId, token, corelationId, null); - var serializer = new JsonObjectSerializer(); - if (response.Content != null) - { - sasUri = response.Content.ToObject(serializer); - } - if (sasUri != null && !string.IsNullOrEmpty(sasUri.Uri)) - { - LogMessage("Test Run Uri: " + sasUri.ToString()); - foreach (TestResults testResult in TestResults) - { - if (RawTestResultsMap.TryGetValue(testResult.TestExecutionId!, out RawTestResult? rawResult) && rawResult != null) - { - // Upload rawResult to blob storage using sasUri - var rawTestResultJson = JsonSerializer.Serialize(rawResult); - var filePath = $"{testResult.TestExecutionId}/rawTestResult.json"; - UploadBuffer(sasUri.Uri!, rawTestResultJson, filePath); - } - else - { - LogMessage("Couldnt find rawResult for Id: " + testResult.TestExecutionId); - } - } - } - else - { - LogMessage("MPT API error: failed to upload artifacts"); - } - LogMessage("Test Results uploaded"); - // Update TestRun with CLIENT_COMPLETE - if (UpdateTestRun(e) == false) - { - LogErrorMessage("Test Run setup issue, Failed to update TestRun"); - } + _testProcessor?.TestRunCompleteHandler(sender, e); + _playwrightService?.Cleanup(); } #endregion - private bool UpdateTestRun(TestRunCompleteEventArgs e) - { - if (EnableResultPublish) - { - if (!IsInitialized || _reportingTestResultsClient == null || _reportingTestRunsClient == null || TestRun == null || TestRunShard == null) - { - // no-op - } - else - { - DateTime testRunStartedOn = DateTime.MinValue; - DateTime testRunEndedOn = DateTime.UtcNow; - long durationInMs = 0; - - var result = FailedTestCount > 0 ? "failed" : "passed"; - - if (e.ElapsedTimeInRunningTests != null) - { - testRunEndedOn = TestRunStartTime.Add(e.ElapsedTimeInRunningTests); - durationInMs = (long)e.ElapsedTimeInRunningTests.TotalMilliseconds; - } - - // Update Shard End - if (TestRunShard.Summary == null) - TestRunShard.Summary = new TestRunShardSummary(); - TestRunShard.Summary.Status = "CLIENT_COMPLETE"; - TestRunShard.Summary.StartTime = TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); - TestRunShard.Summary.EndTime = testRunEndedOn.ToString("yyyy-MM-ddTHH:mm:ssZ"); - TestRunShard.Summary.TotalTime = durationInMs; - TestRunShard.Summary.UploadMetadata = new UploadMetadata() { NumTestResults = TotalTestCount, NumTotalAttachments = 0, SizeTotalAttachments = 0 }; - LogMessage("duration:" + durationInMs); - LogMessage("StartTime:" + TestRunShard.Summary.StartTime); - LogMessage("EndTime:" + TestRunShard.Summary.EndTime); - TestRunShard.ResultsSummary = new TestRunResultsSummary - { - NumTotalTests = TotalTestCount, - NumPassedTests = PassedTestCount, - NumFailedTests = FailedTestCount, - NumSkippedTests = SkippedTestCount, - NumFlakyTests = 0, // TODO: Implement flaky tests - Status = result - }; - TestRunShard.UploadCompleted = "true"; - var token = "Bearer " + AccessToken; - var corelationId = Guid.NewGuid().ToString(); - try - { - _reportingTestRunsClient.PatchTestRunShardInfo(WorkspaceId, RunId, "1", RequestContent.Create(TestRunShard), "application/json", token, corelationId); - } - catch (Exception ex) - { - LogErrorMessage("Test Run shard failed: " + ex.ToString()); - throw; - } - - LogMessage("TestRun Shard updated"); - playwrightService?.Cleanup(); - Console.WriteLine("Visit MPT Portal for Debugging: " + Uri.EscapeUriString(PortalUrl!)); - if (EnableGithubSummary) - GenerateMarkdownSummary(); - } - } - if (informationalMessages.Count > 0) - Console.WriteLine(); - int index = 1; - foreach (string message in informationalMessages) - { - Console.WriteLine($"{index}) {message}"); - } - return true; - } - - private TestResults GetTestCaseResultData(TestResult testResultSource) - { - if (testResultSource == null) - return new TestResults(); - - LogMessage(testResultSource.TestCase.DisplayName); - TestResults testCaseResultData = new() - { - ArtifactsPath = new List(), - - AccountId = WorkspaceId!, - RunId = RunId!, - TestExecutionId = GetExecutionId(testResultSource).ToString() - }; - testCaseResultData.TestCombinationId = testCaseResultData.TestExecutionId; // TODO check - testCaseResultData.TestId = testResultSource.TestCase.Id.ToString(); - testCaseResultData.TestTitle = testResultSource.TestCase.DisplayName; - var className = FetchTestClassName(testResultSource.TestCase.FullyQualifiedName); - testCaseResultData.SuiteTitle = className; - testCaseResultData.SuiteId = className; - testCaseResultData.FileName = FetchFileName(testResultSource.TestCase.Source); - testCaseResultData.LineNumber = testResultSource.TestCase.LineNumber; - testCaseResultData.Retry = 0; // TODO Retry and PreviousRetries - testCaseResultData.WebTestConfig = new WebTestConfig - { - JobName = CIInfo!.JobId ?? "", - //ProjectName = "playwright-dotnet", // TODO no project concept NA?? - //BrowserName = "chromium", // TODO check if possible to get from test - Os = GetCurrentOS(), - }; - //testCaseResultData.Annotations = ["windows"]; // TODO MSTest/Nunit annotation ?? - //testCaseResultData.Tags = ["windows"]; // TODO NA ?? - - TimeSpan duration = testResultSource.Duration; - testCaseResultData.ResultsSummary = new TestResultsSummary - { - Duration = (long)duration.TotalMilliseconds, // TODO fallback get from End-Start - StartTime = testResultSource.StartTime.UtcDateTime.ToString(), - Status = "inconclusive" - }; - TestOutcome outcome = testResultSource.Outcome; - switch (outcome) - { - case TestOutcome.Passed: - testCaseResultData.ResultsSummary.Status = "passed"; - testCaseResultData.Status = "passed"; - break; - case TestOutcome.Failed: - testCaseResultData.ResultsSummary.Status = "failed"; - testCaseResultData.Status = "failed"; - break; - case TestOutcome.Skipped: - testCaseResultData.ResultsSummary.Status = "skipped"; - testCaseResultData.Status = "skipped"; - break; - default: - testCaseResultData.ResultsSummary.Status = "inconclusive"; - testCaseResultData.Status = "inconclusive"; - break; - } - // errorMessage, Stacktrace - RawTestResult rawResult = GetRawResultObject(testResultSource); - RawTestResultsMap.TryAdd(testCaseResultData.TestExecutionId, rawResult); - - if (!string.IsNullOrEmpty(testResultSource.ErrorMessage)) - { - ProcessTestResultMessage(testResultSource.ErrorMessage); - // TODO send it in blob - } - if (!string.IsNullOrEmpty(testResultSource.ErrorStackTrace)) - { - ProcessTestResultMessage(testResultSource.ErrorStackTrace); - // TODO send it in blob - } - - // TODO ArtifactsPaths - return testCaseResultData; - } - - private void ProcessTestResultMessage(string? message) - { - if (string.IsNullOrEmpty(message)) - { - return; - } - foreach (TestResultError testResultErrorObj in TestResultErrorConstants.ErrorConstants) - { - if (processedErrorMessageKeys.Contains(testResultErrorObj.Key!)) - continue; - if (testResultErrorObj.Pattern.IsMatch(message)) - { - AddInformationalMessage(testResultErrorObj.Message!); - processedErrorMessageKeys.Add(testResultErrorObj.Key!); - } - } - } - - private TokenDetails ParseWorkspaceIdFromAccessToken(string accessToken) - { - TokenDetails tokenDetails = new(); - if (accessToken == null) - { - if (string.IsNullOrEmpty(accessToken)) - { - throw new ArgumentNullException("AccessToken is null or empty"); - } - } - try - { - JsonWebToken inputToken = (JsonWebToken)s_tokenHandler.ReadToken(accessToken); - var aid = inputToken.Claims.FirstOrDefault(c => c.Type == "aid")?.Value ?? string.Empty; - - if (!string.IsNullOrEmpty(aid)) // Custom Token - { - LogMessage("Custom Token parsing"); - tokenDetails.aid = aid; - tokenDetails.oid = inputToken.Claims.FirstOrDefault(c => c.Type == "oid")?.Value ?? string.Empty; - tokenDetails.id = inputToken.Claims.FirstOrDefault(c => c.Type == "id")?.Value ?? string.Empty; - tokenDetails.userName = inputToken.Claims.FirstOrDefault(c => c.Type == "name")?.Value ?? string.Empty; - } - else // Entra Token - { - LogMessage("Entra Token parsing"); - tokenDetails.aid = Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_WORKSPACE_ID) ?? string.Empty; - tokenDetails.oid = inputToken.Claims.FirstOrDefault(c => c.Type == "oid")?.Value ?? string.Empty; - tokenDetails.id = string.Empty; - tokenDetails.userName = inputToken.Claims.FirstOrDefault(c => c.Type == "name")?.Value ?? string.Empty; - // TODO add back suport for old claims https://devdiv.visualstudio.com/OnlineServices/_git/PlaywrightService?path=/src/Common/Authorization/JwtSecurityTokenValidator.cs&version=GBmain&line=200&lineEnd=200&lineStartColumn=30&lineEndColumn=52&lineStyle=plain&_a=contents - } - - return tokenDetails; - } - catch (Exception ex) - { - LogErrorMessage(ex.Message); - throw; - } - } - - private static Guid GetExecutionId(TestResult testResult) - { - TestProperty? executionIdProperty = testResult.Properties.FirstOrDefault( - property => property.Id.Equals(PlaywrightConstants.ExecutionIdPropertyIdentifier)); - - Guid executionId = Guid.Empty; - if (executionIdProperty != null) - executionId = testResult.GetPropertyValue(executionIdProperty, Guid.Empty); - - return executionId.Equals(Guid.Empty) ? Guid.NewGuid() : executionId; - } - - private static RawTestResult GetRawResultObject(TestResult testResultSource) - { - List errors = new();//[testResultSource.ErrorMessage]; - if (testResultSource.ErrorMessage != null) - errors.Add(new MPTError() { message = testResultSource.ErrorMessage }); - var rawTestResult = new RawTestResult - { - errors = JsonSerializer.Serialize(errors), - stdErr = testResultSource?.ErrorStackTrace ?? string.Empty - }; - return rawTestResult; - } - - private static string GetCloudFilePath(string uri, string fileRelativePath) - { - // Assuming Constants.SAS_URI_SEPARATOR is a static property or field in a class named Constants - // that holds the character used to split the URI and the SAS token. - string[] parts = uri.Split(new string[] { PlaywrightConstants.SASUriSeparator }, StringSplitOptions.None); - string containerUri = parts[0]; - string sasToken = parts.Length > 1 ? parts[1] : string.Empty; - - return $"{containerUri}/{fileRelativePath}?{sasToken}"; - } - - private void UploadBuffer(string uri, string buffer, string fileRelativePath) - { - string cloudFilePath = GetCloudFilePath(uri, fileRelativePath); - LogMessage(cloudFilePath); - LogMessage(buffer); - BlobClient blobClient = new(new Uri(cloudFilePath)); - byte[] bufferBytes = Encoding.UTF8.GetBytes(buffer); - blobClient.Upload(new BinaryData(bufferBytes), overwrite: true); - LogMessage($"Uploaded buffer to {fileRelativePath}"); - } - - private static string FetchTestClassName(string fullyQualifiedName) - { - string[] parts = fullyQualifiedName.Split('.'); - return string.Join(".", parts.Take(parts.Length - 1)); - } - - private static string FetchFileName(string fullFilePath) - { - char[] delimiters = { '\\', '/' }; - string[] parts = fullFilePath.Split(delimiters); - return parts.Last(); - } - - private static string GetCurrentOS() - { - PlatformID platform = Environment.OSVersion.Platform; - if (platform == PlatformID.Unix) - return "Linux"; - else if (platform == PlatformID.MacOSX) - return "MacOS"; - else - return "Windows"; - } - - private void LogMessage(string message) - { - bool enable = bool.TryParse(EnableConsoleLog, out enable) == true && enable; - Logger.Log(enable, _logLevel, message); - } - - private static void LogErrorMessage(string message) - { - Logger.Log(true, LogLevel.Error, message); - } - private void InitializePlaywrightReporter(string xmlSettings) { - if (IsInitialized) - { - return; - } - Dictionary runParameters = XmlRunSettingsUtilities.GetTestRunParameters(xmlSettings); runParameters.TryGetValue(RunSettingKey.RunId, out var runId); // If run id is not provided and not set via env, try fetching it from CI info. - CIInfo = CiInfoProvider.GetCIInfo(); - if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_RUN_ID))) + CIInfo cIInfo = CiInfoProvider.GetCIInfo(); + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId))) { if (string.IsNullOrEmpty(runId?.ToString())) - Environment.SetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_RUN_ID, ReporterUtils.GetRunId(CIInfo)); + Environment.SetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId, ReporterUtils.GetRunId(cIInfo)); else - Environment.SetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_RUN_ID, runId!.ToString()); + Environment.SetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId, runId!.ToString()); } else { @@ -666,8 +92,8 @@ private void InitializePlaywrightReporter(string xmlSettings) string? enableGithubSummaryString = enableGithubSummary?.ToString(); string? enableResultPublishString = enableResultPublish?.ToString(); - EnableGithubSummary = string.IsNullOrEmpty(enableGithubSummaryString) || bool.Parse(enableGithubSummaryString!); - EnableResultPublish = string.IsNullOrEmpty(enableResultPublishString) || bool.Parse(enableResultPublishString!); + bool _enableGitHubSummary = string.IsNullOrEmpty(enableGithubSummaryString) || bool.Parse(enableGithubSummaryString!); + bool _enableResultPublish = string.IsNullOrEmpty(enableResultPublishString) || bool.Parse(enableResultPublishString!); PlaywrightServiceOptions? playwrightServiceSettings = null; try @@ -676,85 +102,47 @@ private void InitializePlaywrightReporter(string xmlSettings) } catch (Exception ex) { - Console.Error.WriteLine("Failed to initialize PlaywrightServiceSettings: " + ex.Message); + Console.Error.WriteLine("Failed to initialize PlaywrightServiceSettings: " + ex); Environment.Exit(1); } // setup entra rotation handlers - playwrightService = new PlaywrightService(null, playwrightServiceSettings!.RunId, null, playwrightServiceSettings.ServiceAuth, null, playwrightServiceSettings.AzureTokenCredential); + _playwrightService = new PlaywrightService(null, playwrightServiceSettings!.RunId, null, playwrightServiceSettings.ServiceAuth, null, playwrightServiceSettings.AzureTokenCredential); #pragma warning disable AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead. - playwrightService.InitializeAsync().GetAwaiter().GetResult(); + _playwrightService.InitializeAsync().GetAwaiter().GetResult(); #pragma warning restore AZC0102 // Do not use GetAwaiter().GetResult(). Use the TaskExtensions.EnsureCompleted() extension method instead. - RunId = Environment.GetEnvironmentVariable(PlaywrightConstants.PLAYWRIGHT_SERVICE_RUN_ID); - - try + var cloudRunId = Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceRunId); + string baseUrl = Environment.GetEnvironmentVariable(ReporterConstants.s_pLAYWRIGHT_SERVICE_REPORTING_URL); + string accessToken = Environment.GetEnvironmentVariable(ServiceEnvironmentVariable.PlaywrightServiceAccessToken); + if (string.IsNullOrEmpty(baseUrl)) { - ValidateArg.NotNullOrEmpty(BaseUrl, "Playwright Service URL"); - ValidateArg.NotNullOrEmpty(AccessToken, "Playwright Service Access Token"); + Console.Error.WriteLine(Constants.s_no_service_endpoint_error_message); + Environment.Exit(1); } - catch (Exception ex) + if (string.IsNullOrEmpty(accessToken)) { - Console.Error.WriteLine("Missing values : " + ex.Message); + Console.Error.WriteLine(Constants.s_no_auth_error); Environment.Exit(1); } - TotalTestCount = 0; - PassedTestCount = 0; - FailedTestCount = 0; - SkippedTestCount = 0; - - TestRunStartTime = DateTime.UtcNow; - TokenDetails = ParseWorkspaceIdFromAccessToken(AccessToken); - WorkspaceId = TokenDetails.aid; - LogMessage("RunId: " + RunId); - LogMessage("BaseUrl: " + BaseUrl); - LogMessage("Workspace Id: " + WorkspaceId); - - PortalUrl = PlaywrightConstants.PortalBaseUrl + WorkspaceId + PlaywrightConstants.ReportingRoute + RunId; - - _httpClient = new HttpClient(); - var baseUri = new Uri(BaseUrl!); - _reportingTestRunsClient = new ReportingTestRunsClient(baseUri); - _reportingTestResultsClient = new ReportingTestResultsClient(baseUri); - - IsInitialized = true; - - LogMessage("Playwright Service Reporter Intialized"); - } + var baseUri = new Uri(baseUrl); + var reporterUtils = new ReporterUtils(); + TokenDetails tokenDetails = reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler: null, accessToken: accessToken); + var workspaceId = tokenDetails.aid; - internal void GenerateMarkdownSummary() - { - if (CiInfoProvider.GetCIProvider() == PlaywrightConstants.GITHUB_ACTIONS) + var cloudRunMetadata = new CloudRunMetadata { - string markdownContent = @$" -#### Results: - -![pass](https://img.shields.io/badge/status-passed-brightgreen) **Passed:** {TestRunShard!.ResultsSummary!.NumPassedTests} - -![fail](https://img.shields.io/badge/status-failed-red) **Failed:** {TestRunShard.ResultsSummary.NumFailedTests} - -![flaky](https://img.shields.io/badge/status-flaky-yellow) **Flaky:** {"0"} - -![skipped](https://img.shields.io/badge/status-skipped-lightgrey) **Skipped:** {TestRunShard.ResultsSummary.NumSkippedTests} - -#### For more details, visit the [service dashboard]({Uri.EscapeUriString(PortalUrl!)}). -"; - - string filePath = Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY"); - try - { - File.WriteAllText(filePath, markdownContent); - } - catch (Exception ex) - { - LogErrorMessage($"Error writing Markdown summary: {ex}"); - } - } - } + RunId = cloudRunId, + WorkspaceId = workspaceId, + BaseUri = baseUri, + EnableResultPublish = _enableResultPublish, + EnableGithubSummary = _enableGitHubSummary, + TestRunStartTime = DateTime.UtcNow, + AccessTokenDetails = tokenDetails, + }; - private void AddInformationalMessage(string message) - { - informationalMessages.Add(message); + _testProcessor = new TestProcessor(cloudRunMetadata, cIInfo); + _logger.Info("Playwright Service Reporter Initialized"); } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs new file mode 100644 index 0000000000000..c5a4665c09a25 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/DataProcessor.cs @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using System.Linq; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using System.Text.Json; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor +{ + internal class DataProcessor : IDataProcessor + { + private readonly ILogger _logger; + private readonly CIInfo _cIInfo; + private readonly CloudRunMetadata _cloudRunMetadata; + public DataProcessor(CloudRunMetadata cloudRunMetadata, CIInfo cIInfo, ILogger? logger = null) + { + _cloudRunMetadata = cloudRunMetadata; + _cIInfo = cIInfo; + _logger = logger ?? new Logger(); + } + + public TestRunDto GetTestRun() + { + var startTime = _cloudRunMetadata.TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); + var gitBasedRunName = ReporterUtils.GetRunName(CiInfoProvider.GetCIInfo())?.Trim(); + string runName = string.IsNullOrEmpty(gitBasedRunName) ? _cloudRunMetadata.RunId! : gitBasedRunName!; + var run = new TestRunDto + { + TestRunId = _cloudRunMetadata.RunId!, + DisplayName = runName, + StartTime = startTime, + CreatorId = _cloudRunMetadata.AccessTokenDetails!.oid ?? "", + CreatorName = _cloudRunMetadata.AccessTokenDetails!.userName?.Trim() ?? "", + CloudReportingEnabled = true, + CloudRunEnabled = false, + CiConfig = new CIConfig + { + Branch = _cIInfo.Branch, + Author = _cIInfo.Author, + CommitId = _cIInfo.CommitId, + RevisionUrl = _cIInfo.RevisionUrl, + CiProviderName = _cIInfo.Provider ?? CIConstants.s_dEFAULT + }, + TestRunConfig = new ClientConfig // TODO fetch some of these dynamically + { + Workers = 1, + PwVersion = "1.40", + Timeout = 60000, + TestType = "WebTest", + TestSdkLanguage = "CSHARP", + TestFramework = new TestFramework() { Name = "PLAYWRIGHT", RunnerName = "NUNIT", Version = "3.1" }, // TODO fetch runner name MSTest/Nunit + ReporterPackageVersion = "1.0.0-beta.1", + Shards = new Shard() { Total = 1 } + } + }; + return run; + } + + public TestRunShardDto GetTestRunShard() + { + var startTime = _cloudRunMetadata.TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); + var shard = new TestRunShardDto + { + UploadCompleted = false, + ShardId = "1", + Summary = new TestRunShardSummary + { + Status = "RUNNING", + StartTime = startTime, + }, + Workers = 1 + }; + return shard; + } + public TestResults GetTestCaseResultData(TestResult? testResultSource) + { + if (testResultSource == null) + return new TestResults(); + + TestResults testCaseResultData = new() + { + ArtifactsPath = new List(), + AccountId = _cloudRunMetadata.WorkspaceId!, + RunId = _cloudRunMetadata.RunId!, + TestExecutionId = GetExecutionId(testResultSource).ToString() + }; + testCaseResultData.TestCombinationId = testCaseResultData.TestExecutionId; // TODO check + testCaseResultData.TestId = testResultSource.TestCase.Id.ToString(); + testCaseResultData.TestTitle = testResultSource.TestCase.DisplayName; + var className = FetchTestClassName(testResultSource.TestCase.FullyQualifiedName); + testCaseResultData.SuiteTitle = className; + testCaseResultData.SuiteId = className; + testCaseResultData.FileName = FetchFileName(testResultSource.TestCase.Source); + testCaseResultData.LineNumber = testResultSource.TestCase.LineNumber; + testCaseResultData.Retry = 0; // TODO Retry and PreviousRetries + testCaseResultData.WebTestConfig = new WebTestConfig + { + JobName = _cIInfo.JobId ?? "", + //ProjectName = "playwright-dotnet", // TODO no project concept NA?? + //BrowserName = "chromium", // TODO check if possible to get from test + Os = ReporterUtils.GetCurrentOS(), + }; + //testCaseResultData.Annotations = ["windows"]; // TODO MSTest/Nunit annotation ?? + //testCaseResultData.Tags = ["windows"]; // TODO NA ?? + + TimeSpan duration = testResultSource.Duration; + testCaseResultData.ResultsSummary = new TestResultsSummary + { + Duration = (long)duration.TotalMilliseconds, // TODO fallback get from End-Start + StartTime = testResultSource.StartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"), + Status = TestCaseResultStatus.s_iNCONCLUSIVE + }; + TestOutcome outcome = testResultSource.Outcome; + switch (outcome) + { + case TestOutcome.Passed: + testCaseResultData.ResultsSummary.Status = TestCaseResultStatus.s_pASSED; + testCaseResultData.Status = TestCaseResultStatus.s_pASSED; + break; + case TestOutcome.Failed: + testCaseResultData.ResultsSummary.Status = TestCaseResultStatus.s_fAILED; + testCaseResultData.Status = TestCaseResultStatus.s_fAILED; + break; + case TestOutcome.Skipped: + testCaseResultData.ResultsSummary.Status = TestCaseResultStatus.s_sKIPPED; + testCaseResultData.Status = TestCaseResultStatus.s_sKIPPED; + break; + default: + testCaseResultData.ResultsSummary.Status = TestCaseResultStatus.s_iNCONCLUSIVE; + testCaseResultData.Status = TestCaseResultStatus.s_iNCONCLUSIVE; + break; + } + return testCaseResultData; + } + + public static RawTestResult GetRawResultObject(TestResult? testResultSource) + { + if (testResultSource == null) + return new RawTestResult(); + List errors = new();//[testResultSource.ErrorMessage]; + if (testResultSource.ErrorMessage != null) + errors.Add(new MPTError() { message = testResultSource.ErrorMessage }); + var rawTestResult = new RawTestResult + { + errors = JsonSerializer.Serialize(errors), + stdErr = testResultSource?.ErrorStackTrace ?? string.Empty + }; + return rawTestResult; + } + + #region Data Processor Utility Methods + + private static Guid GetExecutionId(TestResult testResult) + { + TestProperty? executionIdProperty = testResult.Properties.FirstOrDefault( + property => property.Id.Equals(ReporterConstants.s_executionIdPropertyIdentifier)); + + Guid executionId = Guid.Empty; + if (executionIdProperty != null) + executionId = testResult.GetPropertyValue(executionIdProperty, Guid.Empty); + + return executionId.Equals(Guid.Empty) ? Guid.NewGuid() : executionId; + } + + private static string FetchTestClassName(string fullyQualifiedName) + { + string[] parts = fullyQualifiedName.Split('.'); + return string.Join(".", parts.Take(parts.Length - 1)); + } + + private static string FetchFileName(string fullFilePath) + { + char[] delimiters = { '\\', '/' }; + string[] parts = fullFilePath.Split(delimiters); + return parts.Last(); + } + #endregion + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs new file mode 100644 index 0000000000000..db8142ca2a407 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Processor/TestProcessor.cs @@ -0,0 +1,285 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.Json; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using Azure.Storage.Blobs; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor +{ + internal class TestProcessor : ITestProcessor + { + // Dependency Injection + private readonly IDataProcessor _dataProcessor; + private readonly ILogger _logger; + private readonly ICloudRunErrorParser _cloudRunErrorParser; + private readonly IServiceClient _serviceClient; + private readonly IConsoleWriter _consoleWriter; + private readonly CIInfo _cIInfo; + private readonly CloudRunMetadata _cloudRunMetadata; + + // Test Metadata + internal int TotalTestCount { get; set; } = 0; + internal int PassedTestCount { get; set; } = 0; + internal int FailedTestCount { get; set; } = 0; + internal int SkippedTestCount { get; set; } = 0; + internal List TestResults { get; set; } = new List(); + internal ConcurrentDictionary RawTestResultsMap { get; set; } = new(); + internal bool FatalTestExecution { get; set; } = false; + internal TestRunShardDto? _testRunShard; + + public TestProcessor(CloudRunMetadata cloudRunMetadata, CIInfo cIInfo, ILogger? logger = null, IDataProcessor? dataProcessor = null, ICloudRunErrorParser? cloudRunErrorParser = null, IServiceClient? serviceClient = null, IConsoleWriter? consoleWriter = null) + { + _cloudRunMetadata = cloudRunMetadata; + _cIInfo = cIInfo; + _logger = logger ?? new Logger(); + _dataProcessor = dataProcessor ?? new DataProcessor(_cloudRunMetadata, _cIInfo, _logger); + _cloudRunErrorParser = cloudRunErrorParser ?? new CloudRunErrorParser(_logger); + _serviceClient = serviceClient ?? new ServiceClient(_cloudRunMetadata, _cloudRunErrorParser); + _consoleWriter = consoleWriter ?? new ConsoleWriter(); + } + + public void TestRunStartHandler(object? sender, TestRunStartEventArgs e) + { + try + { + _logger.Info("Initialising test run"); + if (!_cloudRunMetadata.EnableResultPublish || FatalTestExecution) + { + return; + } + TestRunDto run = _dataProcessor.GetTestRun(); + TestRunShardDto shard = _dataProcessor.GetTestRunShard(); + TestRunDto? testRun = _serviceClient.PatchTestRunInfo(run); + if (testRun == null) + { + _logger.Error("Failed to patch test run info"); + FatalTestExecution = true; + return; + } + _logger.Info("Successfully patched test run - init"); + TestRunShardDto? testShard = _serviceClient.PostTestRunShardInfo(shard); + if (testShard == null) + { + _logger.Error("Failed to patch test run shard info"); + FatalTestExecution = true; + return; + } + _testRunShard = testShard; + _logger.Info("Successfully patched test run shard - init"); + _consoleWriter.WriteLine($"\nInitializing reporting for this test run. You can view the results at: {_cloudRunMetadata.PortalUrl!}"); + } + catch (Exception ex) + { + _logger.Error($"Failed to initialise test run: {ex}"); + FatalTestExecution = true; + } + } + public void TestCaseResultHandler(object? sender, TestResultEventArgs e) + { + try + { + TestResult testResultSource = e.Result; + TestResults? testResult = _dataProcessor.GetTestCaseResultData(testResultSource); + RawTestResult rawResult = DataProcessor.GetRawResultObject(testResultSource); + RawTestResultsMap.TryAdd(testResult.TestExecutionId, rawResult); + + // TODO - Send error to blob + _cloudRunErrorParser.HandleScalableRunErrorMessage(testResultSource.ErrorMessage); + _cloudRunErrorParser.HandleScalableRunErrorMessage(testResultSource.ErrorStackTrace); + if (!_cloudRunMetadata.EnableResultPublish) + { + return; + } + if (testResult != null) + { + TotalTestCount++; + if (testResult.Status == TestCaseResultStatus.s_fAILED) + { + FailedTestCount++; + } + else if (testResult.Status == TestCaseResultStatus.s_pASSED) + { + PassedTestCount++; + } + else if (testResult.Status == TestCaseResultStatus.s_sKIPPED) + { + SkippedTestCount++; + } + TestResults.Add(testResult); + } + } + catch (Exception ex) + { + // test case processing failures should not stop the test run + _logger.Error($"Failed to process test case result: {ex}"); + } + } + public void TestRunCompleteHandler(object? sender, TestRunCompleteEventArgs e) + { + _logger.Info("Test run complete handler - start"); + if (_cloudRunMetadata.EnableResultPublish && !FatalTestExecution) + { + try + { + var body = new UploadTestResultsRequest() { Value = TestResults }; + _serviceClient.UploadBatchTestResults(body); + _logger.Info("Successfully uploaded test results"); + } + catch (Exception ex) + { + _logger.Error($"Failed to upload test results: {ex}"); + } + try + { + TestResultsUri? sasUri = _serviceClient.GetTestRunResultsUri(); + if (!string.IsNullOrEmpty(sasUri?.Uri)) + { + foreach (TestResults testResult in TestResults) + { + if (RawTestResultsMap.TryGetValue(testResult.TestExecutionId!, out RawTestResult? rawResult) && rawResult != null) + { + // Renew the SAS URI if needed + var reporterUtils = new ReporterUtils(); + if (sasUri == null || !reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri.Uri)) + { + sasUri = _serviceClient.GetTestRunResultsUri(); // Create new SAS URI + _logger.Info($"Fetched SAS URI with validity: {sasUri?.ExpiresAt} and access: {sasUri?.AccessLevel}."); + } + if (sasUri == null) + { + _logger.Warning("SAS URI is empty"); + continue; // allow recovery from temporary reporter API failures. In the future, we might consider shortciruiting the upload process. + } + + // Upload rawResult to blob storage using sasUri + var rawTestResultJson = JsonSerializer.Serialize(rawResult); + var filePath = $"{testResult.TestExecutionId}/rawTestResult.json"; + UploadBuffer(sasUri!.Uri!, rawTestResultJson, filePath); + } + else + { + _logger.Info("Couldn't find rawResult for Id: " + testResult.TestExecutionId); + } + } + _logger.Info("Successfully uploaded raw test results"); + } + else + { + _logger.Error("SAS URI is empty"); + } + } + catch (Exception ex) + { + _logger.Error($"Failed to upload artifacts: {ex}"); + } + } + EndTestRun(e); + } + + #region Test Processor Helper Methods + private void EndTestRun(TestRunCompleteEventArgs e) + { + if (_cloudRunMetadata.EnableResultPublish && !FatalTestExecution) + { + try + { + _testRunShard = GetTestRunEndShard(e); + _serviceClient.PostTestRunShardInfo(_testRunShard); + _logger.Info("Successfully ended test run shard"); + } + catch (Exception ex) + { + _logger.Error($"Failed to end test run shard: {ex}"); + } + _consoleWriter.WriteLine($"\nTest Report: {_cloudRunMetadata.PortalUrl!}"); + if (_cloudRunMetadata.EnableGithubSummary) + { + GenerateMarkdownSummary(); + } + } + _cloudRunErrorParser.DisplayMessages(); + } + private static string GetCloudFilePath(string uri, string fileRelativePath) + { + string[] parts = uri.Split(new string[] { ReporterConstants.s_sASUriSeparator }, StringSplitOptions.None); + string containerUri = parts[0]; + string sasToken = parts.Length > 1 ? parts[1] : string.Empty; + + return $"{containerUri}/{fileRelativePath}?{sasToken}"; + } + private void UploadBuffer(string uri, string buffer, string fileRelativePath) + { + string cloudFilePath = GetCloudFilePath(uri, fileRelativePath); + BlobClient blobClient = new(new Uri(cloudFilePath)); + byte[] bufferBytes = Encoding.UTF8.GetBytes(buffer); + blobClient.Upload(new BinaryData(bufferBytes), overwrite: true); + _logger.Info($"Uploaded buffer to {fileRelativePath}"); + } + private TestRunShardDto GetTestRunEndShard(TestRunCompleteEventArgs e) + { + DateTime testRunEndedOn = DateTime.UtcNow; + long durationInMs = 0; + + var result = FailedTestCount > 0 ? TestCaseResultStatus.s_fAILED : TestCaseResultStatus.s_pASSED; + + if (e.ElapsedTimeInRunningTests != null) + { + testRunEndedOn = _cloudRunMetadata.TestRunStartTime.Add(e.ElapsedTimeInRunningTests); + durationInMs = (long)e.ElapsedTimeInRunningTests.TotalMilliseconds; + } + TestRunShardDto? testRunShard = _testRunShard; + // Update Shard End + if (testRunShard!.Summary == null) + testRunShard.Summary = new TestRunShardSummary(); + testRunShard.Summary.Status = "CLIENT_COMPLETE"; + testRunShard.Summary.StartTime = _cloudRunMetadata.TestRunStartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"); + testRunShard.Summary.EndTime = testRunEndedOn.ToString("yyyy-MM-ddTHH:mm:ssZ"); + testRunShard.Summary.TotalTime = durationInMs; + testRunShard.Summary.UploadMetadata = new UploadMetadata() { NumTestResults = TotalTestCount, NumTotalAttachments = 0, SizeTotalAttachments = 0 }; + testRunShard.UploadCompleted = true; + return testRunShard; + } + private void GenerateMarkdownSummary() + { + if (_cIInfo.Provider == CIConstants.s_gITHUB_ACTIONS) + { + string markdownContent = @$" +#### Results: + +![pass](https://img.shields.io/badge/status-passed-brightgreen) **Passed:** {PassedTestCount} + +![fail](https://img.shields.io/badge/status-failed-red) **Failed:** {FailedTestCount} + +![flaky](https://img.shields.io/badge/status-flaky-yellow) **Flaky:** {"0"} + +![skipped](https://img.shields.io/badge/status-skipped-lightgrey) **Skipped:** {SkippedTestCount} + +#### For more details, visit the [service dashboard]({Uri.EscapeUriString(_cloudRunMetadata.PortalUrl!)}). +"; + + string filePath = Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY"); + try + { + File.WriteAllText(filePath, markdownContent); + } + catch (Exception ex) + { + _logger.Error($"Error writing Markdown summary: {ex}"); + } + } + } + #endregion + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/CiInfoProvider.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/CiInfoProvider.cs index 97e332b625056..fa229851e41d3 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/CiInfoProvider.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/CiInfoProvider.cs @@ -3,7 +3,6 @@ using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; using System; -using PlaywrightConstants = Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility.Constants; namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; @@ -23,22 +22,22 @@ private static bool IsAzureDevOps() internal static string GetCIProvider() { if (IsGitHubActions()) - return PlaywrightConstants.GITHUB_ACTIONS; + return CIConstants.s_gITHUB_ACTIONS; else if (IsAzureDevOps()) - return PlaywrightConstants.AZURE_DEVOPS; + return CIConstants.s_aZURE_DEVOPS; else - return PlaywrightConstants.DEFAULT; + return CIConstants.s_dEFAULT; } internal static CIInfo GetCIInfo() { string ciProvider = GetCIProvider(); - if (ciProvider == PlaywrightConstants.GITHUB_ACTIONS) + if (ciProvider == CIConstants.s_gITHUB_ACTIONS) { // Logic to get GitHub Actions CIInfo return new CIInfo { - Provider = PlaywrightConstants.GITHUB_ACTIONS, + Provider = CIConstants.s_gITHUB_ACTIONS, Repo = Environment.GetEnvironmentVariable("GITHUB_REPOSITORY_ID"), Branch = GetGHBranchName(), Author = Environment.GetEnvironmentVariable("GITHUB_ACTOR"), @@ -53,12 +52,12 @@ internal static CIInfo GetCIInfo() JobId = Environment.GetEnvironmentVariable("GITHUB_JOB") }; } - else if (ciProvider == PlaywrightConstants.AZURE_DEVOPS) + else if (ciProvider == CIConstants.s_aZURE_DEVOPS) { // Logic to get Azure DevOps CIInfo return new CIInfo { - Provider = PlaywrightConstants.AZURE_DEVOPS, + Provider = CIConstants.s_aZURE_DEVOPS, Repo = Environment.GetEnvironmentVariable("BUILD_REPOSITORY_ID"), Branch = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH"), Author = Environment.GetEnvironmentVariable("BUILD_REQUESTEDFOR"), @@ -78,7 +77,7 @@ internal static CIInfo GetCIInfo() // Handle unsupported CI provider return new CIInfo { - Provider = PlaywrightConstants.DEFAULT, + Provider = CIConstants.s_dEFAULT, Repo = Environment.GetEnvironmentVariable("REPO"), Branch = Environment.GetEnvironmentVariable("BRANCH"), Author = Environment.GetEnvironmentVariable("AUTHOR"), diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Constants.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Constants.cs deleted file mode 100644 index e04af7928a216..0000000000000 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Constants.cs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System.Collections.Generic; -using System.Text.RegularExpressions; - -namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; - -internal static class Constants -{ - /// - /// Property Id storing the ExecutionId. - /// - internal const string ExecutionIdPropertyIdentifier = "ExecutionId"; - - /// - /// Property Id storing the ParentExecutionId. - /// - internal const string ParentExecutionIdPropertyIdentifier = "ParentExecId"; - - /// - /// Property If storing the TestType. - /// - internal const string TestTypePropertyIdentifier = "TestType"; - - internal const string SASUriSeparator = "?"; - - internal const string PortalBaseUrl = "https://playwright.microsoft.com/workspaces/"; - - internal const string ReportingRoute = "/runs/"; - - internal const string ReportingAPIVersion_2024_04_30_preview = "2024-04-30-preview"; - - internal const string ReportingAPIVersion_2024_05_20_preview = "2024-05-20-preview"; - - internal const string PLAYWRIGHT_SERVICE_REPORTING_URL = "PLAYWRIGHT_SERVICE_REPORTING_URL"; - - internal const string PLAYWRIGHT_SERVICE_WORKSPACE_ID = "PLAYWRIGHT_SERVICE_WORKSPACE_ID"; - - internal const string PLAYWRIGHT_SERVICE_ACCESS_TOKEN = "PLAYWRIGHT_SERVICE_ACCESS_TOKEN"; - - internal const string PLAYWRIGHT_SERVICE_DEBUG = "PLAYWRIGHT_SERVICE_DEBUG"; - - internal const string PLAYWRIGHT_SERVICE_RUN_ID = "PLAYWRIGHT_SERVICE_RUN_ID"; - - internal const string GITHUB_ACTIONS = "GitHub Actions"; - internal const string AZURE_DEVOPS = "Azure DevOps"; - internal const string DEFAULT = "Default"; -} - -internal enum TestErrorType -{ - Scalable -} - -internal class TestResultError -{ - internal string? Key { get; set; } = string.Empty; - internal string? Message { get; set; } = string.Empty; - internal Regex Pattern { get; set; } = new Regex(string.Empty); - internal TestErrorType Type { get; set; } -} - -internal static class TestResultErrorConstants -{ - public static List ErrorConstants = new() - { - new TestResultError - { - Key = "Unauthorized_Scalable", - Message = "The authentication token provided is invalid. Please check the token and try again.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*401 Unauthorized)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "NoPermissionOnWorkspace_Scalable", - Message = @"You do not have the required permissions to run tests. This could be because: - - a. You do not have the required roles on the workspace. Only Owner and Contributor roles can run tests. Contact the service administrator. - b. The workspace you are trying to run the tests on is in a different Azure tenant than what you are signed into. Check the tenant id from Azure portal and login using the command 'az login --tenant '.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=[\s\S]*CheckAccess API call with non successful response)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "InvalidWorkspace_Scalable", - Message = "The specified workspace does not exist. Please verify your workspace settings.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*InvalidAccountOrSubscriptionState)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "AccessKeyBasedAuthNotSupported_Scalable", - Message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*403 Forbidden)(?=.*AccessKeyBasedAuthNotSupported)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "ServiceUnavailable_Scalable", - Message = "The service is currently unavailable. Please check the service status and try again.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*503 Service Unavailable)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "GatewayTimeout_Scalable", - Message = "The request to the service timed out. Please try again later.", - Pattern = new Regex(@"(?=.*Microsoft\.Playwright\.PlaywrightException)(?=.*504 Gateway Timeout)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "QuotaLimitError_Scalable", - Message = "It is possible that the maximum number of concurrent sessions allowed for your workspace has been exceeded.", - Pattern = new Regex(@"(Timeout .* exceeded)(?=[\s\S]*ws connecting)", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - }, - new TestResultError - { - Key = "BrowserConnectionError_Scalable", - Message = "The service is currently unavailable. Please try again after some time.", - Pattern = new Regex(@"Target page, context or browser has been closed", RegexOptions.IgnoreCase), - Type = TestErrorType.Scalable - } - }; -} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Logger.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Logger.cs deleted file mode 100644 index e01a38ca03960..0000000000000 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/Logger.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; - -namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; - -internal enum LogLevel -{ - Debug, - Info, - Warning, - Error -} - -internal static class Logger -{ - public static void Log(bool enableConsoleLog, LogLevel level, string message) - { - if (enableConsoleLog) - { - Console.WriteLine($"{DateTime.Now} [{level}]: {message}"); - } - } -} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs index c25c8d466eb9d..9923a81ab1f38 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/src/Utility/ReporterUtils.cs @@ -3,18 +3,27 @@ using System; using System.Diagnostics; +using System.Linq; using System.Security.Cryptography; using System.Threading.Tasks; using Azure.Core.Pipeline; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Microsoft.IdentityModel.JsonWebTokens; namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility { internal class ReporterUtils { + private readonly ILogger _logger; + public ReporterUtils(ILogger? logger = null) + { + _logger = logger ?? new Logger(); + } internal static string GetRunId(CIInfo cIInfo) { - if (cIInfo.Provider == Constants.DEFAULT) + if (cIInfo.Provider == CIConstants.s_dEFAULT) { return Guid.NewGuid().ToString(); } @@ -37,7 +46,7 @@ internal static string GetRunName(CIInfo ciInfo) string GIT_REV_PARSE = "git rev-parse --is-inside-work-tree"; string GIT_COMMIT_MESSAGE_COMMAND = "git log -1 --pretty=%B"; - if (ciInfo.Provider == Constants.GITHUB_ACTIONS && + if (ciInfo.Provider == CIConstants.s_gITHUB_ACTIONS && Environment.GetEnvironmentVariable("GITHUB_EVENT_NAME") == "pull_request") { var prNumber = Environment.GetEnvironmentVariable("GITHUB_REF_NAME")?.Split('/')[0]; @@ -94,5 +103,100 @@ internal static async Task RunCommandAsync(string command, bool async = return result; } } + + internal static string GetCurrentOS() + { + PlatformID platform = Environment.OSVersion.Platform; + if (platform == PlatformID.Unix) + return OSConstants.s_lINUX; + else if (platform == PlatformID.MacOSX) + return OSConstants.s_mACOS; + else + return OSConstants.s_wINDOWS; + } + + internal TokenDetails ParseWorkspaceIdFromAccessToken(JsonWebTokenHandler? jsonWebTokenHandler, string? accessToken) + { + if (jsonWebTokenHandler == null) + { + jsonWebTokenHandler = new JsonWebTokenHandler(); + } + TokenDetails tokenDetails = new(); + if (string.IsNullOrEmpty(accessToken)) + { + throw new ArgumentNullException(nameof(accessToken), "AccessToken is null or empty"); + } + try + { + JsonWebToken inputToken = (JsonWebToken)jsonWebTokenHandler.ReadToken(accessToken); + var aid = inputToken.Claims.FirstOrDefault(c => c.Type == "aid")?.Value ?? string.Empty; + + if (!string.IsNullOrEmpty(aid)) // Custom Token + { + _logger.Info("Custom Token parsing"); + tokenDetails.aid = aid; + tokenDetails.oid = inputToken.Claims.FirstOrDefault(c => c.Type == "oid")?.Value ?? string.Empty; + tokenDetails.id = inputToken.Claims.FirstOrDefault(c => c.Type == "id")?.Value ?? string.Empty; + tokenDetails.userName = inputToken.Claims.FirstOrDefault(c => c.Type == "name")?.Value ?? string.Empty; + } + else // Entra Token + { + _logger.Info("Entra Token parsing"); + tokenDetails.aid = Environment.GetEnvironmentVariable(ReporterConstants.s_pLAYWRIGHT_SERVICE_WORKSPACE_ID) ?? string.Empty; + tokenDetails.oid = inputToken.Claims.FirstOrDefault(c => c.Type == "oid")?.Value ?? string.Empty; + tokenDetails.id = string.Empty; + tokenDetails.userName = inputToken.Claims.FirstOrDefault(c => c.Type == "name")?.Value ?? string.Empty; + // TODO add back suport for old claims https://devdiv.visualstudio.com/OnlineServices/_git/PlaywrightService?path=/src/Common/Authorization/JwtSecurityTokenValidator.cs&version=GBmain&line=200&lineEnd=200&lineStartColumn=30&lineEndColumn=52&lineStyle=plain&_a=contents + } + + return tokenDetails; + } + catch (Exception) + { + throw; + } + } + + internal bool IsTimeGreaterThanCurrentPlus10Minutes(string sasUri) + { + try + { + // Parse the SAS URI + Uri url = new Uri(sasUri); + string query = url.Query; + var queryParams = System.Web.HttpUtility.ParseQueryString(query); + string expiryTime = queryParams["se"]; // 'se' is the query parameter for the expiry time + + if (!string.IsNullOrEmpty(expiryTime)) + { + // Convert expiry time to a timestamp + DateTime expiryDateTime = DateTime.Parse(expiryTime, null, System.Globalization.DateTimeStyles.RoundtripKind); + long timestampFromIsoString = ((DateTimeOffset)expiryDateTime).ToUnixTimeMilliseconds(); + + // Get current time + 10 minutes in milliseconds + long currentTimestampPlus10Minutes = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + (10 * 60 * 1000); + + bool isSasValidityGreaterThanCurrentTimePlus10Minutes = timestampFromIsoString > currentTimestampPlus10Minutes; + + if (!isSasValidityGreaterThanCurrentTimePlus10Minutes) + { + // Log if SAS is close to expiry + _logger.Info( + $"Sas rotation required because close to expiry, SasUriValidTillTime: {timestampFromIsoString}, CurrentTime: {currentTimestampPlus10Minutes}" + ); + } + + return isSasValidityGreaterThanCurrentTimePlus10Minutes; + } + + _logger.Info("Sas rotation required because expiry param not found."); + return false; + } + catch (Exception ex) + { + _logger.Info($"Sas rotation required because of {ex.Message}."); + return false; + } + } } } diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/CloudRunErrorParserTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/CloudRunErrorParserTests.cs new file mode 100644 index 0000000000000..f9235f972283e --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/CloudRunErrorParserTests.cs @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Text.RegularExpressions; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Microsoft.Extensions.FileSystemGlobbing.Internal; +using Moq; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Implementation +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class CloudRunErrorParserTests + { + private Mock? _loggerMock; + private Mock? _consoleWriterMock; + private CloudRunErrorParser? _errorParser; + + [SetUp] + public void Setup() + { + _loggerMock = new Mock(); + _consoleWriterMock = new Mock(); + _errorParser = new CloudRunErrorParser(_loggerMock.Object, _consoleWriterMock.Object); + } + + [Test] + public void TryPushMessageAndKey_WithValidMessageAndKey_ReturnsTrue() + { + string message = "Test message"; + string key = "Test key"; + + bool result = _errorParser!.TryPushMessageAndKey(message, key); + + Assert.IsTrue(result); + } + + [Test] + public void TryPushMessageAndKey_WithNullOrEmptyMessage_ReturnsFalse() + { + string? message = null; + string key = "Test key"; + + bool result = _errorParser!.TryPushMessageAndKey(message, key); + + Assert.IsFalse(result); + } + + [Test] + public void TryPushMessageAndKey_WithNullOrEmptyKey_ReturnsFalse() + { + string message = "Test message"; + string? key = null; + + bool result = _errorParser!.TryPushMessageAndKey(message, key); + + Assert.IsFalse(result); + } + + [Test] + public void TryPushMessageAndKey_WithExistingKey_ReturnsFalse() + { + string message = "Test message"; + string key = "Existing key"; + _errorParser!.TryPushMessageAndKey(message, key); + + bool result = _errorParser.TryPushMessageAndKey(message, key); + + Assert.IsFalse(result); + } + + [Test] + public void PushMessage_AddsMessageToList() + { + string message = "Test message"; + + _errorParser!.PushMessage(message); + + CollectionAssert.Contains(_errorParser!.InformationalMessages, message); + } + + [Test] + public void DisplayMessages_WithMessages_WritesMessagesToConsole() + { + _errorParser!.PushMessage("Message 1"); + _errorParser.PushMessage("Message 2"); + + _errorParser.DisplayMessages(); + + _consoleWriterMock!.Verify(cw => cw.WriteLine(null), Times.Once); + _consoleWriterMock.Verify(cw => cw.WriteLine("1) Message 1"), Times.Once); + _consoleWriterMock.Verify(cw => cw.WriteLine("2) Message 2"), Times.Once); + } + + [Test] + public void DisplayMessages_WithoutMessages_DoesNotWriteToConsole() + { + _errorParser!.DisplayMessages(); + + _consoleWriterMock!.Verify(cw => cw.WriteLine(null), Times.Never); + _consoleWriterMock.Verify(cw => cw.WriteLine(It.IsAny()), Times.Never); + } + + [Test] + public void PrintErrorToConsole_WritesErrorMessageToConsole() + { + string errorMessage = "Test error message"; + + _errorParser!.PrintErrorToConsole(errorMessage); + + _consoleWriterMock!.Verify(cw => cw.WriteError(errorMessage), Times.Once); + } + + [Test] + public void HandleScalableRunErrorMessage_WithNullMessage_DoesNotPushMessage() + { + _errorParser!.HandleScalableRunErrorMessage(null); + + Assert.IsEmpty(_errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessage_WithoutMatchingPattern_DoesNotPushMessage() + { + string message = "Unknown error"; + + _errorParser!.HandleScalableRunErrorMessage(message); + + Assert.IsEmpty(_errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessage401_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 401 Unauthorized"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The authentication token provided is invalid. Please check the token and try again."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageNoPermissionOnWorkspaceScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nCheckAccess API call with non successful response."; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = @"You do not have the required permissions to run tests. This could be because: + + a. You do not have the required roles on the workspace. Only Owner and Contributor roles can run tests. Contact the service administrator. + b. The workspace you are trying to run the tests on is in a different Azure tenant than what you are signed into. Check the tenant id from Azure portal and login using the command 'az login --tenant '."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageInvalidWorkspaceScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nInvalidAccountOrSubscriptionState"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The specified workspace does not exist. Please verify your workspace settings."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageInvalidAccessToken_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nInvalidAccessToken"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The provided access token does not match the specified workspace URL. Please verify that both values are correct."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageAccessTokenOrUserOrWorkspaceNotFoundScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 404 Not Found\r\nNotFound"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The data for the user, workspace or access token was not found. Please check the request or create new token and try again."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageAccessKeyBasedAuthNotSupportedScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_123/browsers 403 Forbidden\r\nAccessKeyBasedAuthNotSupported"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "Authentication through service access token is disabled for this workspace. Please use Entra ID to authenticate."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageServiceUnavailableScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers 503 Service Unavailable"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The service is currently unavailable. Please check the service status and try again."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageGatewayTimeoutScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = " Microsoft.Playwright.PlaywrightException : WebSocket error: wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers 504 Gateway Timeout"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The request to the service timed out. Please try again later."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageQuotaLimitErrorScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = "Timeout 60000s exceeded,\r\nws connecting wss://eastus.api.playwright.microsoft.com/accounts/eastus_1120dd21-4e05-4b3d-8b54-e329307ff214/browsers"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "It is possible that the maximum number of concurrent sessions allowed for your workspace has been exceeded."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + + [Test] + public void HandleScalableRunErrorMessageBrowserConnectionErrorScalable_WithMatchingPattern_PushesMessage() + { + string errorMessage = "Target page, context or browser has been closed"; + + _errorParser!.HandleScalableRunErrorMessage(errorMessage); + var message = "The service is currently unavailable. Please try again after some time."; + Assert.Contains(message, _errorParser.InformationalMessages); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/ServiceClientTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/ServiceClientTests.cs new file mode 100644 index 0000000000000..30f34150a1f57 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Implementation/ServiceClientTests.cs @@ -0,0 +1,371 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Text.Json; +using Azure.Core; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Client; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Implementation; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Moq; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Implementation +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class ServiceClientTests + { + private ServiceClient? _serviceClient; + private Mock? _mockTestReportingClient; + private Mock? _mockCloudRunErrorParser; + private Mock? _mockLogger; + private CloudRunMetadata? _cloudRunMetadata; + + [SetUp] + public void Setup() + { + _mockTestReportingClient = new Mock(); + _mockCloudRunErrorParser = new Mock(); + _mockLogger = new Mock(); + _cloudRunMetadata = new CloudRunMetadata + { + BaseUri = new Uri("https://example.com"), + WorkspaceId = "workspaceId", + RunId = "runId" + }; + + _serviceClient = new ServiceClient(_cloudRunMetadata, _mockCloudRunErrorParser.Object, _mockTestReportingClient.Object, _mockLogger.Object); + } + + [Test] + public void PatchTestRunInfo_ReturnsTestRunDto() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(200); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestRunDto? result = _serviceClient!.PatchTestRunInfo(run); + + Assert.IsNotNull(result); + } + + [Test] + public void PatchTestRunInfo_On409Conflict_Throws() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(409); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + Assert.Throws(() => _serviceClient!.PatchTestRunInfo(run)); + + _mockCloudRunErrorParser!.Verify(x => x.PrintErrorToConsole(It.IsAny()), Times.Once); + _mockCloudRunErrorParser.Verify(x => x.TryPushMessageAndKey(It.IsAny(), ReporterConstants.s_cONFLICT_409_ERROR_MESSAGE_KEY), Times.Once); + } + + [Test] + public void PatchTestRunInfo_On403Forbidden_Throws() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(403); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + Assert.Throws(() => _serviceClient!.PatchTestRunInfo(run)); + + _mockCloudRunErrorParser!.Verify(x => x.PrintErrorToConsole(It.IsAny()), Times.Once); + _mockCloudRunErrorParser.Verify(x => x.TryPushMessageAndKey(It.IsAny(), ReporterConstants.s_fORBIDDEN_403_ERROR_MESSAGE_KEY), Times.Once); + } + + [Test] + public void PatchTestRunInfo_OnAPIError_ReturnsNull() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(401); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + TestRunDto? result = _serviceClient!.PatchTestRunInfo(run); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "401"), Times.Once); + } + + [Test] + public void PatchTestRunInfo_OnSuccessButNot200_ReturnsNull() + { + var run = new TestRunDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(run)); + + responseMock.SetupGet(r => r.Status).Returns(201); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PatchTestRunInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestRunDto? result = _serviceClient!.PatchTestRunInfo(run); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "201"), Times.Once); + } + + [Test] + public void PostTestRunShardInfo_ReturnsTestRunShardDto() + { + var shard = new TestRunShardDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(shard)); + + responseMock.SetupGet(r => r.Status).Returns(200); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PostTestRunShardInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestRunShardDto? result = _serviceClient!.PostTestRunShardInfo(shard); + + Assert.IsNotNull(result); + } + + [Test] + public void PostTestRunShardInfo_OnAPIError_ReturnsNull() + { + var shard = new TestRunShardDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(shard)); + + responseMock.SetupGet(r => r.Status).Returns(401); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PostTestRunShardInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + TestRunShardDto? result = _serviceClient!.PostTestRunShardInfo(shard); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "401"), Times.Once); + } + + [Test] + public void PostTestRunShardInfo_OnSuccessButNot200_ReturnsNull() + { + var shard = new TestRunShardDto(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(shard)); + + responseMock.SetupGet(r => r.Status).Returns(201); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.PostTestRunShardInfo( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestRunShardDto? result = _serviceClient!.PostTestRunShardInfo(shard); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "201"), Times.Once); + } + + [Test] + public void GetTestRunResultsUri_ReturnsTestResultsUri() + { + var testResultsUri = new TestResultsUri(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(testResultsUri)); + + responseMock.SetupGet(r => r.Status).Returns(200); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.GetTestRunResultsUri( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestResultsUri? result = _serviceClient!.GetTestRunResultsUri(); + + Assert.IsNotNull(result); + } + + [Test] + public void GetTestRunResultsUri_OnAPIError_ReturnsNull() + { + var testResultsUri = new TestResultsUri(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(testResultsUri)); + + responseMock.SetupGet(r => r.Status).Returns(401); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.GetTestRunResultsUri( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + TestResultsUri? result = _serviceClient!.GetTestRunResultsUri(); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "401"), Times.Once); + } + + [Test] + public void GetTestRunResultsUri_OnSuccessButNot200_ReturnsNull() + { + var testResultsUri = new TestResultsUri(); + var responseMock = new Mock(); + var responseContent = new BinaryData(JsonSerializer.Serialize(testResultsUri)); + + responseMock.SetupGet(r => r.Status).Returns(201); + responseMock.SetupGet(r => r.Content).Returns(responseContent!); + + _mockTestReportingClient!.Setup(x => x.GetTestRunResultsUri( + _cloudRunMetadata!.WorkspaceId!, + _cloudRunMetadata.RunId!, + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + TestResultsUri? result = _serviceClient!.GetTestRunResultsUri(); + + Assert.IsNull(result); + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "201"), Times.Once); + } + + [Test] + public void UploadBatchTestResults_Returns() + { + var responseMock = new Mock(); + + responseMock.SetupGet(r => r.Status).Returns(200); + + _mockTestReportingClient!.Setup(x => x.UploadBatchTestResults( + _cloudRunMetadata!.WorkspaceId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + _serviceClient!.UploadBatchTestResults(new UploadTestResultsRequest()); + } + + [Test] + public void UploadBatchTestResults_OnAPIError_Returns() + { + var responseMock = new Mock(); + + responseMock.SetupGet(r => r.Status).Returns(401); + + _mockTestReportingClient!.Setup(x => x.UploadBatchTestResults( + _cloudRunMetadata!.WorkspaceId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Throws(new RequestFailedException(responseMock.Object)); + + _serviceClient!.UploadBatchTestResults(new UploadTestResultsRequest()); + + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "401"), Times.Once); + } + + [Test] + public void UploadBatchTestResults_OnSuccessButNot200_Returns() + { + var responseMock = new Mock(); + + responseMock.SetupGet(r => r.Status).Returns(201); + + _mockTestReportingClient!.Setup(x => x.UploadBatchTestResults( + _cloudRunMetadata!.WorkspaceId!, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(responseMock.Object); + + _serviceClient!.UploadBatchTestResults(new UploadTestResultsRequest()); + + _mockCloudRunErrorParser!.Verify(x => x.TryPushMessageAndKey(It.IsAny(), "201"), Times.Once); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/CloudRunMetadataTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/CloudRunMetadataTests.cs new file mode 100644 index 0000000000000..dfd7b54818530 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/CloudRunMetadataTests.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Model +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class CloudRunMetadataTests + { + [Test] + public void TestPortalUrl() + { + var metadata = new CloudRunMetadata + { + WorkspaceId = "eastus_2e8c076a-b67c-4984-b861-8d22d7b525c6", + RunId = "#run456^1" + }; + + string portalUrl = metadata.PortalUrl!; + + string expectedPortalUrl = "https://playwright.microsoft.com/workspaces/eastus_2e8c076a-b67c-4984-b861-8d22d7b525c6/runs/%23run456%5E1"; + Assert.AreEqual(expectedPortalUrl, portalUrl); + } + + [Test] + public void TestEnableResultPublish() + { + var metadata = new CloudRunMetadata(); + + bool enableResultPublish = metadata.EnableResultPublish; + + Assert.IsTrue(enableResultPublish); + } + + [Test] + public void TestEnableGithubSummary() + { + var metadata = new CloudRunMetadata(); + + bool enableGithubSummary = metadata.EnableGithubSummary; + + Assert.IsTrue(enableGithubSummary); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/MPTResultTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/MPTResultTests.cs new file mode 100644 index 0000000000000..30cb8f3297fe6 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Model/MPTResultTests.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Model +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class MPTResultTests + { + [Test] + public void RawTestResult_Errors_Initialized() + { + var rawTestResult = new RawTestResult(); + Assert.AreEqual("[]", rawTestResult.errors); + Assert.AreEqual("[]", rawTestResult.stdOut); + Assert.AreEqual("[]", rawTestResult.stdErr); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/PlaywrightServiceTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/PlaywrightServiceTests.cs index fc2cc99c749e3..2f35beeeca466 100644 --- a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/PlaywrightServiceTests.cs +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/PlaywrightServiceTests.cs @@ -16,7 +16,6 @@ namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests; [TestFixture] -[Parallelizable(ParallelScope.Self)] public class PlaywrightServiceTests { private static string GetToken(Dictionary claims, DateTime? expires = null) diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/DataProcessorTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/DataProcessorTests.cs new file mode 100644 index 0000000000000..42b423e35767d --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/DataProcessorTests.cs @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Processor +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class DataProcessorTests + { + [Test] + public void GetTestRun_ReturnsTestRunDto() + { + var cloudRunMetadata = new CloudRunMetadata + { + WorkspaceId = "workspaceId", + RunId = "runId", + AccessTokenDetails = new() + { + oid = "oid", + userName = " userName " + } + }; + var cIInfo = new CIInfo + { + Branch = "branch_name", + Author = "author", + CommitId = "commitId", + RevisionUrl = "revisionUrl", + Provider = CIConstants.s_gITHUB_ACTIONS + }; + var dataProcessor = new DataProcessor(cloudRunMetadata, cIInfo); + + TestRunDto result = dataProcessor.GetTestRun(); + + Assert.IsNotNull(result); + Assert.IsInstanceOf(result); + + Assert.AreEqual(cloudRunMetadata.RunId, result.TestRunId); + Assert.IsNotNull(result.DisplayName); + Assert.IsNotNull(result.StartTime); + Assert.AreEqual(cloudRunMetadata.AccessTokenDetails.oid, result.CreatorId); + Assert.AreEqual("userName", result.CreatorName); + Assert.IsTrue(result.CloudReportingEnabled); + Assert.IsFalse(result.CloudRunEnabled); + Assert.IsNotNull(result.CiConfig); + Assert.AreEqual(cIInfo.Branch, result.CiConfig!.Branch); + Assert.AreEqual(cIInfo.Author, result.CiConfig!.Author); + Assert.AreEqual(cIInfo.CommitId, result.CiConfig!.CommitId); + Assert.AreEqual(cIInfo.RevisionUrl, result.CiConfig!.RevisionUrl); + Assert.AreEqual(cIInfo.Provider, result.CiConfig!.CiProviderName); + Assert.IsNotNull(result.TestRunConfig); + Assert.AreEqual(1, result.TestRunConfig!.Workers); + Assert.AreEqual("1.40", result.TestRunConfig!.PwVersion); + Assert.AreEqual(60000, result.TestRunConfig!.Timeout); + Assert.AreEqual("WebTest", result.TestRunConfig!.TestType); + Assert.AreEqual("CSHARP", result.TestRunConfig!.TestSdkLanguage); + Assert.IsNotNull(result.TestRunConfig!.TestFramework); + Assert.AreEqual("PLAYWRIGHT", result.TestRunConfig!.TestFramework!.Name); + Assert.AreEqual("NUNIT", result.TestRunConfig!.TestFramework!.RunnerName); + Assert.AreEqual("3.1", result.TestRunConfig!.TestFramework!.Version); + Assert.AreEqual("1.0.0-beta.1", result.TestRunConfig!.ReporterPackageVersion); + Assert.IsNotNull(result.TestRunConfig!.Shards); + Assert.AreEqual(1, result.TestRunConfig!.Shards!.Total); + } + + [Test] + public void GetTestRunShard_ReturnsTestRunShardDto() + { + var cloudRunMetadata = new CloudRunMetadata(); + var cIInfo = new CIInfo(); + var dataProcessor = new DataProcessor(cloudRunMetadata, cIInfo); + + TestRunShardDto result = dataProcessor.GetTestRunShard(); + + Assert.IsNotNull(result); + Assert.IsInstanceOf(result); + Assert.IsFalse(result.UploadCompleted); + Assert.AreEqual("1", result.ShardId); + Assert.IsNotNull(result.Summary); + Assert.AreEqual("RUNNING", result.Summary!.Status); + Assert.IsNotNull(result.Summary!.StartTime); + Assert.AreEqual(1, result.Workers); + } + + [Test] + public void GetTestCaseResultData_WithNullTestResult_ReturnsEmptyTestResults() + { + var cloudRunMetadata = new CloudRunMetadata(); + var cIInfo = new CIInfo(); + var dataProcessor = new DataProcessor(cloudRunMetadata, cIInfo); + TestResult? testResult = null; + + TestResults result = dataProcessor.GetTestCaseResultData(testResult); + + Assert.IsNotNull(result); + } + + [Test] + public void GetTestCaseResultData_WithNonNullTestResult_ReturnsTestResults() + { + var cloudRunMetadata = new CloudRunMetadata + { + WorkspaceId = "workspaceId", + RunId = "runId", + AccessTokenDetails = new() + { + oid = "oid", + userName = " userName " + } + }; + var cIInfo = new CIInfo + { + Branch = "branch_name", + Author = "author", + CommitId = "commitId", + RevisionUrl = "revisionUrl", + Provider = CIConstants.s_gITHUB_ACTIONS, + JobId = "jobId" + }; + var dataProcessor = new DataProcessor(cloudRunMetadata, cIInfo); + var testResult = new TestResult(new TestCase("Test.Reporting", new System.Uri("file:///test.cs"), "TestNamespace.TestClass")); + + TestResults result = dataProcessor.GetTestCaseResultData(testResult); + + Assert.IsNotNull(result); + Assert.IsEmpty(result.ArtifactsPath); + Assert.AreEqual(cloudRunMetadata.WorkspaceId, result.AccountId); + Assert.AreEqual(cloudRunMetadata.RunId, result.RunId); + Assert.IsNotNull(result.TestExecutionId); + Assert.IsNotNull(result.TestCombinationId); + Assert.IsNotNull(result.TestId); + Assert.AreEqual(testResult.TestCase.DisplayName, result.TestTitle); + Assert.AreEqual("Test", result.SuiteTitle); + Assert.AreEqual("Test", result.SuiteId); + Assert.AreEqual("TestNamespace.TestClass", result.FileName); + Assert.AreEqual(testResult.TestCase.LineNumber, result.LineNumber); + Assert.AreEqual(0, result.Retry); + Assert.IsNotNull(result.WebTestConfig); + Assert.AreEqual(cIInfo.JobId, result.WebTestConfig!.JobName); + Assert.AreEqual(ReporterUtils.GetCurrentOS(), result.WebTestConfig.Os); + Assert.IsNotNull(result.ResultsSummary); + Assert.AreEqual((long)testResult.Duration.TotalMilliseconds, result.ResultsSummary!.Duration); + Assert.AreEqual(testResult.StartTime.ToString("yyyy-MM-ddTHH:mm:ssZ"), result.ResultsSummary.StartTime); + Assert.AreEqual(TestCaseResultStatus.s_iNCONCLUSIVE, result.ResultsSummary.Status); + Assert.AreEqual(TestCaseResultStatus.s_iNCONCLUSIVE, result.Status); + } + + [Test] + public void GetRawResultObject_WithNullTestResult_ReturnsRawTestResultWithEmptyErrorsAndStdErr() + { + RawTestResult result = DataProcessor.GetRawResultObject(null); + + Assert.IsNotNull(result); + Assert.AreEqual("[]", result.errors); + Assert.AreEqual("[]", result.stdErr); + } + + [Test] + public void GetRawResultObject_WithNonNullTestResult_ReturnsRawTestResultWithErrorsAndStdErr() + { + var testResult = new TestResult(new TestCase("Test", new System.Uri("file:///test.cs"), "TestNamespace.TestClass")) + { + ErrorMessage = "An error occurred", + ErrorStackTrace = "Error stack trace" + }; + + RawTestResult result = DataProcessor.GetRawResultObject(testResult); + + Assert.IsNotNull(result); + Assert.AreEqual("[{\"message\":\"An error occurred\"}]", result.errors); + Assert.AreEqual("Error stack trace", result.stdErr); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/TestProcessorTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/TestProcessorTests.cs new file mode 100644 index 0000000000000..2ebe866cc5f2b --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Processor/TestProcessorTests.cs @@ -0,0 +1,584 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Interface; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Processor; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; +using Moq; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Processor +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class TestProcessorTests + { + private CIInfo _cIInfo = new(); + private CloudRunMetadata _cloudRunMetadata = new(); + + [SetUp] + public void Setup() + { + _cloudRunMetadata = new CloudRunMetadata + { + WorkspaceId = "workspaceId", + RunId = "runId", + AccessTokenDetails = new() + { + oid = "oid", + userName = " userName " + }, + EnableGithubSummary = false + }; + _cIInfo = new CIInfo + { + Branch = "branch_name", + Author = "author", + CommitId = "commitId", + RevisionUrl = "revisionUrl", + Provider = CIConstants.s_gITHUB_ACTIONS + }; + } + + [Test] + public void TestRunStartHandler_CreatesTestRunAndShardInfo() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testRunShardDto = new TestRunShardDto(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Returns(new TestRunDto()); + serviceClientMock.Setup(sc => sc.PostTestRunShardInfo(It.IsAny())).Returns(testRunShardDto); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Once); + Assert.AreEqual(testRunShardDto, testProcessor._testRunShard); + Assert.IsFalse(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_PatchTestRunReturnsNull_MarksTestExecutionAsFatal() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Returns((TestRunDto?)null); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Never); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsTrue(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_PatchTestRunThrowsError_MarksTestExecutionAsFatal() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Throws(new System.Exception()); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Never); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsTrue(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_PostTestRunShardReturnsNull_MarksTestExecutionAsFatal() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Returns(new TestRunDto()); + serviceClientMock.Setup(sc => sc.PostTestRunShardInfo(It.IsAny())).Returns((TestRunShardDto?)null); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Once); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsTrue(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_PostTestRunShardThrowsError_MarksTestExecutionAsFatal() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(sc => sc.PatchTestRunInfo(It.IsAny())).Returns(new TestRunDto()); + serviceClientMock.Setup(sc => sc.PostTestRunShardInfo(It.IsAny())).Throws(new System.Exception()); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Once); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Once); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Once); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Once); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsTrue(testProcessor.FatalTestExecution); + } + + [Test] + public void TestRunStartHandler_EnableResultPublishIsFalse_ShouldBeNoOp() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + _cloudRunMetadata.EnableResultPublish = false; + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var sources = new List { "source1", "source2" }; + var testRunCriteria = new TestRunCriteria(sources, 1); + var e = new TestRunStartEventArgs(testRunCriteria); + testProcessor.TestRunStartHandler(sender, e); + + dataProcessorMock.Verify(dp => dp.GetTestRun(), Times.Never); + dataProcessorMock.Verify(dp => dp.GetTestRunShard(), Times.Never); + serviceClientMock.Verify(sc => sc.PatchTestRunInfo(It.IsAny()), Times.Never); + serviceClientMock.Verify(sc => sc.PostTestRunShardInfo(It.IsAny()), Times.Never); + Assert.IsNull(testProcessor._testRunShard); + Assert.IsFalse(testProcessor.FatalTestExecution); + } + + [Test] + public void TestCaseResultHandler_TestPassed_AddsTestResultToTestResultsList() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_pASSED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(1, testProcessor.TestResults.Count); + Assert.AreEqual(testResults, testProcessor.TestResults[0]); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 1); + Assert.IsTrue(testProcessor.PassedTestCount == 1); + Assert.IsTrue(testProcessor.FailedTestCount == 0); + Assert.IsTrue(testProcessor.SkippedTestCount == 0); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestCaseResultHandler_TestFailed_AddsTestResultToTestResultsList() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_fAILED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(1, testProcessor.TestResults.Count); + Assert.AreEqual(testResults, testProcessor.TestResults[0]); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 1); + Assert.IsTrue(testProcessor.PassedTestCount == 0); + Assert.IsTrue(testProcessor.FailedTestCount == 1); + Assert.IsTrue(testProcessor.SkippedTestCount == 0); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestCaseResultHandler_TestSkipped_AddsTestResultToTestResultsList() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_sKIPPED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(1, testProcessor.TestResults.Count); + Assert.AreEqual(testResults, testProcessor.TestResults[0]); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 1); + Assert.IsTrue(testProcessor.PassedTestCount == 0); + Assert.IsTrue(testProcessor.FailedTestCount == 0); + Assert.IsTrue(testProcessor.SkippedTestCount == 1); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestCaseResultHandler_ShouldPassErrorMessageAndStackTraceForScalableErrorParsing() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_sKIPPED + }; + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult) + { + Result = + { + ErrorMessage = "Error message", + ErrorStackTrace = "Error stack trace" + } + }); + + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage("Error message"), Times.Once); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage("Error stack trace"), Times.Once); + } + + [Test] + public void TestCaseResultHandler_EnableResultPublishFalse_OnlyParseScalableErrorMessage() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + _cloudRunMetadata.EnableResultPublish = false; + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_sKIPPED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Returns(testResults); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(0, testProcessor.TestResults.Count); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 1); + Assert.IsTrue(testProcessor.PassedTestCount == 0); + Assert.IsTrue(testProcessor.FailedTestCount == 0); + Assert.IsTrue(testProcessor.SkippedTestCount == 0); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void TestCaseResultHandler_ExceptionThrown_ShouldBeNoOp() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + var testResults = new TestResults + { + Status = TestCaseResultStatus.s_sKIPPED + }; + + dataProcessorMock.Setup(dp => dp.GetTestCaseResultData(It.IsAny())).Throws(new System.Exception()); + + var testResult = new TestResult(new TestCase("Test", new System.Uri("file://test.cs"), "test-source")); + + testProcessor.TestCaseResultHandler(sender, new TestResultEventArgs(testResult)); + + Assert.AreEqual(0, testProcessor.TestResults.Count); + Assert.IsTrue(testProcessor.RawTestResultsMap.Keys.Count == 0); + Assert.IsTrue(testProcessor.PassedTestCount == 0); + Assert.IsTrue(testProcessor.FailedTestCount == 0); + Assert.IsTrue(testProcessor.SkippedTestCount == 0); + cloudRunErrorParserMock.Verify(c => c.HandleScalableRunErrorMessage(It.IsAny()), Times.Never); + } + + [Test] + public void TestRunCompleteHandler_UploadsTestResults() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + var testResults = new List + { + new() { Status = TestCaseResultStatus.s_pASSED }, + new() { Status = TestCaseResultStatus.s_fAILED }, + new() { Status = TestCaseResultStatus.s_sKIPPED } + }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + } + + [Test] + public void TestRunCompleteHandler_UploadsTestResultsThrows_IgnoresException() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(c => c.UploadBatchTestResults(It.IsAny())).Throws(new System.Exception()); + + var testResults = new List + { + new() { Status = TestCaseResultStatus.s_pASSED }, + new() { Status = TestCaseResultStatus.s_fAILED }, + new() { Status = TestCaseResultStatus.s_sKIPPED } + }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + } + + [Test] + public void TestRunCompleteHandler_PatchesTestRunShardInfo() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + testProcessor._testRunShard = new TestRunShardDto(); + testProcessor.TotalTestCount = 100; + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Once); + Assert.AreEqual("CLIENT_COMPLETE", testProcessor._testRunShard.Summary!.Status); + Assert.IsNotNull(testProcessor._testRunShard.Summary!.EndTime); + Assert.IsNotNull(testProcessor._testRunShard.Summary!.TotalTime); + Assert.AreEqual(100, testProcessor._testRunShard.Summary!.UploadMetadata!.NumTestResults); + Assert.AreEqual(0, testProcessor._testRunShard.Summary!.UploadMetadata!.NumTotalAttachments); + Assert.AreEqual(0, testProcessor._testRunShard.Summary!.UploadMetadata!.SizeTotalAttachments); + Assert.IsTrue(testProcessor._testRunShard.UploadCompleted); + } + + [Test] + public void TestRunCompleteHandler_PatchesTestRunShardInfoThrows_DisplaysInformationMessagesAndPortalUrl() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + serviceClientMock.Setup(c => c.PostTestRunShardInfo(It.IsAny())).Throws(new System.Exception()); + + testProcessor._testRunShard = new TestRunShardDto(); + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Once); + consoleWriterMock.Verify(c => c.WriteLine("\nTest Report: " + _cloudRunMetadata.PortalUrl), Times.Once); + cloudRunErrorParserMock.Verify(c => c.DisplayMessages(), Times.Exactly(1)); + } + + [Test] + public void TestRunCompleteHandler_DisplaysTestRunUrl() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + testProcessor._testRunShard = new TestRunShardDto(); + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Once); + consoleWriterMock.Verify(c => c.WriteLine("\nTest Report: " + _cloudRunMetadata.PortalUrl), Times.Once); + } + + [Test] + public void TestRunCompleteHandler_DisplaysMessagesOnEnd() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + testProcessor._testRunShard = new TestRunShardDto(); + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Once); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Once); + consoleWriterMock.Verify(c => c.WriteLine("\nTest Report: " + _cloudRunMetadata.PortalUrl), Times.Once); + cloudRunErrorParserMock.Verify(c => c.DisplayMessages(), Times.Exactly(1)); + } + + [Test] + public void TestRunCompleteHandler_FatalExecutionSetToTrue_DisplaysMessagesOnEnd() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + testProcessor.FatalTestExecution = true; + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Never); + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Never); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Never); + cloudRunErrorParserMock.Verify(c => c.DisplayMessages(), Times.Exactly(1)); + } + + [Test] + public void TestRunCompleteHandler_EnableResultPublishSetToFalse_DisplaysMessagesOnEnd() + { + var loggerMock = new Mock(); + var dataProcessorMock = new Mock(); + var cloudRunErrorParserMock = new Mock(); + var serviceClientMock = new Mock(); + var consoleWriterMock = new Mock(); + var testProcessor = new TestProcessor(_cloudRunMetadata, _cIInfo, loggerMock.Object, dataProcessorMock.Object, cloudRunErrorParserMock.Object, serviceClientMock.Object, consoleWriterMock.Object); + var sender = new object(); + + _cloudRunMetadata.EnableResultPublish = false; + + var testResults = new List { }; + testProcessor.TestResults = testResults; + testProcessor.TestRunCompleteHandler(sender, new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero)); + + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Never); + serviceClientMock.Verify(c => c.UploadBatchTestResults(It.IsAny()), Times.Never); + serviceClientMock.Verify(c => c.PostTestRunShardInfo(It.IsAny()), Times.Never); + cloudRunErrorParserMock.Verify(c => c.DisplayMessages(), Times.Exactly(1)); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/CiInfoProviderTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/CiInfoProviderTests.cs new file mode 100644 index 0000000000000..5c18556d0b396 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/CiInfoProviderTests.cs @@ -0,0 +1,257 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; + +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Utility +{ + [TestFixture] + [Parallelizable(ParallelScope.Self)] + public class CiInfoProviderTests + { + private Dictionary _originalEnvironmentVariables = new(); + + [OneTimeSetUp] + public void Init() + { + _originalEnvironmentVariables = Environment.GetEnvironmentVariables() + .Cast() + .ToDictionary(entry => (string)entry.Key, entry => (string)entry.Value!)!; + } + + [SetUp] + public void Setup() + { + foreach (KeyValuePair kvp in _originalEnvironmentVariables) + { + Environment.SetEnvironmentVariable(kvp.Key, kvp.Value); + } + + var keysToRemove = Environment.GetEnvironmentVariables() + .Cast() + .Select(entry => (string)entry.Key) + .Where(key => key.StartsWith("github", StringComparison.OrdinalIgnoreCase) || key.StartsWith("gh", StringComparison.OrdinalIgnoreCase) || key.StartsWith("ado", StringComparison.OrdinalIgnoreCase) || key.StartsWith("azure", StringComparison.OrdinalIgnoreCase) || key.StartsWith("tf", StringComparison.OrdinalIgnoreCase)) + .ToList(); + + foreach (var key in keysToRemove) + { + Environment.SetEnvironmentVariable(key, null); // Remove environment variable + } + } + + [Test] + public void GetCIProvider_GitHubActions_ReturnsGitHubActions() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", "true"); + + string ciProvider = CiInfoProvider.GetCIProvider(); + + Assert.AreEqual(CIConstants.s_gITHUB_ACTIONS, ciProvider); + } + + [Test] + public void GetCIProvider_AzureDevOps_ReturnsAzureDevOps() + { + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", "some_value"); + Environment.SetEnvironmentVariable("TF_BUILD", "some_value"); + + string ciProvider = CiInfoProvider.GetCIProvider(); + + Assert.AreEqual(CIConstants.s_aZURE_DEVOPS, ciProvider); + } + + [Test] + public void GetCIProvider_Default_ReturnsDefault() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", null); + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", null); + Environment.SetEnvironmentVariable("TF_BUILD", null); + + string ciProvider = CiInfoProvider.GetCIProvider(); + + Assert.AreEqual(CIConstants.s_dEFAULT, ciProvider); + } + [Test] + public void GetCIInfo_GitHubActions_ReturnsGitHubActionsCIInfo() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", "true"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("GITHUB_ACTOR", "actor"); + Environment.SetEnvironmentVariable("GITHUB_SHA", "commit_sha"); + Environment.SetEnvironmentVariable("GITHUB_SERVER_URL", "server_url"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY", "repository"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ID", "run_id"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ATTEMPT", "1"); + Environment.SetEnvironmentVariable("GITHUB_JOB", "job_id"); + Environment.SetEnvironmentVariable("GITHUB_REF_NAME", "refs/heads/branch_name"); + Environment.SetEnvironmentVariable("GITHUB_HEAD_REF", "refs/heads/head_branch_name"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_gITHUB_ACTIONS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("refs/heads/branch_name", ciInfo.Branch); + Assert.AreEqual("actor", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("server_url/repository/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("run_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("job_id", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_GitHubActions_ReturnsGitHubActionsCIInfo_WithRefPrefixWhenEventNameIsPullRequest() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", "true"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("GITHUB_ACTOR", "actor"); + Environment.SetEnvironmentVariable("GITHUB_SHA", "commit_sha"); + Environment.SetEnvironmentVariable("GITHUB_SERVER_URL", "server_url"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY", "repository"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ID", "run_id"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ATTEMPT", "1"); + Environment.SetEnvironmentVariable("GITHUB_JOB", "job_id"); + Environment.SetEnvironmentVariable("GITHUB_REF", "refs/heads/feature/branch_name"); + Environment.SetEnvironmentVariable("GITHUB_HEAD_REF", "refs/heads/head_branch_name"); + Environment.SetEnvironmentVariable("GITHUB_EVENT_NAME", "pull_request"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_gITHUB_ACTIONS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("refs/heads/head_branch_name", ciInfo.Branch); + Assert.AreEqual("actor", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("server_url/repository/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("run_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("job_id", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_GitHubActions_ReturnsGitHubActionsCIInfo_WithRefPrefixWhenEventNameIsPullRequestTarget() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", "true"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("GITHUB_ACTOR", "actor"); + Environment.SetEnvironmentVariable("GITHUB_SHA", "commit_sha"); + Environment.SetEnvironmentVariable("GITHUB_SERVER_URL", "server_url"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY", "repository"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ID", "run_id"); + Environment.SetEnvironmentVariable("GITHUB_RUN_ATTEMPT", "1"); + Environment.SetEnvironmentVariable("GITHUB_JOB", "job_id"); + Environment.SetEnvironmentVariable("GITHUB_REF", "refs/heads/feature/branch_name"); + Environment.SetEnvironmentVariable("GITHUB_HEAD_REF", "refs/heads/head_branch_name"); + Environment.SetEnvironmentVariable("GITHUB_EVENT_NAME", "pull_request_target"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_gITHUB_ACTIONS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("refs/heads/head_branch_name", ciInfo.Branch); + Assert.AreEqual("actor", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("server_url/repository/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("run_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("job_id", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_AzureDevOps_ReturnsAzureDevOpsCIInfo() + { + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", "some_value"); + Environment.SetEnvironmentVariable("TF_BUILD", "some_value"); + Environment.SetEnvironmentVariable("BUILD_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("BUILD_SOURCEBRANCH", "branch_name"); + Environment.SetEnvironmentVariable("BUILD_REQUESTEDFOR", "author"); + Environment.SetEnvironmentVariable("BUILD_SOURCEVERSION", "commit_sha"); + Environment.SetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", "collection_uri/"); + Environment.SetEnvironmentVariable("SYSTEM_TEAMPROJECT", "team_project"); + Environment.SetEnvironmentVariable("BUILD_REPOSITORY_NAME", "repository_name"); + Environment.SetEnvironmentVariable("RELEASE_ATTEMPTNUMBER", "1"); + Environment.SetEnvironmentVariable("SYSTEM_JOBATTEMPT", "2"); + Environment.SetEnvironmentVariable("RELEASE_DEPLOYMENTID", "deployment_id"); + Environment.SetEnvironmentVariable("SYSTEM_DEFINITIONID", "definition_id"); + Environment.SetEnvironmentVariable("SYSTEM_JOBID", "job_id"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_aZURE_DEVOPS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("branch_name", ciInfo.Branch); + Assert.AreEqual("author", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("collection_uri/team_project/_git/repository_name/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("definition_id-job_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("deployment_id", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_AzureDevOpsWithReleaseInfo_ReturnsAzureDevOpsCIInfo() + { + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", "some_value"); + Environment.SetEnvironmentVariable("TF_BUILD", "some_value"); + Environment.SetEnvironmentVariable("BUILD_REPOSITORY_ID", "repo_id"); + Environment.SetEnvironmentVariable("BUILD_SOURCEBRANCH", "branch_name"); + Environment.SetEnvironmentVariable("BUILD_REQUESTEDFOR", "author"); + Environment.SetEnvironmentVariable("BUILD_SOURCEVERSION", "commit_sha"); + Environment.SetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", "collection_uri/"); + Environment.SetEnvironmentVariable("SYSTEM_TEAMPROJECT", "team_project"); + Environment.SetEnvironmentVariable("BUILD_REPOSITORY_NAME", "repository_name"); + Environment.SetEnvironmentVariable("RELEASE_ATTEMPTNUMBER", "1"); + Environment.SetEnvironmentVariable("SYSTEM_JOBATTEMPT", "2"); + Environment.SetEnvironmentVariable("RELEASE_DEPLOYMENTID", "deployment_id"); + Environment.SetEnvironmentVariable("SYSTEM_DEFINITIONID", "definition_id"); + Environment.SetEnvironmentVariable("SYSTEM_JOBID", "job_id"); + Environment.SetEnvironmentVariable("RELEASE_DEFINITIONID", "release-def"); + Environment.SetEnvironmentVariable("RELEASE_DEPLOYMENTID", "release-dep"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_aZURE_DEVOPS, ciInfo.Provider); + Assert.AreEqual("repo_id", ciInfo.Repo); + Assert.AreEqual("branch_name", ciInfo.Branch); + Assert.AreEqual("author", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("collection_uri/team_project/_git/repository_name/commit/commit_sha", ciInfo.RevisionUrl); + Assert.AreEqual("release-def-release-dep", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("release-dep", ciInfo.JobId); + } + + [Test] + public void GetCIInfo_Default_ReturnsDefaultCIInfo() + { + Environment.SetEnvironmentVariable("GITHUB_ACTIONS", null); + Environment.SetEnvironmentVariable("AZURE_HTTP_USER_AGENT", null); + Environment.SetEnvironmentVariable("TF_BUILD", null); + Environment.SetEnvironmentVariable("REPO", "repo"); + Environment.SetEnvironmentVariable("BRANCH", "branch"); + Environment.SetEnvironmentVariable("AUTHOR", "author"); + Environment.SetEnvironmentVariable("COMMIT_ID", "commit_sha"); + Environment.SetEnvironmentVariable("REVISION_URL", "revision_url"); + Environment.SetEnvironmentVariable("RUN_ID", "run_id"); + Environment.SetEnvironmentVariable("RUN_ATTEMPT", "1"); + Environment.SetEnvironmentVariable("JOB_ID", "job_id"); + + CIInfo ciInfo = CiInfoProvider.GetCIInfo(); + + Assert.AreEqual(CIConstants.s_dEFAULT, ciInfo.Provider); + Assert.AreEqual("repo", ciInfo.Repo); + Assert.AreEqual("branch", ciInfo.Branch); + Assert.AreEqual("author", ciInfo.Author); + Assert.AreEqual("commit_sha", ciInfo.CommitId); + Assert.AreEqual("revision_url", ciInfo.RevisionUrl); + Assert.AreEqual("run_id", ciInfo.RunId); + Assert.AreEqual(1, ciInfo.RunAttempt); + Assert.AreEqual("job_id", ciInfo.JobId); + } + } +} diff --git a/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/ReporterUtilsTests.cs b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/ReporterUtilsTests.cs new file mode 100644 index 0000000000000..2456c11617d41 --- /dev/null +++ b/sdk/playwrighttesting/Azure.Developer.MicrosoftPlaywrightTesting.TestLogger/tests/Utility/ReporterUtilsTests.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Model; +using Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Utility; +using Microsoft.IdentityModel.JsonWebTokens; +using Microsoft.IdentityModel.Tokens; +using Moq; +namespace Azure.Developer.MicrosoftPlaywrightTesting.TestLogger.Tests.Utility; + +[TestFixture] +[Parallelizable(ParallelScope.Self)] +public class ReporterUtilsTests +{ + private static string GetToken(Dictionary claims, DateTime? expires = null) + { + var tokenHandler = new JsonWebTokenHandler(); + var token = tokenHandler.CreateToken(new SecurityTokenDescriptor + { + Claims = claims, + Expires = expires ?? DateTime.UtcNow.AddMinutes(10), + }); + return token!; + } + + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_ValidFutureSasUri_ReturnsTrue() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "https://example.com/sas?se=" + DateTime.UtcNow.AddMinutes(15).ToString("o"); // 15 minutes in the future + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsTrue(result); + } + + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_ExpiredSasUri_ReturnsFalse() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "https://example.com/sas?se=" + DateTime.UtcNow.AddMinutes(-5).ToString("o"); // 5 minutes in the past + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsFalse(result); + } + + [Test] + public void IsTimeGreaterThanCurrentPlus10Minutes_InvalidSasUri_ReturnsFalse() + { + var reporterUtils = new ReporterUtils(); + string sasUri = "not_a_valid_sas_uri"; // Invalid SAS URI + bool result = reporterUtils.IsTimeGreaterThanCurrentPlus10Minutes(sasUri); + Assert.IsFalse(result); + } + + [Test] + public void ParseWorkspaceIdFromAccessToken_CustomToken_ReturnsTokenDetails() + { + var reporterUtils = new ReporterUtils(); + var accessToken = GetToken(new Dictionary + { + { "aid", "custom_aid" }, + { "oid", "custom_oid" }, + { "id", "custom_id" }, + { "name", "custom_username" }, + }); + + TokenDetails result = reporterUtils.ParseWorkspaceIdFromAccessToken(null, accessToken); + + Assert.AreEqual("custom_aid", result.aid); + Assert.AreEqual("custom_oid", result.oid); + Assert.AreEqual("custom_id", result.id); + Assert.AreEqual("custom_username", result.userName); + } + + [Test] + public void ParseWorkspaceIdFromAccessToken_EntraToken_ReturnsTokenDetails() + { + var reporterUtils = new ReporterUtils(); + var jsonWebTokenHandler = new JsonWebTokenHandler(); + var accessToken = GetToken(new Dictionary + { + { "oid", "entra_oid" }, + { "name", "entra_username" }, + }); + + TokenDetails result = reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler, accessToken); + + Assert.AreEqual("entra_oid", result.oid); + Assert.AreEqual(string.Empty, result.id); + Assert.AreEqual("entra_username", result.userName); + } + + [Test] + public void ParseWorkspaceIdFromAccessToken_NullToken_ThrowsArgumentNullException() + { + var reporterUtils = new ReporterUtils(); + var jsonWebTokenHandler = new JsonWebTokenHandler(); + string? accessToken = null; + + Assert.Throws(() => reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler, accessToken)); + } + + [Test] + public void ParseWorkspaceIdFromAccessToken_EmptyToken_ThrowsArgumentNullException() + { + var reporterUtils = new ReporterUtils(); + var jsonWebTokenHandler = new JsonWebTokenHandler(); + string accessToken = string.Empty; + + Assert.Throws(() => reporterUtils.ParseWorkspaceIdFromAccessToken(jsonWebTokenHandler, accessToken)); + } + [Test] + public void GetRunId_DefaultProvider_ReturnsNewGuid() + { + var cIInfo = new CIInfo { Provider = CIConstants.s_dEFAULT }; + var result = ReporterUtils.GetRunId(cIInfo); + Assert.IsNotNull(result); + Assert.IsTrue(Guid.TryParse(result, out _)); + } + + [Test] + public void GetRunId_NonDefaultProvider_ReturnsSha1Hash() + { + var cIInfo = new CIInfo { Provider = "NonDefaultProvider", Repo = "Repo", RunId = "RunId", RunAttempt = 1 }; + var expectedRunIdBeforeHash = $"{cIInfo.Provider}-{cIInfo.Repo}-{cIInfo.RunId}-{cIInfo.RunAttempt}"; + var result = ReporterUtils.GetRunId(cIInfo); + Assert.IsNotNull(result); + Assert.AreEqual(40, result.Length); + Assert.AreEqual(ReporterUtils.CalculateSha1Hash(expectedRunIdBeforeHash), result); + } + + [Test] + public void GetRunName_GitHubActionsPullRequest_ReturnsExpectedValue() + { + var ciInfo = new CIInfo { Provider = CIConstants.s_gITHUB_ACTIONS }; + Environment.SetEnvironmentVariable("GITHUB_EVENT_NAME", "pull_request"); + Environment.SetEnvironmentVariable("GITHUB_REF_NAME", "543/refs/merge"); + Environment.SetEnvironmentVariable("GITHUB_REPOSITORY", "owner/repo"); + + var result = ReporterUtils.GetRunName(ciInfo); + + var expected = "PR# 543 on Repo: owner/repo (owner/repo/pull/543)"; + Assert.AreEqual(expected, result); + } +}