diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/MultiTestRunFinalization/MultiTestRunFinalizationManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/MultiTestRunFinalization/MultiTestRunFinalizationManager.cs index 4bb4818ee9..2e0a197bd5 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/MultiTestRunFinalization/MultiTestRunFinalizationManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/MultiTestRunFinalization/MultiTestRunFinalizationManager.cs @@ -99,7 +99,7 @@ private async Task> InternalFinalizeMultiTestRunAsync( finally { stopwatch.Stop(); - requestData.MetricsCollection.Metrics.Add(TelemetryDataConstants.TimeTakenInSecForFinalization, stopwatch.Elapsed.TotalSeconds); + requestData.MetricsCollection.Add(TelemetryDataConstants.TimeTakenInSecForFinalization, stopwatch.Elapsed.TotalSeconds); } } @@ -137,7 +137,7 @@ private Collection FinalizeOperation(IRequestData requestData, Co eventHandler?.HandleMultiTestRunFinalizationComplete(attachments); testPlatformEventSource.MultiTestRunFinalizationStop(attachments.Count); requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsAfterFinalization, attachments.Count); - requestData.MetricsCollection.Add(TelemetryDataConstants.FinalizationState, attachments.Count); + requestData.MetricsCollection.Add(TelemetryDataConstants.FinalizationState, finalizationState); return attachments; } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/MultiTestRunFinalization/MultiTestRunFinalizationManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/MultiTestRunFinalization/MultiTestRunFinalizationManagerTests.cs index e3d0f106f9..ba4f983854 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/MultiTestRunFinalization/MultiTestRunFinalizationManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/MultiTestRunFinalization/MultiTestRunFinalizationManagerTests.cs @@ -8,7 +8,7 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.MultiTestRunFinalizat using System.Linq; using System.Threading; using System.Threading.Tasks; - + using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.MultiTestRunFinalization; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -16,7 +16,7 @@ namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.MultiTestRunFinalizat using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; - using Moq; + using Moq; [TestClass] public class MultiTestRunFinalizationManagerTests @@ -25,6 +25,8 @@ public class MultiTestRunFinalizationManagerTests private const string uri2 = "datacollector://microsoft/some2/2.0"; private const string uri3 = "datacollector://microsoft/some3/2.0"; + private readonly Mock mockRequestData; + private readonly Mock mockMetricsCollection; private readonly Mock mockEventSource; private readonly Mock mockAttachmentHandler1; private readonly Mock mockAttachmentHandler2; @@ -34,6 +36,10 @@ public class MultiTestRunFinalizationManagerTests public MultiTestRunFinalizationManagerTests() { + mockRequestData = new Mock(); + mockMetricsCollection = new Mock(); + mockRequestData.Setup(r => r.MetricsCollection).Returns(mockMetricsCollection.Object); + mockEventSource = new Mock(); mockAttachmentHandler1 = new Mock(); mockAttachmentHandler2 = new Mock(); @@ -48,23 +54,25 @@ public MultiTestRunFinalizationManagerTests() } [TestMethod] - public async Task FinalizeMultiTestRunAsync_ShouldReturnNoAttachmentsThroughEventsHandler_IfNoAttachmentsOnInput() + public async Task FinalizeMultiTestRunAsync_ShouldReturnInitialAttachmentsThroughEventsHandler_IfNoAttachmentsOnInput() { // arrange List inputAttachments = new List(); // act - await manager.FinalizeMultiTestRunAsync(inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); // assert mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(It.Is>(c => c.Count == 0))); mockEventsHandler.Verify(h => h.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never); mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(0)); mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(0)); - mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler2.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.GetExtensionUri(), Times.Never); + mockAttachmentHandler2.Verify(h => h.GetExtensionUri(), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(0, 0); } [TestMethod] @@ -74,16 +82,16 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNoAttachments_IfNoAttach List inputAttachments = new List(); // act - var result = await manager.FinalizeMultiTestRunAsync(inputAttachments, cancellationTokenSource.Token); + var result = await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); // assert Assert.AreEqual(0, result.Count); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(0)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(0)); - mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler2.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.GetExtensionUri(), Times.Never); + mockAttachmentHandler2.Verify(h => h.GetExtensionUri(), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(0, 0); } [TestMethod] @@ -96,17 +104,17 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturn1NotProcessedAttachmentT }; // act - await manager.FinalizeMultiTestRunAsync(inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); // assert mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])))); mockEventsHandler.Verify(h => h.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(1)); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1); } [TestMethod] @@ -119,17 +127,17 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturn1NotProcessedAttachment_ }; // act - var result = await manager.FinalizeMultiTestRunAsync(inputAttachments, cancellationTokenSource.Token); + var result = await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); // assert Assert.AreEqual(1, result.Count); Assert.IsTrue(result.Contains(inputAttachments[0])); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(1)); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1); } [TestMethod] @@ -146,20 +154,20 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturn1ProcessedAttachmentThro new AttachmentSet(new Uri(uri1), "uri1_output") }; - mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Returns(outputAttachments); + mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Returns(outputAttachments); // act - await manager.FinalizeMultiTestRunAsync(inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); // assert mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(It.Is>(c => c.Count == 1 && c.Contains(outputAttachments[0])))); mockEventsHandler.Verify(h => h.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(1)); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1); } [TestMethod] @@ -176,10 +184,10 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturn1ProcessedAttachment_IfR new AttachmentSet(new Uri(uri1), "uri1_output") }; - mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Returns(outputAttachments); + mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Returns(outputAttachments); // act - var result = await manager.FinalizeMultiTestRunAsync(inputAttachments, cancellationTokenSource.Token); + var result = await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); // assert Assert.AreEqual(1, result.Count); @@ -188,12 +196,14 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturn1ProcessedAttachment_IfR mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(1)); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1); } [TestMethod] - public async Task FinalizeMultiTestRunAsync_ShouldReturnNullThroughEventsHandler_IfRelatedAttachmentOnInputButHandlerThrowsException() + public async Task FinalizeMultiTestRunAsync_ShouldReturnInitialAttachmentsThroughEventsHandler_IfRelatedAttachmentOnInputButHandlerThrowsException() { // arrange List inputAttachments = new List @@ -201,24 +211,26 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNullThroughEventsHandler new AttachmentSet(new Uri(uri1), "uri1_input") }; - mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Throws(new Exception("exception message")); + mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Throws(new Exception("exception message")); // act - await manager.FinalizeMultiTestRunAsync(inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); // assert - mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(null)); + mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(It.Is>( + c => c.Count == 1 && + c.Contains(inputAttachments[0])))); mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Error, "exception message"), Times.Once); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(0)); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri(), Times.Never); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1, "Failed"); } [TestMethod] - public async Task FinalizeMultiTestRunAsync_ShouldReturnNull_IfRelatedAttachmentOnInputButHandlerThrowsException() + public async Task FinalizeMultiTestRunAsync_ShouldReturnInitialAttachments_IfRelatedAttachmentOnInputButHandlerThrowsException() { // arrange List inputAttachments = new List @@ -226,23 +238,24 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNull_IfRelatedAttachment new AttachmentSet(new Uri(uri1), "uri1_input") }; - mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Throws(new Exception("exception message")); + mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Throws(new Exception("exception message")); // act - var result = await manager.FinalizeMultiTestRunAsync(inputAttachments, cancellationTokenSource.Token); + var result = await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); // assert - Assert.IsNull(result); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(0)); + Assert.AreEqual(1, result.Count); + Assert.IsTrue(result.Contains(inputAttachments[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri(), Times.Never); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1, "Failed"); } [TestMethod] - public async Task FinalizeMultiTestRunAsync_ShouldReturnNullThroughEventsHandler_IfOperationIsCancelled() + public async Task FinalizeMultiTestRunAsync_ShouldReturnInitialAttachmentsThroughEventsHandler_IfOperationIsCancelled() { // arrange cancellationTokenSource.Cancel(); @@ -252,21 +265,22 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNullThroughEventsHandler }; // act - await manager.FinalizeMultiTestRunAsync(inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); // assert - mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(null)); - mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Error, "The operation was canceled."), Times.Once); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(0)); + mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(It.Is>( + c => c.Count == 1 && + c.Contains(inputAttachments[0])))); mockAttachmentHandler1.Verify(h => h.GetExtensionUri(), Times.Never); mockAttachmentHandler2.Verify(h => h.GetExtensionUri(), Times.Never); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1, "Canceled"); } [TestMethod] - public async Task FinalizeMultiTestRunAsync_ShouldReturnNull_IfOperationIsCancelled() + public async Task FinalizeMultiTestRunAsync_ShouldReturnInitialAttachments_IfOperationIsCancelled() { // arrange cancellationTokenSource.Cancel(); @@ -276,16 +290,17 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNull_IfOperationIsCancel }; // act - var result = await manager.FinalizeMultiTestRunAsync(inputAttachments, cancellationTokenSource.Token); + var result = await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); // assert - Assert.IsNull(result); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(0)); + Assert.AreEqual(1, result.Count); + Assert.IsTrue(result.Contains(inputAttachments[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUri(), Times.Never); mockAttachmentHandler2.Verify(h => h.GetExtensionUri(), Times.Never); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1, "Canceled"); } [TestMethod] @@ -311,11 +326,11 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnProcessedAttachmentsThro new AttachmentSet(new Uri(uri2), "uri2_output") }; - mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Returns(outputAttachmentsForHandler1); - mockAttachmentHandler2.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Returns(outputAttachmentsForHandler2); + mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Returns(outputAttachmentsForHandler1); + mockAttachmentHandler2.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Returns(outputAttachmentsForHandler2); // act - await manager.FinalizeMultiTestRunAsync(inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); // assert mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(It.Is>( @@ -324,12 +339,12 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnProcessedAttachmentsThro c.Contains(outputAttachmentsForHandler1.First()) && c.Contains(outputAttachmentsForHandler2.First())))); mockEventsHandler.Verify(h => h.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(5)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(3)); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[0]) && c.Contains(inputAttachments[1])), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[2]) && c.Contains(inputAttachments[3])), cancellationTokenSource.Token)); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[0]) && c.Contains(inputAttachments[1])), It.IsAny>(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[2]) && c.Contains(inputAttachments[3])), It.IsAny>(), cancellationTokenSource.Token)); + + VerifyMetrics(5, 3); } [TestMethod] @@ -355,27 +370,27 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnProcessedAttachments_IfR new AttachmentSet(new Uri(uri2), "uri2_output") }; - mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Returns(outputAttachmentsForHandler1); - mockAttachmentHandler2.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Returns(outputAttachmentsForHandler2); + mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Returns(outputAttachmentsForHandler1); + mockAttachmentHandler2.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Returns(outputAttachmentsForHandler2); // act - var result = await manager.FinalizeMultiTestRunAsync(inputAttachments, cancellationTokenSource.Token); + var result = await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); // assert Assert.AreEqual(3, result.Count); Assert.IsTrue(result.Contains(inputAttachments[4])); Assert.IsTrue(result.Contains(outputAttachmentsForHandler1[0])); Assert.IsTrue(result.Contains(outputAttachmentsForHandler2[0])); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(5)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(3)); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri()); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[0]) && c.Contains(inputAttachments[1])), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[2]) && c.Contains(inputAttachments[3])), cancellationTokenSource.Token)); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[0]) && c.Contains(inputAttachments[1])), It.IsAny>(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 2 && c.Contains(inputAttachments[2]) && c.Contains(inputAttachments[3])), It.IsAny>(), cancellationTokenSource.Token)); + + VerifyMetrics(5, 3); } [TestMethod] - public async Task FinalizeMultiTestRunAsync_ShouldReturnNullThroughEventsHandler_IfOperationCancelled() + public async Task FinalizeMultiTestRunAsync_ShouldReturnInitialAttachmentsThroughEventsHandler_IfOperationCancelled() { // arrange List inputAttachments = new List @@ -390,7 +405,7 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNullThroughEventsHandler var innerTaskCompletionSource = new TaskCompletionSource(); - mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Returns((ICollection i1, CancellationToken cancellation) => + mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Returns((ICollection i1, IProgress progress, CancellationToken cancellation) => { try { @@ -412,28 +427,30 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNullThroughEventsHandler { innerTaskCompletionSource.TrySetResult(null); } - + return outputAttachments; }); // act - await manager.FinalizeMultiTestRunAsync(inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); + await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, mockEventsHandler.Object, cancellationTokenSource.Token); Console.WriteLine("Finalization done"); await innerTaskCompletionSource.Task; // assert - mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(null)); + mockEventsHandler.Verify(h => h.HandleMultiTestRunFinalizationComplete(It.Is>( + c => c.Count == 1 && + c.Contains(inputAttachments[0])))); mockEventsHandler.Verify(h => h.HandleLogMessage(TestMessageLevel.Informational, "Finalization was cancelled.")); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(0)); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri(), Times.Never); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1, "Canceled"); } [TestMethod] - public async Task FinalizeMultiTestRunAsync_ShouldReturnNull_IfOperationCancelled() + public async Task FinalizeMultiTestRunAsync_ShouldReturnInitialAttachments_IfOperationCancelled() { // arrange List inputAttachments = new List @@ -448,7 +465,7 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNull_IfOperationCancelle var innerTaskCompletionSource = new TaskCompletionSource(); - mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny())).Returns((ICollection i1, CancellationToken cancellation) => + mockAttachmentHandler1.Setup(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny())).Returns((ICollection i1, IProgress p, CancellationToken cancellation) => { try { @@ -475,18 +492,31 @@ public async Task FinalizeMultiTestRunAsync_ShouldReturnNull_IfOperationCancelle }); // act - var result = await manager.FinalizeMultiTestRunAsync(inputAttachments, cancellationTokenSource.Token); + var result = await manager.FinalizeMultiTestRunAsync(mockRequestData.Object, inputAttachments, cancellationTokenSource.Token); Console.WriteLine("Finalization done"); await innerTaskCompletionSource.Task; // assert - Assert.IsNull(result); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(1)); - mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(0)); + Assert.IsNotNull(result); + Assert.AreEqual(1, result.Count); + Assert.IsTrue(result.Contains(inputAttachments[0])); mockAttachmentHandler1.Verify(h => h.GetExtensionUri()); mockAttachmentHandler2.Verify(h => h.GetExtensionUri(), Times.Never); - mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), cancellationTokenSource.Token)); - mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny()), Times.Never); + mockAttachmentHandler1.Verify(h => h.HandleDataCollectionAttachmentSets(It.Is>(c => c.Count == 1 && c.Contains(inputAttachments[0])), It.IsAny>(), cancellationTokenSource.Token)); + mockAttachmentHandler2.Verify(h => h.HandleDataCollectionAttachmentSets(It.IsAny>(), It.IsAny>(), It.IsAny()), Times.Never); + + VerifyMetrics(1, 1, "Canceled"); + } + + private void VerifyMetrics(int inputCount, int outputCount, string status = "Completed") + { + mockEventSource.Verify(s => s.MultiTestRunFinalizationStart(inputCount)); + mockEventSource.Verify(s => s.MultiTestRunFinalizationStop(outputCount)); + + mockMetricsCollection.Verify(m => m.Add(TelemetryDataConstants.NumberOfAttachmentsSentForFinalization, inputCount)); + mockMetricsCollection.Verify(m => m.Add(TelemetryDataConstants.NumberOfAttachmentsAfterFinalization, outputCount)); + mockMetricsCollection.Verify(m => m.Add(TelemetryDataConstants.FinalizationState, status)); + mockMetricsCollection.Verify(m => m.Add(TelemetryDataConstants.TimeTakenInSecForFinalization, It.IsAny())); } } } diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index fcb2619e0a..768263b8a1 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -2183,7 +2183,11 @@ public void FinalizeMultiTestRunShouldSucceedWithTelemetryEnabled() var mockEventsHandler = new Mock(); mockFinalizationManager .Setup(m => m.FinalizeMultiTestRunAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) - .Returns(Task.FromResult(true)); + .Returns((IRequestData r, ICollection a, IMultiTestRunFinalizationEventsHandler h, CancellationToken token) => Task.Run(() => + { + r.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsSentForFinalization, 5); + r.MetricsCollection.Add(TelemetryDataConstants.NumberOfAttachmentsAfterFinalization, 1); + })); var payload = new MultiTestRunFinalizationPayload() { @@ -2196,6 +2200,11 @@ public void FinalizeMultiTestRunShouldSucceedWithTelemetryEnabled() mockFinalizationManager.Verify(m => m.FinalizeMultiTestRunAsync(It.Is(r => r.IsTelemetryOptedIn), payload.Attachments, mockEventsHandler.Object, It.IsAny())); mockTestPlatformEventSource.Verify(es => es.MultiTestRunFinalizationRequestStart()); mockTestPlatformEventSource.Verify(es => es.MultiTestRunFinalizationRequestStop()); + + mockMetricsPublisher.Verify(p => p.PublishMetrics(TelemetryDataConstants.TestFinalizationCompleteEvent, + It.Is>(m => m.Count == 2 && + m.ContainsKey(TelemetryDataConstants.NumberOfAttachmentsSentForFinalization) && (int)m[TelemetryDataConstants.NumberOfAttachmentsSentForFinalization] == 5 && + m.ContainsKey(TelemetryDataConstants.NumberOfAttachmentsAfterFinalization) && (int)m[TelemetryDataConstants.NumberOfAttachmentsAfterFinalization] == 1))); } [TestMethod] @@ -2225,7 +2234,7 @@ public async Task CancelMultiTestRunFinalizationShouldSucceedIfRequestInProgress var mockEventsHandler = new Mock(); mockFinalizationManager .Setup(m => m.FinalizeMultiTestRunAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) - .Returns((ICollection a, IMultiTestRunFinalizationEventsHandler h, CancellationToken token) => Task.Run(() => + .Returns((IRequestData r, ICollection a, IMultiTestRunFinalizationEventsHandler h, CancellationToken token) => Task.Run(() => { int i = 0; while (!token.IsCancellationRequested) @@ -2234,11 +2243,14 @@ public async Task CancelMultiTestRunFinalizationShouldSucceedIfRequestInProgress Console.WriteLine($"Iteration {i}"); Task.Delay(5).Wait(); } + + r.MetricsCollection.Add(TelemetryDataConstants.FinalizationState, "Canceled"); })); var payload = new MultiTestRunFinalizationPayload() { - Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") } + Attachments = new List { new AttachmentSet(new Uri("http://www.bing.com"), "out") }, + CollectMetrics = true }; Task task = Task.Run(() => testRequestManager.FinalizeMultiTestRun(payload, mockEventsHandler.Object, this.protocolConfig)); @@ -2250,6 +2262,9 @@ public async Task CancelMultiTestRunFinalizationShouldSucceedIfRequestInProgress mockFinalizationManager.Verify(m => m.FinalizeMultiTestRunAsync(It.IsAny(), payload.Attachments, mockEventsHandler.Object, It.IsAny())); mockTestPlatformEventSource.Verify(es => es.MultiTestRunFinalizationRequestStart()); mockTestPlatformEventSource.Verify(es => es.MultiTestRunFinalizationRequestStop()); + + mockMetricsPublisher.Verify(p => p.PublishMetrics(TelemetryDataConstants.TestFinalizationCompleteEvent, + It.Is>(m => m.Count == 1 && m.ContainsKey(TelemetryDataConstants.FinalizationState) && (string)m[TelemetryDataConstants.FinalizationState] == "Canceled"))); } [TestMethod]