Add support for VSCode problem-tracking for failed assertions

README.md

@@ -511,7 +511,6 @@ ShowFullErrors         : Show full errors including Pester internal stack. (Fals
 WriteDebugMessages     : Write Debug messages to screen. (False, default: False)
 WriteDebugMessagesFrom : Write Debug messages from a given source, WriteDebugMessages must be set to true for this to work. You can use like wildcards to get messages from multiple sources, as well as * to get everything. (*, default: *)
 ShowNavigationMarkers  : Write paths after every block and test, for easy navigation in VSCode. (False, default: False)
-WriteVSCodeMarker      : Write VSCode marker for better integration with VSCode. (False, default: False)
 ```
 
 The configuration object can be constructed either via the Default static property or by casting a hashtable to it. You can also cast a hashtable to any of its sections. Here are three different ways to the same goal:

src/Pester.ps1

@@ -428,7 +428,6 @@ function Invoke-Pester {
     Debug.WriteDebugMessages - Write Debug messages to screen.
     Debug.WriteDebugMessagesFrom - Write Debug messages from a given source, WriteDebugMessages must be set to true for this to work. You can use like wildcards to get messages from multiple sources, as well as * to get everything.
         Available options: "Discovery", "Skip", "Filter", "Mock", "CodeCoverage"
-    Debug.WriteVSCodeMarker - Write VSCode marker for better integration with VSCode.
 
     .PARAMETER EnableExit
     (Deprecated v4)

src/csharp/Pester/Configuration.cs

@@ -462,7 +462,6 @@ public static DebugConfiguration ShallowClone(DebugConfiguration configuration)
             WriteDebugMessages = new BoolOption("Write Debug messages to screen.", false);
             WriteDebugMessagesFrom = new StringArrayOption("Write Debug messages from a given source, WriteDebugMessages must be set to true for this to work. You can use like wildcards to get messages from multiple sources, as well as * to get everything.", new string[] { "Discovery", "Skip", "Filter", "Mock", "CodeCoverage" });
             ShowNavigationMarkers = new BoolOption("Write paths after every block and test, for easy navigation in VSCode.", false);
-            WriteVSCodeMarker = new BoolOption("Write VSCode marker for better integration with VSCode.", false);
             ReturnRawResultObject = new BoolOption("Returns unfiltered result object, this is for development only. Do not rely on this object for additional properties, non-public properties will be renamed without previous notice.", false);
         }
 
@@ -474,7 +473,6 @@ public DebugConfiguration(IDictionary configuration) : this()
                 WriteDebugMessages = configuration.GetValueOrNull<bool>("WriteDebugMessages") ?? WriteDebugMessages;
                 WriteDebugMessagesFrom = configuration.GetArrayOrNull<string>("WriteDebugMessagesFrom") ?? WriteDebugMessagesFrom;
                 ShowNavigationMarkers = configuration.GetValueOrNull<bool>("ShowNavigationMarkers") ?? ShowNavigationMarkers;
-                WriteVSCodeMarker = configuration.GetValueOrNull<bool>("WriteVSCodeMarker") ?? WriteVSCodeMarker;
                 ReturnRawResultObject = configuration.GetValueOrNull<bool>("ReturnRawResultObject") ?? ReturnRawResultObject;
             }
         }
@@ -483,7 +481,6 @@ public DebugConfiguration(IDictionary configuration) : this()
         private BoolOption _writeDebugMessages;
         private StringArrayOption _writeDebugMessagesFrom;
         private BoolOption _showNavigationMarkers;
-        private BoolOption _writeVsCodeMarker;
         private BoolOption _returnRawResultObject;
 
         public BoolOption ShowFullErrors
@@ -550,22 +547,6 @@ public BoolOption ShowNavigationMarkers
             }
         }
 
-        public BoolOption WriteVSCodeMarker
-        {
-            get { return _writeVsCodeMarker; }
-            set
-            {
-                if (_writeVsCodeMarker == null)
-                {
-                    _writeVsCodeMarker = value;
-                }
-                else
-                {
-                    _writeVsCodeMarker = new BoolOption(_writeVsCodeMarker, value.Value);
-                }
-            }
-        }
-
         public BoolOption ReturnRawResultObject
         {
             get { return _returnRawResultObject; }

src/functions/Output.ps1

@@ -586,18 +586,24 @@ function Get-WriteScreenPlugin ($Verbosity) {
             }
 
             Failed {
-                & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$margin[-] $out" -NoNewLine
-                & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailTime " $humanTime"
-
-                # review how we should write errors for VS code based on https://github.com/PowerShell/vscode-powershell/pull/2447
-                # and use the env variable mentioned there
-                # if($PesterPreference.Debug.WriteVSCodeMarker.Value) {
-                #     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail $($_test.ErrorRecord[-1].DisplayStackTrace -replace '(?m)^',$error_margin)
-                #     & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail $($_test.ErrorRecord[-1].DisplayErrorMessage -replace '(?m)^',$error_margin)
-                # }
-                # else {
+                # If VSCode and not Integrated Terminal (usually a test-task), output Pester 4-format to match 'pester'-problemMatcher in VSCode.
+                if($env:TERM_PROGRAM -eq 'vscode' -and -not $psEditor) {
+
+                    # Loop to generate problem for every failed assertion per test (when $PesterPreference.Should.ErrorAction.Value = "Continue")
+                    foreach($e in $_test.ErrorRecord) {
+                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$margin[-] $out" -NoNewLine
+                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailTime " $humanTime"
+
+                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail $($e.DisplayStackTrace -replace '(?m)^',$error_margin)
+                        & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail $($e.DisplayErrorMessage -replace '(?m)^',$error_margin)
+                    }
+
+                } else {
+                    & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.Fail "$margin[-] $out" -NoNewLine
+                    & $SafeCommands['Write-Host'] -ForegroundColor $ReportTheme.FailTime " $humanTime"
+
                     Write-ErrorToScreen $_test.ErrorRecord -ErrorMargin $error_margin
-                # }
+                }
                 break
             }
 

tst/Pester.RSpec.Configuration.ts.ps1

@@ -156,10 +156,6 @@ i -PassThru:$PassThru {
         t "Debug.ShowNavigationMarkers is `$false" {
             [PesterConfiguration]::Default.Debug.ShowNavigationMarkers.Value | Verify-False
         }
-
-        t "Debug.WriteVSCodeMarker is `$false" {
-            [PesterConfiguration]::Default.Debug.WriteVSCodeMarker.Value | Verify-False
-        }
     }
 
     b "Assignment" {

tst/Pester.RSpec.Output.ts.ps1

@@ -0,0 +1,101 @@
+param ([switch] $PassThru)
+
+Get-Module Pester.Runtime, Pester.Utility, P, Pester, Axiom, Stack | Remove-Module
+
+Import-Module $PSScriptRoot\p.psm1 -DisableNameChecking
+Import-Module $PSScriptRoot\axiom\Axiom.psm1 -DisableNameChecking
+
+& "$PSScriptRoot\..\build.ps1"
+Import-Module $PSScriptRoot\..\bin\Pester.psd1
+
+$global:PesterPreference = @{
+    Debug = @{
+        ShowFullErrors         = $true
+        WriteDebugMessages     = $false
+        WriteDebugMessagesFrom = "Mock"
+        ReturnRawResultObject  = $true
+    }
+    Output = @{
+        Verbosity = "None"
+    }
+}
+$PSDefaultParameterValues = @{}
+
+function Invoke-PesterInProcess ([ScriptBlock] $ScriptBlock, [ScriptBlock] $Setup) {
+    # get the path of the currently loaded Pester to re-import it in the child process
+    $pesterPath = Get-Module Pester | Select-Object -ExpandProperty Path
+    $powershell = Get-Process -Id $pid | Select-Object -ExpandProperty Path
+    # run the test in a separate process to be able to grab all the output
+    $command = {
+        param ($PesterPath, [ScriptBlock] $ScriptBlock, [ScriptBlock] $Setup)
+        Import-Module $PesterPath
+
+        # Modify environment
+        . $Setup
+
+        $container = New-TestContainer -ScriptBlock $ScriptBlock
+        Invoke-Pester -Container $container
+    }.ToString()
+
+    $cmd = "& { $command } -PesterPath ""$PesterPath"" -ScriptBlock { $ScriptBlock } -Setup { $Setup }"
+    & $powershell -NoProfile -ExecutionPolicy Bypass -Command $cmd
+}
+
+# VSCode-powershell Problem Matcher pattern
+# Storing the original pattern below for easier comparison and maintenance
+$titlePattern = ('^\\s*(?:\\[-\\]\\s+)(.*?)(?:\\s+\\d+\\.?\\d*\\s*m?s)(?:\\s+\\(\\d+\\.?\\d*m?s\\|\\d+\\.?\\d*m?s\\))?\\s*$' -replace '\\\\', '\')
+$atPattern = ('^\\s+[Aa]t\\s+([^,]+,)?(.+?):(\\s+line\\s+)?(\\d+)(\\s+char:\\d+)?$' -replace '\\\\', '\')
+
+
+i -PassThru:$PassThru {
+    b "Output in VSCode mode" {
+        t "Matches problem pattern with single error" {
+            $setup = {
+                $env:TERM_PROGRAM = 'vscode'
+                $psEditor = $null
+            }
+
+            $sb = {
+                Describe 'VSCode Output Test' {
+                    It 'Single error' {
+                        1 | Should -Be 2
+                    }
+                }
+            }
+
+            $output = Invoke-PesterInProcess -ScriptBlock $sb -Setup $setup
+
+            $hostSingleError = $output | Select-String -Pattern 'Single error' -Context 0, 1
+            $hostSingleError.Line -match $titlePattern | Verify-True
+            $hostSingleError.Context.PostContext[0] -match $atPattern | Verify-True
+        }
+
+        t "Matches problem pattern with multiple errors" {
+            $setup = {
+                $env:TERM_PROGRAM = 'vscode'
+                $psEditor = $null
+
+                $PesterPreference = [PesterConfiguration]::Default
+                $PesterPreference.Should.ErrorAction = 'Continue'
+            }
+
+            $sb = {
+                Describe 'VSCode Output Test' {
+                    It 'Multiple errors' {
+                        1 | Should -Be 2
+                        1 | Should -Be 3
+                    }
+                }
+            }
+
+            $output = Invoke-PesterInProcess $sb -Setup $setup
+
+            $hostMultipleErrors = $output | Select-String -Pattern 'Multiple errors' -Context 0, 1
+            $hostMultipleErrors.Count | Verify-Equal 2
+            $hostMultipleErrors[0].Line -match $titlePattern | Verify-True
+            $hostMultipleErrors[0].Context.PostContext[0] -match $atPattern | Verify-True
+            $hostMultipleErrors[1].Line -match $titlePattern | Verify-True
+            $hostMultipleErrors[1].Context.PostContext[0] -match $atPattern | Verify-True
+        }
+    }
+}