Tests/pester5runner (#9521)

appveyor.yml

@@ -4,9 +4,6 @@ configuration: "Debug"
 
 build_script:
  - ps: Set-Service wuauserv -StartupType Manual #otherwise, choco command exits with code 1058
- - ps: choco install dotnet-5.0-sdk | Out-String | Out-Null
- - ps: choco install dotnetcore-sdk | Out-String | Out-Null
-# - ps: Push-Location bin\projects\dbatools; dotnet build ;Pop-Location
 
 version: 2.1.{build}
 

tests/appveyor.pester.ps1

@@ -3,7 +3,7 @@
 This script will invoke Pester tests, then serialize XML results and pull them in appveyor.yml
 
 .DESCRIPTION
-Internal function that creates SMO server object.
+Internal function that runs pester tests
 
 .PARAMETER Finalize
 If Finalize is specified, we collect XML output, upload tests, and indicate build errors
@@ -159,23 +159,13 @@ function Get-CodecovReport($Results, $ModuleBase) {
     $newreport
 }
 
-function Send-CodecovReport($CodecovReport) {
-    $params = @{ }
-    $params['branch'] = $env:APPVEYOR_REPO_BRANCH
-    $params['service'] = "appveyor"
-    $params['job'] = $env:APPVEYOR_ACCOUNT_NAME
-    if ($params['job']) { $params['job'] += '/' + $env:APPVEYOR_PROJECT_SLUG }
-    if ($params['job']) { $params['job'] += '/' + $env:APPVEYOR_BUILD_VERSION }
-    $params['build'] = $env:APPVEYOR_JOB_ID
-    $params['pr'] = $env:APPVEYOR_PULL_REQUEST_NUMBER
-    $params['slug'] = $env:APPVEYOR_REPO_NAME
-    $params['commit'] = $env:APPVEYOR_REPO_COMMIT
-    Add-Type -AssemblyName System.Web
-    $CodeCovParams = [System.Web.HttpUtility]::ParseQueryString([String]::Empty)
-    $params.GetEnumerator() | Where-Object Value | ForEach-Object { $CodeCovParams.Add($_.Name, $_.Value) }
-    $Request = [System.UriBuilder]('https://codecov.io/upload/v2')
-    $Request.Query = $CodeCovParams.ToString()
-    Invoke-RestMethod -Uri $Request.Uri -Method Post -InFile $CodecovReport -ContentType 'multipart/form-data'
+function Get-PesterTestVersion($testFilePath) {
+    $testFileContent = Get-Content -Path $testFilePath -Raw
+    if ($testFileContent -Like '*#pester5*')
+    {
+        return '5'
+    }
+    return '4'
 }
 
 
@@ -188,15 +178,19 @@ if (-not $Finalize) {
 #Run a test with the current version of PowerShell
 #Make things faster by removing most output
 if (-not $Finalize) {
-    Import-Module Pester
-    Write-Host -Object "appveyor.pester: Running with Pester Version $((Get-Command Invoke-Pester -ErrorAction SilentlyContinue).Version)" -ForegroundColor DarkGreen
     Set-Variable ProgressPreference -Value SilentlyContinue
     if ($AllScenarioTests.Count -eq 0) {
         Write-Host -ForegroundColor DarkGreen "Nothing to do in this scenario"
         return
     }
+    # Remove any previously loaded pester module
+    Remove-Module -Name pester -ErrorAction SilentlyContinue
+    # Import pester 4
+    Import-Module pester -RequiredVersion 4.4.2
+    Write-Host -Object "appveyor.pester: Running with Pester Version $((Get-Command Invoke-Pester -ErrorAction SilentlyContinue).Version)" -ForegroundColor DarkGreen
     # invoking a single invoke-pester consumes too much memory, let's go file by file
     $AllTestsWithinScenario = Get-ChildItem -File -Path $AllScenarioTests
+    #start the round for pester 4 tests
     $Counter = 0
     foreach ($f in $AllTestsWithinScenario) {
         $Counter += 1
@@ -205,6 +199,13 @@ if (-not $Finalize) {
             'Show'     = 'None'
             'PassThru' = $true
         }
+        #get if this test should run on pester 4 or pester 5
+        $pesterVersionToUse = Get-PesterTestVersion -testFilePath $f.FullName
+        if ($pesterVersionToUse -eq '5') {
+            # we're in the "region" of pester 4, so skip
+            continue
+        }
+
         #opt-in
         if ($IncludeCoverage) {
             $CoverFiles = Get-CoverageIndications -Path $f -ModuleBase $ModuleBase
@@ -218,7 +219,7 @@ if (-not $Finalize) {
             if ($trialNo -eq 1) {
                 $appvTestName = $f.Name
             } else {
-                $appvTestName = "$f.Name, attempt #$trialNo"
+                $appvTestName = "$($f.Name), attempt #$trialNo"
             }
             Add-AppveyorTest -Name $appvTestName -Framework NUnit -FileName $f.FullName -Outcome Running
             $PesterRun = Invoke-Pester @PesterSplat
@@ -233,6 +234,56 @@ if (-not $Finalize) {
             }
         }
     }
+
+    #start the round for pester 5 tests
+    # Remove any previously loaded pester module
+    Remove-Module -Name pester -ErrorAction SilentlyContinue
+    # Import pester 4
+    Import-Module pester -RequiredVersion 5.6.1
+    Write-Host -Object "appveyor.pester: Running with Pester Version $((Get-Command Invoke-Pester -ErrorAction SilentlyContinue).Version)" -ForegroundColor DarkGreen
+    $Counter = 0
+    foreach ($f in $AllTestsWithinScenario) {
+        $Counter += 1
+
+        #get if this test should run on pester 4 or pester 5
+        $pesterVersionToUse = Get-PesterTestVersion -testFilePath $f.FullName
+        if ($pesterVersionToUse -eq '4') {
+            # we're in the "region" of pester 5, so skip
+            continue
+        }
+        $pester5Config = New-PesterConfiguration
+        $pester5Config.Run.Path = $f.FullName
+        $pester5config.Run.PassThru = $true
+        #opt-in
+        if ($IncludeCoverage) {
+            $CoverFiles = Get-CoverageIndications -Path $f -ModuleBase $ModuleBase
+            $pester5Config.CodeCoverage.Enabled = $true
+            $pester5Config.CodeCoverage.Path = $CoverFiles
+            $pester5Config.CodeCoverage.OutputFormat = 'JaCoCo'
+            $pester5Config.CodeCoverage.OutputPath = "$ModuleBase\Pester5Coverage$PSVersion$Counter.xml"
+        }
+
+        $trialNo = 1
+        while ($trialNo -le 3) {
+            if ($trialNo -eq 1) {
+                $appvTestName = $f.Name
+            } else {
+                $appvTestName = "$($f.Name), attempt #$trialNo"
+            }
+            Add-AppveyorTest -Name $appvTestName -Framework NUnit -FileName $f.FullName -Outcome Running
+            $PesterRun = Invoke-Pester -Configuration $pester5config
+            $PesterRun | Export-Clixml -Path "$ModuleBase\Pester5Results$PSVersion$Counter.xml"
+            $outcome = "Passed"
+            if ($PesterRun.FailedCount -gt 0) {
+                $trialno += 1
+                Update-AppveyorTest -Name $appvTestName -Framework NUnit -FileName $f.FullName -Outcome "Failed" -Duration $PesterRun.Time.TotalMilliseconds
+            } else {
+                Update-AppveyorTest -Name $appvTestName -Framework NUnit -FileName $f.FullName -Outcome "Passed" -Duration $PesterRun.Time.TotalMilliseconds
+                break
+            }
+        }
+    }
+
     # Gather support package as an artifact
     # New-DbatoolsSupportPackage -Path $ModuleBase - turns out to be too heavy
     try {
@@ -281,9 +332,10 @@ if (-not $Finalize) {
     #$totalcount = $results | Select-Object -ExpandProperty TotalCount | Measure-Object -Sum | Select-Object -ExpandProperty Sum
     $failedcount = $results | Select-Object -ExpandProperty FailedCount | Measure-Object -Sum | Select-Object -ExpandProperty Sum
     if ($failedcount -gt 0) {
+        # pester 4 output
         $faileditems = $results | Select-Object -ExpandProperty TestResult | Where-Object { $_.Passed -notlike $True }
         if ($faileditems) {
-            Write-Warning "Failed tests summary:"
+            Write-Warning "Failed tests summary (pester 4):"
             $faileditems | ForEach-Object {
                 $name = $_.Name
                 [pscustomobject]@{
@@ -297,8 +349,30 @@ if (-not $Finalize) {
             throw "$failedcount tests failed."
         }
     }
+
+
+    $results5 = @(Get-ChildItem -Path "$ModuleBase\Pester5Results*.xml" | Import-Clixml)
+    # pester 5 output
+    $faileditems = $results5 | Select-Object -ExpandProperty Tests | Where-Object { $_.Passed -notlike $True }
+    if ($faileditems) {
+        Write-Warning "Failed tests summary (pester 5):"
+        $faileditems | ForEach-Object {
+            $name = $_.Name
+            [pscustomobject]@{
+                Path = $_.Path -Join '/'
+                Name     = "It $name"
+                Result   = $_.Result
+                Message  = $_.ErrorRecord -Join ""
+            }
+        } | Sort-Object Path, Name, Result, Message | Format-List
+        throw "$failedcount tests failed."
+    }
+
     #opt-in
     if ($IncludeCoverage) {
+        # for now, this manages recreating a codecov-ingestable format for pester 4. Pester 5 uses JaCoCo natively, which
+        # codecov accepts ... there's only the small matter that we generate one coverage per run, and there's a run per test file
+        # and there's no native-powershelly-way to merge JaCoCo reports. Let's start small, and complicate our lives farther down the line.
         $CodecovReport = Get-CodecovReport -Results $results -ModuleBase $ModuleBase
         $CodecovReport | ConvertTo-Json -Depth 4 -Compress | Out-File -FilePath "$ModuleBase\PesterResultsCoverage.json" -Encoding utf8
     }

tests/appveyor.post.ps1

@@ -1,7 +1,18 @@
 Add-AppveyorTest -Name "appveyor.post" -Framework NUnit -FileName "appveyor.post.ps1" -Outcome Running
 $sw = [system.diagnostics.stopwatch]::startNew()
-Write-Host -Object "appveyor.post: Sending coverage data" -ForeGroundColor DarkGreen
+Write-Host -Object "appveyor.post: Sending coverage data (pester 4)" -ForeGroundColor DarkGreen
 Push-AppveyorArtifact PesterResultsCoverage.json -FileName "PesterResultsCoverage"
 codecov -f PesterResultsCoverage.json --flag "ps,$($env:SCENARIO.ToLowerInvariant())" | Out-Null
+
+Write-Host -Object "appveyor.post: Sending coverage data (pester 5)" -ForeGroundColor DarkGreen
+$ProjectRoot = $env:APPVEYOR_BUILD_FOLDER,
+$ModuleBase = $ProjectRoot,
+$pester5CoverageFiles = Get-ChildItem -Path "$ModuleBase\Pester5Coverage*.xml"
+foreach($coverageFile in $pester5CoverageFiles)
+{
+    Push-AppveyorArtifact $coverageFile.FullName -FileName $coverageFile.Name
+    codecov -f $coverageFile.FullName --flag "ps,$($env:SCENARIO.ToLowerInvariant())" | Out-Null
+}
+
 $sw.Stop()
 Update-AppveyorTest -Name "appveyor.post" -Framework NUnit -FileName "appveyor.post.ps1" -Outcome Passed -Duration $sw.ElapsedMilliseconds

tests/appveyor.prep.ps1

@@ -19,6 +19,7 @@ git clone -q --branch=master --depth=1 https://github.com/dataplat/appveyor-lab.
 #Get codecov (to upload coverage results)
 Write-Host -Object "appveyor.prep: Install codecov" -ForegroundColor DarkGreen
 choco install codecov | Out-Null
+#FIXME : read about the new uploader https://docs.codecov.com/docs/codecov-uploader#using-the-uploader
 
 #Get PSScriptAnalyzer (to check warnings)
 Write-Host -Object "appveyor.prep: Install PSScriptAnalyzer" -ForegroundColor DarkGreen
@@ -33,10 +34,14 @@ if (-not(Test-Path 'C:\Program Files\WindowsPowerShell\Modules\dbatools.library'
 }
 
 #Get Pester (to run tests) - choco isn't working onall scenarios, weird
-Write-Host -Object "appveyor.prep: Install Pester" -ForegroundColor DarkGreen
+Write-Host -Object "appveyor.prep: Install Pester4" -ForegroundColor DarkGreen
 if (-not(Test-Path 'C:\Program Files\WindowsPowerShell\Modules\Pester\4.4.2')) {
     Install-Module -Name Pester -Force -SkipPublisherCheck -MaximumVersion 4.4.2 | Out-Null
 }
+Write-Host -Object "appveyor.prep: Install Pester5" -ForegroundColor DarkGreen
+if (-not(Test-Path 'C:\Program Files\WindowsPowerShell\Modules\Pester\5.6.1')) {
+    Install-Module -Name Pester -Force -SkipPublisherCheck -RequiredVersion 5.6.1 | Out-Null
+}
 
 #Setup DbatoolsConfig Path.DbatoolsExport path
 Write-Host -Object "appveyor.prep: Create Path.DbatoolsExport" -ForegroundColor DarkGreen
@@ -45,10 +50,6 @@ if (-not(Test-Path 'C:\Users\appveyor\Documents\DbatoolsExport')) {
 }
 
 
-#Get opencover.portable (to run DLL tests)
-Write-Host -Object "appveyor.prep: Install opencover.portable" -ForegroundColor DarkGreen
-choco install opencover.portable | Out-Null
-
 Write-Host -Object "appveyor.prep: Trust SQL Server Cert (now required)" -ForegroundColor DarkGreen
 Import-Module dbatools.library
 Import-Module C:\github\dbatools\dbatools.psd1