From 88ebd31140ca86aa5095764ce1e62a91dbff2485 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 29 Jan 2021 23:18:14 -0500 Subject: [PATCH] Copy job matrix functionality --- eng/scripts/job-matrix/Create-JobMatrix.ps1 | 26 + eng/scripts/job-matrix/README.md | 274 +++++++++ .../job-matrix-functions.filter.tests.ps1 | 84 +++ .../job-matrix/job-matrix-functions.ps1 | 461 ++++++++++++++ .../job-matrix/job-matrix-functions.tests.ps1 | 580 ++++++++++++++++++ .../job-matrix/samples/matrix-test.yml | 25 + eng/scripts/job-matrix/samples/matrix.json | 30 + 7 files changed, 1480 insertions(+) create mode 100644 eng/scripts/job-matrix/Create-JobMatrix.ps1 create mode 100644 eng/scripts/job-matrix/README.md create mode 100644 eng/scripts/job-matrix/job-matrix-functions.filter.tests.ps1 create mode 100644 eng/scripts/job-matrix/job-matrix-functions.ps1 create mode 100644 eng/scripts/job-matrix/job-matrix-functions.tests.ps1 create mode 100644 eng/scripts/job-matrix/samples/matrix-test.yml create mode 100644 eng/scripts/job-matrix/samples/matrix.json diff --git a/eng/scripts/job-matrix/Create-JobMatrix.ps1 b/eng/scripts/job-matrix/Create-JobMatrix.ps1 new file mode 100644 index 000000000000..c98e08ce5e91 --- /dev/null +++ b/eng/scripts/job-matrix/Create-JobMatrix.ps1 @@ -0,0 +1,26 @@ +<# + .SYNOPSIS + Generates a JSON object representing an Azure Pipelines Job Matrix. + See https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#parallelexec + + .EXAMPLE + .\eng\scripts\Create-JobMatrix $context +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory=$True)][string] $ConfigPath, + [Parameter(Mandatory=$True)][string] $Selection, + [Parameter(Mandatory=$False)][string] $DisplayNameFilter, + [Parameter(Mandatory=$False)][array] $Filters +) + +. $PSScriptRoot/job-matrix-functions.ps1 + +$config = GetMatrixConfigFromJson (Get-Content $ConfigPath) + +[array]$matrix = GenerateMatrix $config $Selection $DisplayNameFilter $Filters +$serialized = SerializePipelineMatrix $matrix + +Write-Output $serialized.pretty +Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" diff --git a/eng/scripts/job-matrix/README.md b/eng/scripts/job-matrix/README.md new file mode 100644 index 000000000000..61df7a2435b5 --- /dev/null +++ b/eng/scripts/job-matrix/README.md @@ -0,0 +1,274 @@ +# Azure Pipelines Matrix Generator + +* [Usage in a pipeline](#usage-in-a-pipeline) +* [Matrix config file syntax](#matrix-config-file-syntax) + * [Fields](#fields) + * [matrix](#matrix) + * [include](#include) + * [exclude](#exclude) + * [displayNames](#displaynames) +* [Matrix Generation behavior](#matrix-generation-behavior) + * [all](#all) + * [sparse](#sparse) + * [include/exclude](#includeexclude) + * [displayNames](#displaynames-1) + * [Filters](#filters) + * [Under the hood](#under-the-hood) +* [Testing](#testing) + + +This directory contains scripts supporting dynamic, cross-product matrix generation for azure pipeline jobs. +It aims to replicate the [cross-product matrix functionality in github actions](https://docs.github.com/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#example-running-with-more-than-one-version-of-nodejs), +but also adds some additional features like sparse matrix generation, cross-product includes and excludes, and programmable matrix filters. + +This functionality is made possible by the ability for the azure pipelines yaml to take a [dynamic variable as an input +for a job matrix definition](https://docs.microsoft.com/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration) (see the code sample at the bottom of the linked section). + +## Usage in a pipeline + +In order to use these scripts in a pipeline, you must provide a config file and call the matrix creation script within a powershell job. + +For a single matrix, you can include the `eng/pipelines/templates/jobs/job-matrix.yml` template in a pipeline: + +``` +jobs: +- template: /eng/pipelines/templates/jobs/job-matrix.yml + parameters: + MatrixConfigs: + - Name: base_product_matrix + Path: /eng/pipelines/matrix.json + Selection: sparse + GenerateVMJobs: true + - Name: sdk_specific_matrix + Path: /sdk/foobar/matrix.json + Selection: all + GenerateContainerJobs: true + steps: + - pwsh: + ... +``` + +## Matrix config file syntax + +Matrix parameters can either be a list of strings, or a set of grouped strings (represented as a hash). The latter parameter +type is useful for when 2 or more parameters need to be grouped together, but without generating more than one matrix permutation. + +``` +"matrix": { + "": [ ], + "": [ ], + "": { + "": { + "", + }, + "": { + "", + } + } +} +"include": [ , , ... ], +"exclude": [ , , ... ], +"displayNames": { : } +``` + +See `samples/matrix.json` for a full sample. + +### Fields + +#### matrix + +The `matrix` field defines the base cross-product matrix. The generated matrix can be full or sparse. + +Example: +``` +"matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1", + "net50" + ], + "additionalTestArguments": [ + "", + "/p:UseProjectReferenceToAzureClients=true", + ] +} +``` + +#### include + +The `include` field defines any number of matrices to be appended to the base matrix after processing exclusions. + +#### exclude + +The `include` field defines any number of matrices to be removed from the base matrix. Exclude parameters can be a partial +set, meaning as long as all exclude parameters match against a matrix entry (even if the matrix entry has additional parameters), +then it will be excluded from the matrix. For example, the below entry will match the exclusion and be removed: + +``` +matrix entry: +{ + "a": 1, + "b": 2, + "c": 3, +} + +"exclude": [ + { + "a": 1, + "b": 2 + } +] +``` + +#### displayNames + +Specify any overrides for the azure pipelines definition and UI that determines the matrix job name. If some parameter +values are too long or unreadable for this purpose (e.g. a command line argument), then you can replace them with a more +readable value here. For example: + +``` +"displayNames": { + "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef" +}, +"matrix": { + "additionalTestArguments": [ + "/p:UseProjectReferenceToAzureClients=true" + ] +} +``` + +## Matrix Generation behavior + +#### all + +`all` will output the full matrix, i.e. every possible permutation of all parameters given (p1.Length * p2.Length * ...). + +#### sparse + +`sparse` outputs the minimum number of parameter combinations while ensuring that all parameter values are present in at least one matrix job. +Effectively this means the total length of a sparse matrix will be equal to the largest matrix dimension, i.e. `max(p1.Length, p2.Length, ...)`. + +To build a sparse matrix, a full matrix is generated, and then walked diagonally N times where N is the largest matrix dimension. +This pattern works for any N-dimensional matrix, via an incrementing index (n, n, n, ...), (n+1, n+1, n+1, ...), etc. +Index lookups against matrix dimensions are calculated modulus the dimension size, so a two-dimensional matrix of 4x2 might be walked like this: + +``` +index: 0, 0: +o . . . +. . . . + +index: 1, 1: +. . . . +. o . . + +index: 2, 2 (modded to 2, 0): +. . o . +. . . . + +index: 3, 3 (modded to 3, 1): +. . . . +. . . o +``` + +#### include/exclude + +Include and exclude support additions and subtractions off the base matrix. Both include and exclude take an array of matrix values. +Typically these values will be a single entry, but they also support the cross-product matrix definition syntax of the base matrix. + +Include and exclude are parsed fully. So if a sparse matrix is called for, a sparse version of the base matrix will be generated, but +the full matrix of both include and exclude will be processed. + +Excludes are processed first, so includes can be used to add back any specific jobs to the matrix. + +#### displayNames + +In the matrix job output that azure pipelines consumes, the format is a dictionary of dictionaries. For example: + +``` +{ + "net461_macOS1015": { + "framework": "net461", + "operatingSystem": "macOS-10.15" + }, + "net50_ubuntu1804": { + "framework": "net50", + "operatingSystem": "ubuntu-18.04" + }, + "netcoreapp21_windows2019": { + "framework": "netcoreapp2.1", + "operatingSystem": "windows-2019" + }, + "UseProjectRef_net461_windows2019": { + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true", + "framework": "net461", + "operatingSystem": "windows-2019" + } +} +``` + +The top level keys are used as job names, meaning they get displayed in the azure pipelines UI when running the pipeline. + +The logic for generating display names works like this: + +- Join parameter values by "_" + a. If the parameter value exists as a key in `displayNames` in the matrix config, replace it with that value. + b. For each name value, strip all non-alphanumeric characters (excluding "_"). + c. If the name is greater than 100 characters, truncate it. + +#### Filters + +Filters can be passed to the matrix as an array of strings, each matching the format of =. When a matrix entry +does not contain the specified key, it will default to a value of empty string for regex parsing. This can be used to specify +filters for keys that don't exist or keys that optionally exist and match a regex, as seen in the below example. + +Display name filters can also be passed as a single regex string that runs against the [generated display name](#displaynames) of the matrix job. +The intent of display name filters is to be defined primarily as a top level variable at template queue time in the azure pipelines UI. + +For example, the below command will filter for matrix entries with "windows" in the job display name, no matrix variable +named "ExcludedKey", a framework variable containing either "461" or "5.0", and an optional key "SupportedClouds" that, if exists, must contain "Public": + +``` +./Create-JobMatrix.ps1 ` + -ConfigPath samples/matrix.json ` + -Selection all ` + -DisplayNameFilter ".*windows.*" ` + -Filters @("ExcludedKey=^$", "framework=(461|5\.0)", "SupportedClouds=^$|.*Public.*") +``` + +#### Under the hood + +The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example, +the below config would generate a 2x2x1x1x1 matrix (five-dimensional): + +``` +"matrix": { + "framework": [ "net461", "netcoreapp2.1" ], + "additionalTestArguments": [ "", "/p:SuperTest=true" ] + "pool": [ "ubuntu-18.04" ], + "container": [ "ubuntu-18.04" ], + "testMode": [ "Record" ] +} +``` + +The matrix is stored as a one-dimensional array, with a row-major indexing scheme (e.g. `(2, 1, 0, 1, 0)`). + +## Testing + +The matrix functions can be tested using [pester](https://pester.dev/): + +``` +$ Invoke-Pester + +Starting discovery in 1 files. +Discovery finished in 384ms. +[+] /home/ben/sdk/azure-sdk-for-net/eng/scripts/job-matrix/job-matrix-functions.tests.ps1 4.09s (1.52s|2.22s) +Tests completed in 4.12s +Tests Passed: 120, Failed: 0, Skipped: 4 NotRun: 0 +``` diff --git a/eng/scripts/job-matrix/job-matrix-functions.filter.tests.ps1 b/eng/scripts/job-matrix/job-matrix-functions.filter.tests.ps1 new file mode 100644 index 000000000000..52ab626166d0 --- /dev/null +++ b/eng/scripts/job-matrix/job-matrix-functions.filter.tests.ps1 @@ -0,0 +1,84 @@ +Import-Module Pester + +BeforeAll { + . ./job-matrix-functions.ps1 + + $matrixConfig = @" +{ + "matrix": { + "operatingSystem": [ "windows-2019", "ubuntu-18.04", "macOS-10.15" ], + "framework": [ "net461", "netcoreapp2.1" ], + "additionalArguments": [ "", "mode=test" ] + } +} +"@ + $config = GetMatrixConfigFromJson $matrixConfig +} + +Describe "Matrix Filter" -Tag "filter" { + It "Should filter by matrix display name" -TestCases @( + @{ regex = "windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ regex = "windows2019_netcoreapp21_modetest"; expectedFirst = "windows2019_netcoreapp21_modetest"; length = 1 } + @{ regex = ".*ubuntu.*"; expectedFirst = "ubuntu1804_net461"; length = 4 } + ) { + [array]$matrix = GenerateMatrix $config "all" $regex + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should handle no display name filter matches" { + $matrix = GenerateMatrix $config "all" + [array]$filtered = FilterMatrixDisplayName $matrix "doesnotexist" + $filtered | Should -BeNullOrEmpty + } + + It "Should filter by matrix key/value" -TestCases @( + @{ filterString = "operatingSystem=windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "operatingSystem=windows-2019"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "framework=.*"; expectedFirst = "windows2019_net461"; length = 12 } + @{ filterString = "additionalArguments=mode=test"; expectedFirst = "windows2019_net461_modetest"; length = 6 } + @{ filterString = "additionalArguments=^$"; expectedFirst = "windows2019_net461"; length = 6 } + ) { + [array]$matrix = GenerateMatrix $config "all" -filters @($filterString) + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should filter by optional matrix key/value" -TestCases @( + @{ filterString = "operatingSystem=^$|windows.*"; expectedFirst = "windows2019_net461"; length = 4 } + @{ filterString = "doesnotexist=^$|.*"; expectedFirst = "windows2019_net461"; length = 12 } + ) { + [array]$matrix = GenerateMatrix $config "all" -filters @($filterString) + $matrix.Length | Should -Be $length + $matrix[0].Name | Should -Be $expectedFirst + } + + It "Should handle multiple matrix key/value filters " { + [array]$matrix = GenerateMatrix $config "all" -filters "operatingSystem=windows.*","framework=.*","additionalArguments=mode=test" + $matrix.Length | Should -Be 2 + $matrix[0].Name | Should -Be "windows2019_net461_modetest" + } + + It "Should handle no matrix key/value filter matches" { + [array]$matrix = GenerateMatrix $config "all" -filters @("doesnot=exist") + $matrix | Should -BeNullOrEmpty + } + + It "Should handle invalid matrix key/value filter syntax" { + { GenerateMatrix $config "all" -filters @("invalid") } | Should -Throw + { GenerateMatrix $config "all" -filters @("emptyvalue=") } | Should -Throw + { GenerateMatrix $config "all" -filters @("=emptykey") } | Should -Throw + { GenerateMatrix $config "all" -filters @("=") } | Should -Throw + } + + It "Should filter by key exclude" { + [array]$matrix = GenerateMatrix $config "all" -filters @("operatingSystem=^$") + $matrix | Should -BeNullOrEmpty + + [array]$matrix = GenerateMatrix $config "all" + $matrix.Length | Should -Be 12 + $matrix += @{ Name = "excludeme"; Parameters = [Ordered]@{ "foo" = 1 } } + [array]$matrix = FilterMatrix $matrix @("foo=^$") + $matrix.Length | Should -Be 12 + } +} diff --git a/eng/scripts/job-matrix/job-matrix-functions.ps1 b/eng/scripts/job-matrix/job-matrix-functions.ps1 new file mode 100644 index 000000000000..2fbd8958acf7 --- /dev/null +++ b/eng/scripts/job-matrix/job-matrix-functions.ps1 @@ -0,0 +1,461 @@ +Set-StrictMode -Version "4.0" + +class MatrixConfig { + [PSCustomObject]$displayNames + [Hashtable]$displayNamesLookup + [PSCustomObject]$matrix + [System.Collections.Specialized.OrderedDictionary]$orderedMatrix + [Array]$include + [Array]$exclude +} + +function CreateDisplayName([string]$parameter, [Hashtable]$displayNamesLookup) +{ + $name = $parameter.ToString() + + if ($displayNamesLookup.ContainsKey($parameter)) { + $name = $displayNamesLookup[$parameter] + } + + # Matrix naming restrictions: + # https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration + $name = $name -replace "[^A-Za-z0-9_]", "" + return $name +} + +function GenerateMatrix( + [MatrixConfig]$config, + [string]$selectFromMatrixType, + [string]$displayNameFilter = ".*", + [array]$filters = @() +) { + if ($selectFromMatrixType -eq "sparse") { + [Array]$matrix = GenerateSparseMatrix $config.orderedMatrix $config.displayNamesLookup + } elseif ($selectFromMatrixType -eq "all") { + [Array]$matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + } else { + throw "Matrix generator not implemented for selectFromMatrixType: $($platform.selectFromMatrixType)" + } + + if ($config.exclude) { + [Array]$matrix = ProcessExcludes $matrix $config.exclude + } + if ($config.include) { + [Array]$matrix = ProcessIncludes $matrix $config.include $config.displayNamesLookup + } + + [Array]$matrix = FilterMatrixDisplayName $matrix $displayNameFilter + [Array]$matrix = FilterMatrix $matrix $filters + return $matrix +} + +function FilterMatrixDisplayName([array]$matrix, [string]$filter) { + return $matrix | ForEach-Object { + if ($_.Name -match $filter) { + return $_ + } + } +} + +# Filters take the format of key=valueregex,key2=valueregex2 +function FilterMatrix([array]$matrix, [array]$filters) { + $matrix = $matrix | ForEach-Object { + if (MatchesFilters $_ $filters) { + return $_ + } + } + return $matrix +} + +function MatchesFilters([hashtable]$entry, [array]$filters) { + foreach ($filter in $filters) { + $key, $regex = ParseFilter $filter + # Default all regex checks to go against empty string when keys are missing. + # This simplifies the filter syntax/interface to be regex only. + $value = "" + if ($null -ne $entry -and $entry.parameters.Contains($key)) { + $value = $entry.parameters[$key] + } + if ($value -notmatch $regex) { + return $false + } + } + + return $true +} + +function ParseFilter([string]$filter) { + # Lazy match key in case value contains '=' + if ($filter -match "(.+?)=(.+)") { + $key = $matches[1] + $regex = $matches[2] + return $key, $regex + } else { + throw "Invalid filter: `"${filter}`", expected = format" + } +} + +# Importing the JSON as PSCustomObject preserves key ordering, +# whereas ConvertFrom-Json -AsHashtable does not +function GetMatrixConfigFromJson($jsonConfig) +{ + [MatrixConfig]$config = $jsonConfig | ConvertFrom-Json + $config.orderedMatrix = [ordered]@{} + $config.displayNamesLookup = @{} + + if ($null -ne $config.matrix) { + $config.matrix.PSObject.Properties | ForEach-Object { + $config.orderedMatrix.Add($_.Name, $_.Value) + } + } + if ($null -ne $config.displayNames) { + $config.displayNames.PSObject.Properties | ForEach-Object { + $config.displayNamesLookup.Add($_.Name, $_.Value) + } + } + $config.include = $config.include | Where-Object { $null -ne $_ } | ForEach-Object { + $ordered = [ordered]@{} + $_.PSObject.Properties | ForEach-Object { + $ordered.Add($_.Name, $_.Value) + } + return $ordered + } + $config.exclude = $config.exclude | Where-Object { $null -ne $_ } | ForEach-Object { + $ordered = [ordered]@{} + $_.PSObject.Properties | ForEach-Object { + $ordered.Add($_.Name, $_.Value) + } + return $ordered + } + + return $config +} + +function ProcessExcludes([Array]$matrix, [Array]$excludes) +{ + $deleteKey = "%DELETE%" + $exclusionMatrix = @() + + foreach ($exclusion in $excludes) { + $converted = ConvertToMatrixArrayFormat $exclusion + $full = GenerateFullMatrix $converted + $exclusionMatrix += $full + } + + foreach ($element in $matrix) { + foreach ($exclusion in $exclusionMatrix) { + $match = MatrixElementMatch $element.parameters $exclusion.parameters + if ($match) { + $element.parameters[$deleteKey] = $true + } + } + } + + return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } +} + +function ProcessIncludes([Array]$matrix, [Array]$includes, [Hashtable]$displayNamesLookup) +{ + foreach ($inclusion in $includes) { + $converted = ConvertToMatrixArrayFormat $inclusion + $full = GenerateFullMatrix $converted $displayNamesLookup + $matrix += $full + } + + return $matrix +} + +function MatrixElementMatch([System.Collections.Specialized.OrderedDictionary]$source, [System.Collections.Specialized.OrderedDictionary]$target) +{ + if ($target.Count -eq 0) { + return $false + } + + foreach ($key in $target.Keys) { + if (-not $source.Contains($key) -or $source[$key] -ne $target[$key]) { + return $false + } + } + + return $true +} + +function ConvertToMatrixArrayFormat([System.Collections.Specialized.OrderedDictionary]$matrix) +{ + $converted = [Ordered]@{} + + foreach ($key in $matrix.Keys) { + if ($matrix[$key] -isnot [Array]) { + $converted[$key] = ,$matrix[$key] + } else { + $converted[$key] = $matrix[$key] + } + } + + return $converted +} + +function CloneOrderedDictionary([System.Collections.Specialized.OrderedDictionary]$dictionary) { + $newDictionary = [Ordered]@{} + foreach ($element in $dictionary.GetEnumerator()) { + $newDictionary[$element.Name] = $element.Value + } + return $newDictionary +} + +function SerializePipelineMatrix([Array]$matrix) +{ + $pipelineMatrix = [Ordered]@{} + foreach ($entry in $matrix) { + $pipelineMatrix.Add($entry.name, [Ordered]@{}) + foreach ($key in $entry.parameters.Keys) { + $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key]) + } + } + + return @{ + compressed = $pipelineMatrix | ConvertTo-Json -Compress ; + pretty = $pipelineMatrix | ConvertTo-Json; + } +} + +function GenerateSparseMatrix([System.Collections.Specialized.OrderedDictionary]$parameters, [Hashtable]$displayNamesLookup) +{ + [Array]$dimensions = GetMatrixDimensions $parameters + $size = ($dimensions | Measure-Object -Maximum).Maximum + + [Array]$matrix = GenerateFullMatrix $parameters $displayNamesLookup + $sparseMatrix = @() + + # With full matrix, retrieve items by doing diagonal lookups across the matrix N times. + # For example, given a matrix with dimensions 3, 2, 2: + # 0, 0, 0 + # 1, 1, 1 + # 2, 2, 2 + # 3, 0, 0 <- 3, 3, 3 wraps to 3, 0, 0 given the dimensions + for ($i = 0; $i -lt $size; $i++) { + $idx = @() + for ($j = 0; $j -lt $dimensions.Length; $j++) { + $idx += $i % $dimensions[$j] + } + $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions + } + + return $sparseMatrix +} + +function GenerateFullMatrix([System.Collections.Specialized.OrderedDictionary] $parameters, [Hashtable]$displayNamesLookup = @{}) +{ + # Handle when the config does not have a matrix specified (e.g. only the include field is specified) + if ($parameters.Count -eq 0) { + return @() + } + + $parameterArray = $parameters.GetEnumerator() | ForEach-Object { $_ } + + $matrix = [System.Collections.ArrayList]::new() + InitializeMatrix $parameterArray $displayNamesLookup $matrix + + return $matrix.ToArray() +} + +function CreateMatrixEntry([System.Collections.Specialized.OrderedDictionary]$permutation, [Hashtable]$displayNamesLookup = @{}) +{ + $names = @() + $splattedParameters = [Ordered]@{} + + foreach ($entry in $permutation.GetEnumerator()) { + $nameSegment = "" + + if ($entry.Value -is [PSCustomObject]) { + $nameSegment = CreateDisplayName $entry.Name $displayNamesLookup + foreach ($toSplat in $entry.Value.PSObject.Properties) { + $splattedParameters.Add($toSplat.Name, $toSplat.Value) + } + } else { + $nameSegment = CreateDisplayName $entry.Value $displayNamesLookup + $splattedParameters.Add($entry.Name, $entry.Value) + } + + if ($nameSegment) { + $names += $nameSegment + } + } + + # The maximum allowed matrix name length is 100 characters + $name = $names -join "_" + if ($name.Length -gt 100) { + $name = $name[0..99] -join "" + } + $stripped = $name -replace "^[^A-Za-z]*", "" # strip leading digits + if ($stripped -eq "") { + $name = "job_" + $name # Handle names that consist entirely of numbers + } else { + $name = $stripped + } + + return @{ + name = $name + parameters = $splattedParameters + } +} + +function InitializeMatrix +{ + param( + [Array]$parameters, + [Hashtable]$displayNamesLookup, + [System.Collections.ArrayList]$permutations, + $permutation = [Ordered]@{} + ) + + if (-not $parameters) { + $entry = CreateMatrixEntry $permutation $displayNamesLookup + $permutations.Add($entry) | Out-Null + return + } + + $head, $tail = $parameters + foreach ($value in $head.value) { + $newPermutation = CloneOrderedDictionary($permutation) + if ($value -is [PSCustomObject]) { + foreach ($nestedParameter in $value.PSObject.Properties) { + $nestedPermutation = CloneOrderedDictionary($newPermutation) + $nestedPermutation[$nestedParameter.Name] = $nestedParameter.Value + InitializeMatrix $tail $displayNamesLookup $permutations $nestedPermutation + } + } else { + $newPermutation[$head.Name] = $value + InitializeMatrix $tail $displayNamesLookup $permutations $newPermutation + } + } +} + +function GetMatrixDimensions([System.Collections.Specialized.OrderedDictionary]$parameters) +{ + $dimensions = @() + foreach ($val in $parameters.Values) { + if ($val -is [PSCustomObject]) { + $dimensions += ($val.PSObject.Properties | Measure-Object).Count + } elseif ($val -is [Array]) { + $dimensions += $val.Length + } else { + $dimensions += 1 + } + } + + return $dimensions +} + +function SetNdMatrixElement +{ + param( + $element, + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query $($idx.Length) must be the same length as its dimensions $($dimensions.Length)" + } + + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + $matrix[$arrayIndex] = $element +} + +function GetNdMatrixArrayIndex +{ + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query length ($($idx.Length)) must be the same as dimension length ($($dimensions.Length))" + } + + $stride = 1 + # Commented out does lookup with wrap handling + # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1] + $index = $idx[$idx.Length-1] + + for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + # Commented out does lookup with wrap handling + # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride + $index += $idx[$i-1] * $stride + } + + return $index +} + +function GetNdMatrixElement +{ + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + return $matrix[$arrayIndex] +} + +function GetNdMatrixIndex +{ + param( + [int]$index, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + $matrixIndex = @() + $stride = 1 + + for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + $page = [math]::floor($index / $stride) % $dimensions[$i-1] + $matrixIndex = ,$page + $matrixIndex + } + $col = $index % $dimensions[$dimensions.Length-1] + $matrixIndex += $col + + return $matrixIndex +} + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +# The below functions are non-dynamic examples that # +# help explain the above N-dimensional algorithm # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +function Get4dMatrixElement([Array]$idx, [Array]$matrix, [Array]$dimensions) +{ + $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3] + $stride2 = $idx[1] * $dimensions[2] * $dimensions[3] + $stride3 = $idx[2] * $dimensions[3] + $stride4 = $idx[3] + + return $matrix[$stride1 + $stride2 + $stride3 + $stride4] +} + +function Get4dMatrixIndex([int]$index, [Array]$dimensions) +{ + $stride1 = $dimensions[3] + $stride2 = $dimensions[2] + $stride3 = $dimensions[1] + $page1 = [math]::floor($index / $stride1) % $dimensions[2] + $page2 = [math]::floor($index / ($stride1 * $stride2)) % $dimensions[1] + $page3 = [math]::floor($index / ($stride1 * $stride2 * $stride3)) % $dimensions[0] + $remainder = $index % $dimensions[3] + + return @($page3, $page2, $page1, $remainder) +} + diff --git a/eng/scripts/job-matrix/job-matrix-functions.tests.ps1 b/eng/scripts/job-matrix/job-matrix-functions.tests.ps1 new file mode 100644 index 000000000000..5e50596e6cf6 --- /dev/null +++ b/eng/scripts/job-matrix/job-matrix-functions.tests.ps1 @@ -0,0 +1,580 @@ +Import-Module Pester + +BeforeAll { + . ./job-matrix-functions.ps1 + + $matrixConfig = @" +{ + "displayNames": { + "--enableFoo": "withfoo" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1" + ], + "additionalArguments": [ + "", + "--enableFoo" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": ["net461", "netcoreapp2.1", "net50"], + "additionalArguments": "--enableWindowsFoo" + } + ], + "exclude": [ + { + "operatingSystem": "windows-2019", + "framework": "net461" + }, + { + "operatingSystem": "macOS-10.15", + "framework": "netcoreapp2.1" + }, + { + "operatingSystem": ["macOS-10.15", "ubuntu-18.04"], + "additionalArguments": "--enableFoo" + } + ] +} +"@ + $config = GetMatrixConfigFromJson $matrixConfig +} + +Describe "Matrix-Lookup" -Tag "lookup" { + It "Should navigate a 2d matrix: " -TestCases @( + @{ row = 0; col = 0; expected = 1 }, + @{ row = 0; col = 1; expected = 2 }, + @{ row = 1; col = 0; expected = 3 }, + @{ row = 1; col = 1; expected = 4 } + ) { + $dimensions = @(2, 2) + $matrix = @( + 1, 2, 3, 4 + ) + GetNdMatrixElement @($row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 3d matrix: " -TestCases @( + @{ z = 0; row = 0; col = 0; expected = 1 } + @{ z = 0; row = 0; col = 1; expected = 2 } + @{ z = 0; row = 1; col = 0; expected = 3 } + @{ z = 0; row = 1; col = 1; expected = 4 } + @{ z = 1; row = 0; col = 0; expected = 5 } + @{ z = 1; row = 0; col = 1; expected = 6 } + @{ z = 1; row = 1; col = 0; expected = 7 } + @{ z = 1; row = 1; col = 1; expected = 8 } + ) { + $dimensions = @(2, 2, 2) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8 + ) + GetNdMatrixElement @($z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 4d matrix: " -TestCases @( + @{ t = 0; z = 0; row = 0; col = 0; expected = 1 } + @{ t = 0; z = 0; row = 0; col = 1; expected = 2 } + @{ t = 0; z = 0; row = 1; col = 0; expected = 3 } + @{ t = 0; z = 0; row = 1; col = 1; expected = 4 } + @{ t = 0; z = 1; row = 0; col = 0; expected = 5 } + @{ t = 0; z = 1; row = 0; col = 1; expected = 6 } + @{ t = 0; z = 1; row = 1; col = 0; expected = 7 } + @{ t = 0; z = 1; row = 1; col = 1; expected = 8 } + @{ t = 1; z = 0; row = 0; col = 0; expected = 9 } + @{ t = 1; z = 0; row = 0; col = 1; expected = 10 } + @{ t = 1; z = 0; row = 1; col = 0; expected = 11 } + @{ t = 1; z = 0; row = 1; col = 1; expected = 12 } + @{ t = 1; z = 1; row = 0; col = 0; expected = 13 } + @{ t = 1; z = 1; row = 0; col = 1; expected = 14 } + @{ t = 1; z = 1; row = 1; col = 0; expected = 15 } + @{ t = 1; z = 1; row = 1; col = 1; expected = 16 } + ) { + $dimensions = @(2, 2, 2, 2) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ) + GetNdMatrixElement @($t, $z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 4d matrix: " -TestCases @( + @{ t = 0; z = 0; row = 0; col = 0; expected = 1 } + @{ t = 0; z = 0; row = 0; col = 1; expected = 2 } + @{ t = 0; z = 0; row = 0; col = 2; expected = 3 } + @{ t = 0; z = 0; row = 0; col = 3; expected = 4 } + + @{ t = 0; z = 0; row = 1; col = 0; expected = 5 } + @{ t = 0; z = 0; row = 1; col = 1; expected = 6 } + @{ t = 0; z = 0; row = 1; col = 2; expected = 7 } + @{ t = 0; z = 0; row = 1; col = 3; expected = 8 } + + @{ t = 0; z = 1; row = 0; col = 0; expected = 9 } + @{ t = 0; z = 1; row = 0; col = 1; expected = 10 } + @{ t = 0; z = 1; row = 0; col = 2; expected = 11 } + @{ t = 0; z = 1; row = 0; col = 3; expected = 12 } + + @{ t = 0; z = 1; row = 1; col = 0; expected = 13 } + @{ t = 0; z = 1; row = 1; col = 1; expected = 14 } + @{ t = 0; z = 1; row = 1; col = 2; expected = 15 } + @{ t = 0; z = 1; row = 1; col = 3; expected = 16 } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ) + GetNdMatrixElement @($t, $z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + # Skipping since by default wrapping behavior on indexing is disabled. + # Keeping here in case we want to enable it. + It -Skip "Should handle index wrapping: " -TestCases @( + @{ row = 2; col = 2; expected = 1 } + @{ row = 2; col = 3; expected = 2 } + @{ row = 4; col = 4; expected = 1 } + @{ row = 4; col = 5; expected = 2 } + ) { + $dimensions = @(2, 2) + $matrix = @( + 1, 2, 3, 4 + ) + GetNdMatrixElement @($row, $col) $matrix $dimensions | Should -Be $expected + } +} + +Describe "Matrix-Reverse-Lookup" -Tag "lookup" { + It "Should lookup a 2d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0) } + @{ index = 1; expected = @(0,1) } + @{ index = 2; expected = @(1,0) } + @{ index = 3; expected = @(1,1) } + ) { + $dimensions = @(2, 2) + $matrix = @(1, 2, 3, 4) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,1,0) } + @{ index = 3; expected = @(0,1,1) } + + @{ index = 4; expected = @(1,0,0) } + @{ index = 5; expected = @(1,0,1) } + @{ index = 6; expected = @(1,1,0) } + @{ index = 7; expected = @(1,1,1) } + ) { + $dimensions = @(2, 2, 2) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,0,2) } + + @{ index = 3; expected = @(0,1,0) } + @{ index = 4; expected = @(0,1,1) } + @{ index = 5; expected = @(0,1,2) } + + @{ index = 6; expected = @(1,0,0) } + @{ index = 7; expected = @(1,0,1) } + @{ index = 8; expected = @(1,0,2) } + + @{ index = 9; expected = @(1,1,0) } + @{ index = 10; expected = @(1,1,1) } + @{ index = 11; expected = @(1,1,2) } + ) { + $dimensions = @(2, 2, 3) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,1,0) } + @{ index = 3; expected = @(0,1,1) } + + @{ index = 4; expected = @(1,0,0) } + @{ index = 5; expected = @(1,0,1) } + @{ index = 6; expected = @(1,1,0) } + @{ index = 7; expected = @(1,1,1) } + + @{ index = 8; expected = @(2,0,0) } + @{ index = 9; expected = @(2,0,1) } + @{ index = 10; expected = @(2,1,0) } + @{ index = 11; expected = @(2,1,1) } + ) { + $dimensions = @(3, 2, 2) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 4d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0,0) } + @{ index = 1; expected = @(0,0,0,1) } + @{ index = 2; expected = @(0,0,0,2) } + @{ index = 3; expected = @(0,0,0,3) } + + @{ index = 4; expected = @(0,0,1,0) } + @{ index = 5; expected = @(0,0,1,1) } + @{ index = 6; expected = @(0,0,1,2) } + @{ index = 7; expected = @(0,0,1,3) } + + @{ index = 8; expected = @(0,1,0,0) } + @{ index = 9; expected = @(0,1,0,1) } + @{ index = 10; expected = @(0,1,0,2) } + @{ index = 11; expected = @(0,1,0,3) } + + @{ index = 12; expected = @(0,1,1,0) } + @{ index = 13; expected = @(0,1,1,1) } + @{ index = 14; expected = @(0,1,1,2) } + @{ index = 15; expected = @(0,1,1,3) } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } +} + +Describe 'Matrix-Set' -Tag "set" { + It "Should set a matrix element" -TestCases @( + @{ value = "set"; index = @(0,0,0,0); arrayIndex = 0 } + @{ value = "ones"; index = @(0,1,1,1); arrayIndex = 13 } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + + SetNdMatrixElement $value $index $matrix $dimensions + $matrix[$arrayIndex] | Should -Be $value + } +} + +Describe "Platform Matrix Generation" -Tag "generate" { + BeforeEach { + $matrixConfigForGenerate = @" +{ + "displayNames": { + "--enableFoo": "withfoo" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1" + ], + "additionalArguments": [ + "", + "--enableFoo" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": "net461", + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true" + } + ], + "exclude": [ + { + "foo": "bar" + }, + { + "foo2": "bar2" + } + ] +} +"@ + $generateConfig = GetMatrixConfigFromJson $matrixConfigForGenerate + } + + It "Should get matrix dimensions from Nd parameters" { + GetMatrixDimensions $generateConfig.orderedMatrix | Should -Be 3, 2, 2 + + $generateConfig.orderedMatrix.Add("testStringParameter", "test") + GetMatrixDimensions $generateConfig.orderedMatrix | Should -Be 3, 2, 2, 1 + } + + It "Should use name overrides from displayNames" { + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateConfig.orderedMatrix $generateconfig.displayNamesLookup + + $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions + $element.name | Should -Be "windows2019_net461" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.name | Should -Be "ubuntu1804_netcoreapp21_withFoo" + + $element = GetNdMatrixElement @(2, 1, 1) $matrix $dimensions + $element.name | Should -Be "macOS1015_netcoreapp21_withFoo" + } + + It "Should enforce valid display name format" { + $generateconfig.displayNamesLookup["net461"] = '123.Some.456.Invalid_format-name$(foo)' + $generateconfig.displayNamesLookup["netcoreapp2.1"] = (New-Object string[] 150) -join "a" + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateconfig.orderedMatrix $generateconfig.displayNamesLookup + + $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions + $element.name | Should -Be "windows2019_123some456invalid_formatnamefoo" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.name.Length | Should -Be 100 + # The withfoo part of the argument gets cut off at the character limit + $element.name | Should -BeLike "ubuntu1804_aaaaaaaaaaaaaaaaa*" + } + + + It "Should initialize an N-dimensional matrix from all parameter permutations" { + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $matrix = GenerateFullMatrix $generateConfig.orderedMatrix $generateConfig.displayNamesLookup + $matrix.Count | Should -Be 12 + + $element = $matrix[0].parameters + $element.operatingSystem | Should -Be "windows-2019" + $element.framework | Should -Be "net461" + $element.additionalArguments | Should -Be "" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.parameters.operatingSystem | Should -Be "ubuntu-18.04" + $element.parameters.framework | Should -Be "netcoreapp2.1" + $element.parameters.additionalArguments | Should -Be "--enableFoo" + + $element = GetNdMatrixElement @(2, 1, 1) $matrix $dimensions + $element.parameters.operatingSystem | Should -Be "macOS-10.15" + $element.parameters.framework | Should -Be "netcoreapp2.1" + $element.parameters.additionalArguments | Should -Be "--enableFoo" + } + + It "Should initialize a sparse matrix from an N-dimensional matrix" -TestCases @( + @{ i = 0; name = "windows2019_net461"; operatingSystem = "windows-2019"; framework = "net461"; additionalArguments = ""; } + @{ i = 1; name = "ubuntu1804_netcoreapp21_withfoo"; operatingSystem = "ubuntu-18.04"; framework = "netcoreapp2.1"; additionalArguments = "--enableFoo"; } + @{ i = 2; name = "macOS1015_net461"; operatingSystem = "macOS-10.15"; framework = "net461"; additionalArguments = ""; } + ) { + $sparseMatrix = GenerateSparseMatrix $generateConfig.orderedMatrix $generateConfig.displayNamesLookup + $dimensions = GetMatrixDimensions $generateConfig.orderedMatrix + $size = ($dimensions | Measure-Object -Maximum).Maximum + $sparseMatrix.Count | Should -Be $size + + $sparseMatrix[$i].name | Should -Be $name + $element = $sparseMatrix[$i].parameters + $element.operatingSystem | Should -Be $operatingSystem + $element.framework | Should -Be $framework + $element.additionalArguments | Should -Be $additionalArguments + } + + It "Should generate a sparse matrix from an N-dimensional matrix config" { + $sparseMatrix = GenerateMatrix $generateConfig "sparse" + $sparseMatrix.Length | Should -Be 4 + } + + It "Should initialize a full matrix from an N-dimensional matrix config" { + $matrix = GenerateMatrix $generateConfig "all" + $matrix.Length | Should -Be 13 + } +} + +Describe "Config File Object Conversion" -Tag "convert" { + It "Should convert a matrix config" { + $converted = GetMatrixConfigFromJson $matrixConfig + + $converted.orderedMatrix | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + $converted.orderedMatrix.operatingSystem[0] | Should -Be "windows-2019" + + $converted.displayNamesLookup | Should -BeOfType [Hashtable] + $converted.displayNamesLookup["--enableFoo"] | Should -Be "withFoo" + + $converted.include | ForEach-Object { + $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + } + $converted.exclude | ForEach-Object { + $_ | Should -BeOfType [System.Collections.Specialized.OrderedDictionary] + } + } +} + +Describe "Platform Matrix Post Transformation" -Tag "transform" { + It "Should match partial matrix elements" -TestCases @( + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1 }; expected = $true } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1; b = 2 }; expected = $true } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ a = 1; b = 2; c = 3 }; expected = $false } + @{ source = [Ordered]@{ a = 1; b = 2; }; target = [Ordered]@{ }; expected = $false } + @{ source = [Ordered]@{ }; target = [Ordered]@{ a = 1; b = 2; }; expected = $false } + ) { + MatrixElementMatch $source $target | Should -Be $expected + } + + It "Should convert singular elements" { + $ordered = [Ordered]@{} + $ordered.Add("a", 1) + $ordered.Add("b", 2) + $matrix = ConvertToMatrixArrayFormat $ordered + $matrix.a.Length | Should -Be 1 + $matrix.b.Length | Should -Be 1 + + $ordered = [Ordered]@{} + $ordered.Add("a", 1) + $ordered.Add("b", @(1, 2)) + $matrix = ConvertToMatrixArrayFormat $ordered + $matrix.a.Length | Should -Be 1 + $matrix.b.Length | Should -Be 2 + + $ordered = [Ordered]@{} + $ordered.Add("a", @(1, 2)) + $ordered.Add("b", @()) + $matrix = ConvertToMatrixArrayFormat $ordered + $matrix.a.Length | Should -Be 2 + $matrix.b.Length | Should -Be 0 + } + + It "Should remove matrix elements based on exclude filters" { + $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + $withExclusion = ProcessExcludes $matrix $config.exclude + $withExclusion.Length | Should -Be 5 + + $matrix = GenerateSparseMatrix $config.orderedMatrix $config.displayNamesLookup + [array]$withExclusion = ProcessExcludes $matrix $config.exclude + $withExclusion.Length | Should -Be 1 + } + + It "Should add matrix elements based on include elements" { + $matrix = GenerateFullMatrix $config.orderedMatrix $config.displayNamesLookup + $withInclusion = ProcessIncludes $matrix $config.include $config.displayNamesLookup + $withInclusion.Length | Should -Be 15 + } + + It "Should include and exclude values with a matrix" { + [Array]$matrix = GenerateMatrix $config "all" + $matrix.Length | Should -Be 8 + + $matrix[0].name | Should -Be "windows2019_netcoreapp21" + $matrix[0].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[0].parameters.framework | Should -Be "netcoreapp2.1" + $matrix[0].parameters.additionalArguments | Should -Be "" + + $matrix[1].name | Should -Be "windows2019_netcoreapp21_withfoo" + $matrix[1].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[1].parameters.framework | Should -Be "netcoreapp2.1" + $matrix[1].parameters.additionalArguments | Should -Be "--enableFoo" + + $matrix[2].name | Should -Be "ubuntu1804_net461" + $matrix[2].parameters.framework | Should -Be "net461" + $matrix[2].parameters.operatingSystem | Should -Be "ubuntu-18.04" + $matrix[2].parameters.additionalArguments | Should -Be "" + + $matrix[4].name | Should -Be "macOS1015_net461" + $matrix[4].parameters.framework | Should -Be "net461" + $matrix[4].parameters.operatingSystem | Should -Be "macOS-10.15" + $matrix[4].parameters.additionalArguments | Should -Be "" + + $matrix[7].name | Should -Be "windows2019_net50_enableWindowsFoo" + $matrix[7].parameters.framework | Should -Be "net50" + $matrix[7].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[7].parameters.additionalArguments | Should -Be "--enableWindowsFoo" + } +} + +Describe "Platform Matrix Generation With Object Fields" -Tag "objectfields" { + BeforeEach { + $matrixConfigForObject = @" +{ + "matrix": { + "testObject": { + "testObjectName1": { "testObject1Value1": "1", "testObject1Value2": "2" }, + "testObjectName2": { "testObject2Value1": "1", "testObject2Value2": "2" } + }, + "secondTestObject": { + "secondTestObjectName1": { "secondTestObject1Value1": "1", "secondTestObject1Value2": "2" } + }, + "testField": [ "footest", "bartest" ] + }, + "include": [ + { + "testObjectInclude": { + "testObjectIncludeName": { "testObjectValue1": "1", "testObjectValue2": "2" } + }, + "testField": "footest" + } + ] +} +"@ + $objectFieldConfig = GetMatrixConfigFromJson $matrixConfigForObject + } + + It "Should parse dimensions properly" { + [Array]$dimensions = GetMatrixDimensions $objectFieldConfig.orderedMatrix + $dimensions.Length | Should -Be 3 + $dimensions[0] | Should -Be 2 + $dimensions[1] | Should -Be 1 + $dimensions[2] | Should -Be 2 + } + + It "Should populate a sparse matrix dimensions properly" { + [Array]$matrix = GenerateMatrix $objectFieldConfig "sparse" + $matrix.Length | Should -Be 3 + + $matrix[0].name | Should -Be "testObjectName1_secondTestObjectName1_footest" + $matrix[0].parameters.testField | Should -Be "footest" + $matrix[0].parameters.testObject1Value1 | Should -Be "1" + $matrix[0].parameters.testObject1Value2 | Should -Be "2" + $matrix[0].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[0].parameters.Count | Should -Be 5 + + $matrix[1].name | Should -Be "testObjectName2_secondTestObjectName1_bartest" + $matrix[1].parameters.testField | Should -Be "bartest" + $matrix[1].parameters.testObject2Value1 | Should -Be "1" + $matrix[1].parameters.testObject2Value2 | Should -Be "2" + $matrix[1].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[1].parameters.Count | Should -Be 5 + + $matrix[2].name | Should -Be "testObjectIncludeName_footest" + $matrix[2].parameters.testField | Should -Be "footest" + $matrix[2].parameters.testObjectValue1 | Should -Be "1" + $matrix[2].parameters.testObjectValue2 | Should -Be "2" + $matrix[2].parameters.Count | Should -Be 3 + } + + It "Should splat matrix entries that are objects into key/values" { + [Array]$matrix = GenerateMatrix $objectFieldConfig "all" + $matrix.Length | Should -Be 5 + + $matrix[0].name | Should -Be "testObjectName1_secondTestObjectName1_footest" + $matrix[0].parameters.testField | Should -Be "footest" + $matrix[0].parameters.testObject1Value1 | Should -Be "1" + $matrix[0].parameters.testObject1Value2 | Should -Be "2" + $matrix[0].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[0].parameters.Count | Should -Be 5 + + $matrix[3].name | Should -Be "testObjectName2_secondTestObjectName1_bartest" + $matrix[3].parameters.testField | Should -Be "bartest" + $matrix[3].parameters.testObject2Value1 | Should -Be "1" + $matrix[3].parameters.testObject2Value2 | Should -Be "2" + $matrix[3].parameters.secondTestObject1Value1 | Should -Be "1" + $matrix[3].parameters.Count | Should -Be 5 + + $matrix[4].name | Should -Be "testObjectIncludeName_footest" + $matrix[4].parameters.testField | Should -Be "footest" + $matrix[4].parameters.testObjectValue1 | Should -Be "1" + $matrix[4].parameters.testObjectValue2 | Should -Be "2" + $matrix[4].parameters.Count | Should -Be 3 + } +} diff --git a/eng/scripts/job-matrix/samples/matrix-test.yml b/eng/scripts/job-matrix/samples/matrix-test.yml new file mode 100644 index 000000000000..45d869753c1d --- /dev/null +++ b/eng/scripts/job-matrix/samples/matrix-test.yml @@ -0,0 +1,25 @@ +jobs: + - template: /eng/pipelines/templates/jobs/job-matrix.yml + parameters: + MatrixConfigs: + - Name: base_product_matrix + Path: eng/scripts/job-matrix/samples/matrix.json + Selection: all + - Name: sparse_product_matrix + Path: eng/scripts/job-matrix/samples/matrix.json + Selection: sparse + steps: + - pwsh: | + Write-Output "MATRIX JOB PARAMETERS" + Write-Output $(Agent.JobName) + Write-Output "-----------------" + Write-Output $(operatingSystem) + Write-Output $(framework) + try { + Write-Output $(additionalTestArguments) + } catch {} + displayName: Print matrix job variables + - pwsh: | + Write-Output "Success" + displayName: OS condition example + condition: and(succeededOrFailed(), contains(variables['operatingSystem'], 'windows')) diff --git a/eng/scripts/job-matrix/samples/matrix.json b/eng/scripts/job-matrix/samples/matrix.json new file mode 100644 index 000000000000..10d67730f0ec --- /dev/null +++ b/eng/scripts/job-matrix/samples/matrix.json @@ -0,0 +1,30 @@ +{ + "displayNames": { + "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1", + "net50" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": "net461", + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true" + } + ], + "exclude": [ + { + "operatingSystem": "windows-2019", + "framework": "net461" + } + ] +}