From cf89cfd5602b9eeb843cf807232c5e25ad3b0313 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 4 Dec 2020 07:57:30 -0500 Subject: [PATCH] Add cross product matrix tooling --- eng/common/matrix/Create-JobMatrix.ps1 | 23 + eng/common/matrix/Create-SparseJobMatrix.ps1 | 23 + eng/common/matrix/README.md | 242 ++++++++++ eng/common/matrix/functions.ps1 | 345 ++++++++++++++ eng/common/matrix/functions.tests.ps1 | 475 +++++++++++++++++++ eng/common/matrix/samples/matrix-test.yml | 33 ++ eng/common/matrix/samples/matrix.json | 30 ++ 7 files changed, 1171 insertions(+) create mode 100644 eng/common/matrix/Create-JobMatrix.ps1 create mode 100644 eng/common/matrix/Create-SparseJobMatrix.ps1 create mode 100644 eng/common/matrix/README.md create mode 100644 eng/common/matrix/functions.ps1 create mode 100644 eng/common/matrix/functions.tests.ps1 create mode 100644 eng/common/matrix/samples/matrix-test.yml create mode 100644 eng/common/matrix/samples/matrix.json diff --git a/eng/common/matrix/Create-JobMatrix.ps1 b/eng/common/matrix/Create-JobMatrix.ps1 new file mode 100644 index 00000000000..cbca9e3825d --- /dev/null +++ b/eng/common/matrix/Create-JobMatrix.ps1 @@ -0,0 +1,23 @@ +<# + .SYNOPSIS + Generates a JSON object representing an Azure Pipelines Job Matrix. + See https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#parallelexec + + .EXAMPLE + .\eng\scripts\Create-JobMatrix $context +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory=$True)][object] $ConfigPath +) + +. $PSScriptRoot/functions.ps1 + +$config = Get-Content $ConfigPath | ConvertFrom-Json -AsHashtable + +[Array]$matrix = GenerateMatrix $config "all" +$serialized = SerializePipelineMatrix $matrix + +Write-Output $serialized.pretty +Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" diff --git a/eng/common/matrix/Create-SparseJobMatrix.ps1 b/eng/common/matrix/Create-SparseJobMatrix.ps1 new file mode 100644 index 00000000000..594b9a48faf --- /dev/null +++ b/eng/common/matrix/Create-SparseJobMatrix.ps1 @@ -0,0 +1,23 @@ +<# + .SYNOPSIS + Generates a JSON object representing an Azure Pipelines Job Matrix. + See https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#parallelexec + + .EXAMPLE + .\eng\scripts\Create-JobMatrix $context +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory=$True)][object] $ConfigPath +) + +. $PSScriptRoot/functions.ps1 + +$config = Get-Content $ConfigPath | ConvertFrom-Json -AsHashtable + +[Array]$matrix = GenerateMatrix $config "sparse" +$serialized = SerializePipelineMatrix $matrix + +Write-Output $serialized.pretty +Write-Output "##vso[task.setVariable variable=matrix;isOutput=true]$($serialized.compressed)" diff --git a/eng/common/matrix/README.md b/eng/common/matrix/README.md new file mode 100644 index 00000000000..2b184e8f180 --- /dev/null +++ b/eng/common/matrix/README.md @@ -0,0 +1,242 @@ +# Azure Pipelines Matrix Generator + +* [Usage in a pipeline](#usage-in-a-pipeline) +* [Matrix config file syntax](#matrix-config-file-syntax) + * [Fields](#fields) + * [matrix](#matrix) + * [include](#include) + * [exclude](#exclude) + * [displayNames](#displaynames) +* [Matrix Generation behavior](#matrix-generation-behavior) + * [all](#all) + * [sparse](#sparse) + * [include/exclude](#includeexclude) + * [displayNames](#displaynames-1) + * [Filters](#filters) + * [Under the hood](#under-the-hood) + + +This directory contains scripts supporting dynamic, cross-product matrix generation for azure pipeline jobs. +It aims to replicate the [cross-product matrix functionality in github actions](https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#example-running-with-more-than-one-version-of-nodejs), +but also adds some additional features like sparse matrix generation, cross-product includes and excludes, and programmable matrix filters. + +This functionality is made possible by the ability for the azure pipelines yaml to take a [dynamic variable as an input +for a job matrix definition](https://docs.microsoft.com/en-us/azure/devops/pipelines/process/phases?view=azure-devops&tabs=yaml#multi-job-configuration) (see the code sample at the bottom of the linked section). + +## Usage in a pipeline + +In order to use these scripts in a pipeline, you must provide a config file and call the matrix creation script within a powershell job. +The job must then be marked as a dependency by a subsequent matrix job, and the matrix value pulled from a variable set by the powershell job. +For example: + +``` +parameters: + - name: ProductMatrix + type: string + default: 'matrix.json' + +jobs: + - job: generate_matrix + steps: + - pwsh: | + eng/common/matrix/Create-JobMatrix.ps1 -ConfigPath ${{ parameters.ProductMatrix }} + name: generate_job_matrix + displayName: Generate Job Matrix + + - job: + dependsOn: generate_matrix + strategy: + maxParallel: 0 + matrix: $[ dependencies.generate_matrix.outputs['generate_job_matrix.matrix'] ] + steps: + ... +``` + +The generate_matrix job will log the matrix json for debugging, and run the task.setVariable vso command to add the generated matrix to the variable context. + +## Matrix config file syntax + +``` +"matrix": { + "": [ ], + "": [ ] +} +"include": [ , , ... ], +"exclude": [ , , ... ], +"displayNames": { : } +``` + +See `samples/matrix.json` for a full sample. + +### Fields + +#### matrix + +The `matrix` field defines the base cross-product matrix. The generated matrix can be full or sparse. + +Example: +``` +"matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1", + "net50" + ], + "additionalTestArguments": [ + "", + "/p:UseProjectReferenceToAzureClients=true", + ] +} +``` + +#### include + +The `include` field defines any number of matrices to be appended to the base matrix after processing exclusions. + +#### exclude + +The `include` field defines any number of matrices to be removed from the base matrix. Exclude parameters can be a partial +set, meaning as long as all exclude parameters match against a matrix entry (even if the matrix entry has additional parameters), +then it will be excluded from the matrix. For example, the below entry will match the exclusion and be removed: + +``` +matrix entry: +{ + "a": 1, + "b": 2, + "c": 3, +} + +"exclude": [ + { + "a": 1, + "b": 2 + } +] +``` + +#### displayNames + +Specify any overrides for the azure pipelines definition and UI that determines the matrix job name. If some parameter +values are too long or unreadable for this purpose (e.g. a command line argument), then you can replace them with a more +readable value here. For example: + +``` +"displayNames": { + "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef" +}, +"matrix": { + "additionalTestArguments": [ + "/p:UseProjectReferenceToAzureClients=true" + ] +} +``` + +## Matrix Generation behavior + +#### all + +`all` will output the full matrix, i.e. every possible permutation of all parameters given (p1.Length * p2.Length * ...). + +#### sparse + +`sparse` outputs the minimum number of parameter combinations while ensuring that all parameter values are present in at least one matrix job. +Effectively this means the total length of a sparse matrix will be equal to the largest matrix dimension, i.e. `max(p1.Length, p2.Length, ...)`. + +To build a sparse matrix, a full matrix is generated, and then walked diagonally N times where N is the largest matrix dimension. +This pattern works for any N-dimensional matrix, via an incrementing index (n, n, n, ...), (n+1, n+1, n+1, ...), etc. +Index lookups against matrix dimensions are calculated modulus the dimension size, so a two-dimensional matrix of 4x2 might be walked like this: + +``` +index: 0, 0: +o . . . +. . . . + +index: 1, 1: +. . . . +. o . . + +index: 2, 2 (modded to 2, 0): +. . o . +. . . . + +index: 3, 3 (modded to 3, 1): +. . . . +. . . o +``` + +#### include/exclude + +Include and exclude support additions and subtractions off the base matrix. Both include and exclude take an array of matrix values. +Typically these values will be a single entry, but they also support the cross-product matrix definition syntax of the base matrix. + +Include and exclude are parsed fully. So if a sparse matrix is called for, a sparse version of the base matrix will be generated, but +the full matrix of both include and exclude will be processed. + +Excludes are processed first, so includes can be used to add back any specific jobs to the matrix. + +#### displayNames + +In the matrix job output that azure pipelines consumes, the format is a dictionary of dictionaries. For example: + +``` +{ + "net461_macOS1015": { + "framework": "net461", + "operatingSystem": "macOS-10.15" + }, + "net50_ubuntu1804": { + "framework": "net50", + "operatingSystem": "ubuntu-18.04" + }, + "netcoreapp21_windows2019": { + "framework": "netcoreapp2.1", + "operatingSystem": "windows-2019" + }, + "UseProjectRef_net461_windows2019": { + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true", + "framework": "net461", + "operatingSystem": "windows-2019" + } +} +``` + +The top level keys are used as job names, meaning they get displayed in the azure pipelines UI when running the pipeline. + +The logic for generating display names works like this: + +1. Sort the matrix by: parameter length, parameter name, and sort the parameter values arrays. +2. In sorted order, join parameter values by "_" + a. If the parameter value exists as a key in `displayNames` in the matrix config, replace it with that value. + b. For each name value, strip all `.-_` characters from the string. + +The matrix is then sorted by display name, before being sent to azure pipelines. The underlying matrix may have a different +sorted order than the display name output. The sorting needs to be separate so that sparse matrix generation can be deterministic. + +#### Filters + +To be implemented. A basic example of a filter is "all" vs. "sparse" generation, but eventually will be a more programmable +way of processing excludes, such as an expression that only includes entries with a container image specified. The intent +is that these filters can be entered at runtime, as opposed to statically in yaml. + +#### Under the hood + +The script generates an N-dimensional matrix with dimensions equal to the parameter array lengths. For example, +the below config would generate a 2x2x1x1x1 matrix (five-dimensional): + +``` +"matrix": { + "framework": [ "net461", "netcoreapp2.1" ], + "additionalTestArguments": [ "", "/p:SuperTest=true" ] + "pool": [ "ubuntu-18.04" ], + "container": [ "ubuntu-18.04" ], + "testMode": [ "Record" ] +} +``` + +The matrix is stored as a one-dimensional array, with a row-major indexing scheme (e.g. `(2, 1, 0, 1, 0)`). diff --git a/eng/common/matrix/functions.ps1 b/eng/common/matrix/functions.ps1 new file mode 100644 index 00000000000..e8c7fa092a5 --- /dev/null +++ b/eng/common/matrix/functions.ps1 @@ -0,0 +1,345 @@ +Set-StrictMode -Version "4.0" + +function CreateDisplayName { + param([string]$parameter, [hashtable]$displayNames) + + $name = $parameter + + if ($displayNames[$parameter]) { + $name = $displayNames[$parameter] + } + + return $name -replace "[\-\._]", "" +} + +function GenerateMatrix { + param([HashTable]$config, [string]$selectFromMatrixType) + + if ($selectFromMatrixType -eq "sparse") { + [Array]$dimensions = GetMatrixDimensions $config.matrix + $size = $dimensions[0] + [Array]$matrix = GenerateSparseMatrix $config.matrix $config.displayNames $size + } elseif ($selectFromMatrixType -eq "all") { + [Array]$matrix, $_ = GenerateFullMatrix $config.matrix $config.displayNames + } else { + throw "Matrix generator not implemented for selectFromMatrix: $($platform.selectFromMatrix)" + } + + if ($config["exclude"]) { + [Array]$matrix = ProcessExcludes $matrix $config.exclude + } + if ($config["include"]) { + [Array]$matrix = ProcessIncludes $matrix $config.include $config.displayNames + } + + return $matrix | Sort-Object -Property name +} + +function ProcessExcludes { + param([Array]$matrix, [Array]$excludes) + + $deleteKey = "%DELETE%" + $exclusionMatrix = @() + + foreach ($exclusion in $excludes) { + $converted = ConvertToMatrixArrayFormat $exclusion + $full, $_ = GenerateFullMatrix $converted + $exclusionMatrix += $full + } + + foreach ($element in $matrix) { + foreach ($exclusion in $exclusionMatrix) { + $match = MatrixElementMatch $element.parameters $exclusion.parameters + if ($match) { + $element.parameters[$deleteKey] = $true + } + } + } + + return $matrix | Where-Object { !$_.parameters.Contains($deleteKey) } +} + +function ProcessIncludes { + param([Array]$matrix, [Array]$includes, [HashTable]$displayNames) + + foreach ($inclusion in $includes) { + $converted = ConvertToMatrixArrayFormat $inclusion + $full, $_ = GenerateFullMatrix $converted $displayNames + $matrix += $full + } + + return $matrix +} + +function MatrixElementMatch { + param([HashTable]$source, [HashTable]$target) + + if ($target.Count -eq 0) { + return $false + } + + foreach ($key in $target.Keys) { + if ($source[$key] -ne $target[$key]) { + return $false + } + } + + return $true +} + +function ConvertToMatrixArrayFormat { + param([HashTable]$matrix) + + $converted = @{} + + foreach ($key in $matrix.Keys) { + if ($matrix[$key] -isnot [Array]) { + $converted[$key] = ,$matrix[$key] + } else { + $converted[$key] = $matrix[$key] + } + } + + return $converted +} + +function SerializePipelineMatrix { + param([Array]$matrix) + + $matrix = $matrix | Sort-Object -Property name + $pipelineMatrix = [ordered]@{} + foreach ($entry in $matrix) { + $pipelineMatrix.Add($entry.name, [ordered]@{}) + foreach ($key in $entry.parameters.Keys) { + $pipelineMatrix[$entry.name].Add($key, $entry.parameters[$key]) + } + } + + return @{ + compressed = $pipelineMatrix | ConvertTo-Json -Compress ; + pretty = $pipelineMatrix | ConvertTo-Json; + } +} + +function GenerateSparseMatrix { + param([HashTable]$parameters, [HashTable]$displayNames, [int]$count) + + [Array]$matrix, [Array]$dimensions = GenerateFullMatrix $parameters $displayNames + $sparseMatrix = @() + + # With full matrix, retrieve items by doing diagonal lookups across the matrix N times. + # For example, given a matrix with dimensions 3, 2, 2: + # 0, 0, 0 + # 1, 1, 1 + # 2, 2, 2 + # 3, 0, 0 <- 3, 3, 3 wraps to 3, 0, 0 given the dimensions + for ($i = 0; $i -lt $count; $i++) { + $idx = @() + for ($j = 0; $j -lt $dimensions.Length; $j++) { + $idx += $i % $dimensions[$j] + } + $sparseMatrix += GetNdMatrixElement $idx $matrix $dimensions + } + + return $sparseMatrix +} + +function GenerateFullMatrix { + param([HashTable]$parameters, [HashTable]$displayNames = @{}) + + $sortedParameters = SortMatrix $parameters + + $matrix = [System.Collections.ArrayList]::new() + InitializeMatrix $sortedParameters $displayNames $matrix + + return $matrix.ToArray(), (GetMatrixDimensions $parameters) +} + +# SortMatrix sorts a matrix by three properties: +# 1. Descending element length or parameters +# 2. Ascending parameter names alphabetically +# 3. Ascending parameter values alphabetically +# +# The matrix must be sorted in order to have a deterministic layout. +# This guarantees that a sparse matrix will always be generated with the +# same set of elements given the same input. +# +# Additionally, parameter value arrays should be sorted, so that the azure pipelines +# matrix yaml gets generated consistently. +function SortMatrix { + param([HashTable]$parameters) + + $sortedParameters = $parameters.GetEnumerator() ` + | Sort-Object -Property ` + @{ Expression = { $_.Value.Length }; Descending=$true }, ` + @{ Expression = { $_.Name }; Descending=$false } + + for ($i = 0; $i -lt $sortedParameters.Length; $i++) { + $sortedParameters[$i].Value = $sortedParameters[$i].Value | Sort-Object + } + + return $sortedParameters +} + +function CreateMatrixEntry { + param([System.Collections.Specialized.OrderedDictionary]$permutation, [HashTable]$displayNames = @{}) + + $names = @() + foreach ($key in $permutation.Keys) { + $nameSegment = CreateDisplayName $permutation[$key] $displayNames + if ($nameSegment) { + $names += $nameSegment + } + } + return @{ + name = $names -join "_" + parameters = $permutation + } +} + +function InitializeMatrix { + param( + [Array]$parameters, + [HashTable]$displayNames, + [System.Collections.ArrayList]$permutations, + [System.Collections.Specialized.OrderedDictionary]$permutation = @{} + ) + + if (!$parameters) { + $entry = CreateMatrixEntry $permutation $displayNames + $permutations.Add($entry) | Out-Null + return + } + + $head, $tail = $parameters + foreach ($value in $head.value) { + $newPermutation = [ordered]@{} + foreach ($element in $permutation.GetEnumerator()) { + $newPermutation[$element.Name] = $element.Value + } + $newPermutation[$head.name] = $value + InitializeMatrix $tail $displayNames $permutations $newPermutation + } +} + +function GetMatrixDimensions { + param([HashTable]$parameters) + + $dimensions = @() + foreach ($param in $parameters.GetEnumerator()) { + $dimensions += $param.Value.Length + } + + return $dimensions | Sort-Object -Descending +} + +function SetNdMatrixElement { + param( + $element, + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query $($idx.Length) must be the same length as its dimensions $($dimensions.Length)" + } + + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + $matrix[$arrayIndex] = $element +} + +function GetNdMatrixArrayIndex { + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + if ($idx.Length -ne $dimensions.Length) { + throw "Matrix index query length ($($idx.Length)) must be the same as dimension length ($($dimensions.Length))" + } + + $stride = 1 + # Commented out does lookup with wrap handling + # $index = $idx[$idx.Length-1] % $dimensions[$idx.Length-1] + $index = $idx[$idx.Length-1] + + for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + # Commented out does lookup with wrap handling + # $index += ($idx[$i-1] % $dimensions[$i-1]) * $stride + $index += $idx[$i-1] * $stride + } + + return $index +} + +function GetNdMatrixElement { + param( + [ValidateNotNullOrEmpty()] + [Array]$idx, + [ValidateNotNullOrEmpty()] + [Array]$matrix, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + $arrayIndex = GetNdMatrixArrayIndex $idx $dimensions + return $matrix[$arrayIndex] +} + +function GetNdMatrixIndex { + param( + [int]$index, + [ValidateNotNullOrEmpty()] + [Array]$dimensions + ) + + $matrixIndex = @() + $stride = 1 + + for ($i = $dimensions.Length-1; $i -ge 1; $i--) { + $stride *= $dimensions[$i] + $page = [math]::floor($index / $stride) % $dimensions[$i-1] + $matrixIndex = ,$page + $matrixIndex + } + $col = $index % $dimensions[$dimensions.Length-1] + $matrixIndex += $col + + return $matrixIndex +} + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +# The below functions are non-dynamic examples that # +# help explain the above N-dimensional algorithm # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # +function Get4dMatrixElement { + param([Array]$idx, [Array]$matrix, [Array]$dimensions) + + $stride1 = $idx[0] * $dimensions[1] * $dimensions[2] * $dimensions[3] + $stride2 = $idx[1] * $dimensions[2] * $dimensions[3] + $stride3 = $idx[2] * $dimensions[3] + $stride4 = $idx[3] + + return $matrix[$stride1 + $stride2 + $stride3 + $stride4] +} + +function Get4dMatrixIndex { + param([int]$index, [Array]$dimensions) + + $stride1 = $dimensions[3] + $stride2 = $dimensions[2] + $stride3 = $dimensions[1] + $page1 = [math]::floor($index / $stride1) % $dimensions[2] + $page2 = [math]::floor($index / ($stride1 * $stride2)) % $dimensions[1] + $page3 = [math]::floor($index / ($stride1 * $stride2 * $stride3)) % $dimensions[0] + $remainder = $index % $dimensions[3] + + return @($page3, $page2, $page1, $remainder) +} + diff --git a/eng/common/matrix/functions.tests.ps1 b/eng/common/matrix/functions.tests.ps1 new file mode 100644 index 00000000000..36b84cfb5c7 --- /dev/null +++ b/eng/common/matrix/functions.tests.ps1 @@ -0,0 +1,475 @@ +Import-Module Pester + +BeforeAll { + . $PSScriptRoot/functions.ps1 + + $matrixConfig = @" +{ + "displayNames": { + "--enableFoo": "withfoo" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1" + ], + "additionalArguments": [ + "", + "--enableFoo" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": ["net461", "netcoreapp2.1", "net50"], + "additionalArguments": "--enableWindowsFoo" + } + ], + "exclude": [ + { + "operatingSystem": "windows-2019", + "framework": "net461" + }, + { + "operatingSystem": "macOS-10.15", + "framework": "netcoreapp2.1" + }, + { + "operatingSystem": ["macOS-10.15", "ubuntu-18.04"], + "additionalArguments": "--enableFoo" + } + ] +} +"@ + $config = $matrixConfig | ConvertFrom-Json -AsHashtable +} + +Describe "Matrix-Lookup" -Tag "lookup" { + It "Should navigate a 2d matrix: " -TestCases @( + @{ row = 0; col = 0; expected = 1 }, + @{ row = 0; col = 1; expected = 2 }, + @{ row = 1; col = 0; expected = 3 }, + @{ row = 1; col = 1; expected = 4 } + ) { + $dimensions = @(2, 2) + $matrix = @( + 1, 2, 3, 4 + ) + GetNdMatrixElement @($row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 3d matrix: " -TestCases @( + @{ z = 0; row = 0; col = 0; expected = 1 } + @{ z = 0; row = 0; col = 1; expected = 2 } + @{ z = 0; row = 1; col = 0; expected = 3 } + @{ z = 0; row = 1; col = 1; expected = 4 } + @{ z = 1; row = 0; col = 0; expected = 5 } + @{ z = 1; row = 0; col = 1; expected = 6 } + @{ z = 1; row = 1; col = 0; expected = 7 } + @{ z = 1; row = 1; col = 1; expected = 8 } + ) { + $dimensions = @(2, 2, 2) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8 + ) + GetNdMatrixElement @($z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 4d matrix: " -TestCases @( + @{ t = 0; z = 0; row = 0; col = 0; expected = 1 } + @{ t = 0; z = 0; row = 0; col = 1; expected = 2 } + @{ t = 0; z = 0; row = 1; col = 0; expected = 3 } + @{ t = 0; z = 0; row = 1; col = 1; expected = 4 } + @{ t = 0; z = 1; row = 0; col = 0; expected = 5 } + @{ t = 0; z = 1; row = 0; col = 1; expected = 6 } + @{ t = 0; z = 1; row = 1; col = 0; expected = 7 } + @{ t = 0; z = 1; row = 1; col = 1; expected = 8 } + @{ t = 1; z = 0; row = 0; col = 0; expected = 9 } + @{ t = 1; z = 0; row = 0; col = 1; expected = 10 } + @{ t = 1; z = 0; row = 1; col = 0; expected = 11 } + @{ t = 1; z = 0; row = 1; col = 1; expected = 12 } + @{ t = 1; z = 1; row = 0; col = 0; expected = 13 } + @{ t = 1; z = 1; row = 0; col = 1; expected = 14 } + @{ t = 1; z = 1; row = 1; col = 0; expected = 15 } + @{ t = 1; z = 1; row = 1; col = 1; expected = 16 } + ) { + $dimensions = @(2, 2, 2, 2) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ) + GetNdMatrixElement @($t, $z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + It "Should navigate a 4d matrix: " -TestCases @( + @{ t = 0; z = 0; row = 0; col = 0; expected = 1 } + @{ t = 0; z = 0; row = 0; col = 1; expected = 2 } + @{ t = 0; z = 0; row = 0; col = 2; expected = 3 } + @{ t = 0; z = 0; row = 0; col = 3; expected = 4 } + + @{ t = 0; z = 0; row = 1; col = 0; expected = 5 } + @{ t = 0; z = 0; row = 1; col = 1; expected = 6 } + @{ t = 0; z = 0; row = 1; col = 2; expected = 7 } + @{ t = 0; z = 0; row = 1; col = 3; expected = 8 } + + @{ t = 0; z = 1; row = 0; col = 0; expected = 9 } + @{ t = 0; z = 1; row = 0; col = 1; expected = 10 } + @{ t = 0; z = 1; row = 0; col = 2; expected = 11 } + @{ t = 0; z = 1; row = 0; col = 3; expected = 12 } + + @{ t = 0; z = 1; row = 1; col = 0; expected = 13 } + @{ t = 0; z = 1; row = 1; col = 1; expected = 14 } + @{ t = 0; z = 1; row = 1; col = 2; expected = 15 } + @{ t = 0; z = 1; row = 1; col = 3; expected = 16 } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ) + GetNdMatrixElement @($t, $z, $row, $col) $matrix $dimensions | Should -Be $expected + } + + # Skipping since by default wrapping behavior on indexing is disabled. + # Keeping here in case we want to enable it. + It -Skip "Should handle index wrapping: " -TestCases @( + @{ row = 2; col = 2; expected = 1 } + @{ row = 2; col = 3; expected = 2 } + @{ row = 4; col = 4; expected = 1 } + @{ row = 4; col = 5; expected = 2 } + ) { + $dimensions = @(2, 2) + $matrix = @( + 1, 2, 3, 4 + ) + GetNdMatrixElement @($row, $col) $matrix $dimensions | Should -Be $expected + } +} + +Describe "Matrix-Reverse-Lookup" -Tag "lookup" { + It "Should lookup a 2d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0) } + @{ index = 1; expected = @(0,1) } + @{ index = 2; expected = @(1,0) } + @{ index = 3; expected = @(1,1) } + ) { + $dimensions = @(2, 2) + $matrix = @(1, 2, 3, 4) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,1,0) } + @{ index = 3; expected = @(0,1,1) } + + @{ index = 4; expected = @(1,0,0) } + @{ index = 5; expected = @(1,0,1) } + @{ index = 6; expected = @(1,1,0) } + @{ index = 7; expected = @(1,1,1) } + ) { + $dimensions = @(2, 2, 2) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,0,2) } + + @{ index = 3; expected = @(0,1,0) } + @{ index = 4; expected = @(0,1,1) } + @{ index = 5; expected = @(0,1,2) } + + @{ index = 6; expected = @(1,0,0) } + @{ index = 7; expected = @(1,0,1) } + @{ index = 8; expected = @(1,0,2) } + + @{ index = 9; expected = @(1,1,0) } + @{ index = 10; expected = @(1,1,1) } + @{ index = 11; expected = @(1,1,2) } + ) { + $dimensions = @(2, 2, 3) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 3d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0) } + @{ index = 1; expected = @(0,0,1) } + @{ index = 2; expected = @(0,1,0) } + @{ index = 3; expected = @(0,1,1) } + + @{ index = 4; expected = @(1,0,0) } + @{ index = 5; expected = @(1,0,1) } + @{ index = 6; expected = @(1,1,0) } + @{ index = 7; expected = @(1,1,1) } + + @{ index = 8; expected = @(2,0,0) } + @{ index = 9; expected = @(2,0,1) } + @{ index = 10; expected = @(2,1,0) } + @{ index = 11; expected = @(2,1,1) } + ) { + $dimensions = @(3, 2, 2) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } + + It "Should lookup a 4d matrix index: " -TestCases @( + @{ index = 0; expected = @(0,0,0,0) } + @{ index = 1; expected = @(0,0,0,1) } + @{ index = 2; expected = @(0,0,0,2) } + @{ index = 3; expected = @(0,0,0,3) } + + @{ index = 4; expected = @(0,0,1,0) } + @{ index = 5; expected = @(0,0,1,1) } + @{ index = 6; expected = @(0,0,1,2) } + @{ index = 7; expected = @(0,0,1,3) } + + @{ index = 8; expected = @(0,1,0,0) } + @{ index = 9; expected = @(0,1,0,1) } + @{ index = 10; expected = @(0,1,0,2) } + @{ index = 11; expected = @(0,1,0,3) } + + @{ index = 12; expected = @(0,1,1,0) } + @{ index = 13; expected = @(0,1,1,1) } + @{ index = 14; expected = @(0,1,1,2) } + @{ index = 15; expected = @(0,1,1,3) } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + GetNdMatrixElement $expected $matrix $dimensions | Should -Be $matrix[$index] + GetNdMatrixIndex $index $dimensions | Should -Be $expected + } +} + +Describe 'Matrix-Set' -Tag "set" { + It "Should set a matrix element" -TestCases @( + @{ value = "set"; index = @(0,0,0,0); arrayIndex = 0 } + @{ value = "ones"; index = @(0,1,1,1); arrayIndex = 13 } + ) { + $dimensions = @(1, 2, 2, 4) + $matrix = @(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) + + SetNdMatrixElement $value $index $matrix $dimensions + $matrix[$arrayIndex] | Should -Be $value + } +} + +Describe "Platform Matrix Generation" -Tag "generate" { + BeforeEach { + $matrixConfig = @" +{ + "displayNames": { + "--enableFoo": "withfoo" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1" + ], + "additionalArguments": [ + "", + "--enableFoo" + ] + } +} +"@ + $config = $matrixConfig | ConvertFrom-Json -AsHashtable + } + + It "Should get matrix dimensions from Nd parameters" { + GetMatrixDimensions $config.matrix | Should -Be 3, 2, 2 + } + + It "Should use name overrides from displayNames" { + $matrix, $dimensions = GenerateFullMatrix $config.matrix $config.displayNames + + $element = GetNdMatrixElement @(0, 0, 0) $matrix $dimensions + $element.name | Should -Be "macOS1015_net461" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.name | Should -Be "ubuntu1804_withFoo_netcoreapp21" + + $element = GetNdMatrixElement @(2, 1, 1) $matrix $dimensions + $element.name | Should -Be "windows2019_withFoo_netcoreapp21" + } + + It "Should initialize an N-dimensional matrix from all parameter permutations" { + $matrix, $dimensions = GenerateFullMatrix $config.matrix $config.displayNames + $matrix.Count | Should -Be 12 + + $element = $matrix[0].parameters + $element.operatingSystem | Should -Be "macOS-10.15" + $element.framework | Should -Be "net461" + $element.additionalArguments | Should -Be "" + + $element = GetNdMatrixElement @(1, 1, 1) $matrix $dimensions + $element.parameters.operatingSystem | Should -Be "ubuntu-18.04" + $element.parameters.framework | Should -Be "netcoreapp2.1" + $element.parameters.additionalArguments | Should -Be "--enableFoo" + + $element = GetNdMatrixElement @(2, 1, 1) $matrix $dimensions + $element.parameters.operatingSystem | Should -Be "windows-2019" + $element.parameters.framework | Should -Be "netcoreapp2.1" + $element.parameters.additionalArguments | Should -Be "--enableFoo" + } + + It "Should initialize a sparse matrix of X length from an N-dimensional matrix" -TestCases @( + @{ size = 3; i = 0; name = "macOS1015_net461"; operatingSystem = "macOS-10.15"; framework = "net461"; additionalArguments = ""; } + @{ size = 5; i = 0; name = "macOS1015_net461"; operatingSystem = "macOS-10.15"; framework = "net461"; additionalArguments = ""; } + @{ size = 4; i = 1; name = "ubuntu1804_withFoo_netcoreapp21"; operatingSystem = "ubuntu-18.04"; framework = "netcoreapp2.1"; additionalArguments = "--enableFoo"; } + @{ size = 5; i = 2; name = "windows2019_net461"; operatingSystem = "windows-2019"; framework = "net461"; additionalArguments = ""; } + @{ size = 5; i = 4; name = "ubuntu1804_net461"; operatingSystem = "ubuntu-18.04"; framework = "net461"; additionalArguments = ""; } + ) { + $sparseMatrix = GenerateSparseMatrix $config.matrix $config.displayNames $size + $sparseMatrix.Count | Should -Be $size + + $sparseMatrix[$i].name | Should -Be $name + $element = $sparseMatrix[$i].parameters + $element.operatingSystem | Should -Be $operatingSystem + $element.framework | Should -Be $framework + $element.additionalArguments | Should -Be $additionalArguments + } + + It "Should generate a sparse matrix from an N-dimensional matrix config" { + $sparseMatrix = GenerateMatrix $config "sparse" + $sparseMatrix.Length | Should -Be 3 + } + + It "Should initialize a full matrix from an N-dimensional matrix config" { + $matrix = GenerateMatrix $config "all" + $matrix.Length | Should -Be 12 + } +} + +Describe "Matrix sort" -Tag "sort" { + It "Should sort matrix parameters by length first, then name and values alphabetically" { + $matrix = @{ + "bbb" = @("b", "a", "c") + "aaa" = @("d", "b", "c") + "zzz" = @("d", "b", "c", "a") + } + $expected = @( + @{ Name = "zzz"; Value = @("a", "b", "c", "d") } + @{ Name = "aaa"; Value = @("b", "c", "d") } + @{ Name = "bbb"; Value = @("a", "b", "c") } + ) + $sorted = SortMatrix $matrix + + for ($i = 0; $i -lt $matrix.Count; $i++) { + $sorted[$i].Name | Should -Be $expected[$i].Name + $sorted[$i].Value | Should -Be $expected[$i].Value + } + } + + It "Should sort matrix dimensions descending" { + $matrix = @{ + "bbb" = @("b", "a", "c") + "aaa" = @("d", "b", "c") + "zzz" = @("d", "b", "c", "a") + } + GetMatrixDimensions $matrix | Should -Be @(4,3,3) + } + + It "Should sort matrix output array by display name" { + $output = GenerateMatrix $config "all" + + for ($i = 0; $i -lt $output.Length-1; $i++) { + $output[$i].name | Should -BeLessThan $output[$i+1].name + } + } +} + +Describe "Platform Matrix Post Transformation" -Tag "transform" { + It "Should match partial matrix elements" -TestCases @( + @{ source = @{ a = 1; b = 2; }; target = @{ a = 1 }; expected = $true } + @{ source = @{ a = 1; b = 2; }; target = @{ a = 1; b = 2 }; expected = $true } + @{ source = @{ a = 1; b = 2; }; target = @{ a = 1; b = 2; c = 3 }; expected = $false } + @{ source = @{ a = 1; b = 2; }; target = @{ }; expected = $false } + @{ source = @{ }; target = @{ a = 1; b = 2; }; expected = $false } + ) { + MatrixElementMatch $source $target | Should -Be $expected + } + + It "Should convert singular elements" { + $matrix = ConvertToMatrixArrayFormat @{ a = 1; b = 2 } + $matrix.a.Length | Should -Be 1 + $matrix.b.Length | Should -Be 1 + + $matrix = ConvertToMatrixArrayFormat @{ a = 1; b = @(1, 2) } + $matrix.a.Length | Should -Be 1 + $matrix.b.Length | Should -Be 2 + + $matrix = ConvertToMatrixArrayFormat @{ a = @(1, 2); b = @() } + $matrix.a.Length | Should -Be 2 + $matrix.b.Length | Should -Be 0 + } + + It "Should remove matrix elements based on exclude filters" { + $matrix, $dimensions = GenerateFullMatrix $config.matrix $config.displayNames + $withExclusion = ProcessExcludes $matrix $config.exclude + $withExclusion.Length | Should -Be 5 + + $dimensions = GetMatrixDimensions $config.matrix + $size = $dimensions[0] + $matrix, $dimensions = GenerateSparseMatrix $config.matrix $config.displayNames $size + [array]$withExclusion = ProcessExcludes $matrix $config.exclude + $withExclusion.Length | Should -Be 1 + } + + It "Should add matrix elements based on include elements" { + $matrix, $dimensions = GenerateFullMatrix $config.matrix $config.displayNames + $withInclusion = ProcessIncludes $matrix $config.include $config.displayNames + $withInclusion.Length | Should -Be 15 + } + + It "Should include and exclude values with a matrix" { + [Array]$matrix = GenerateMatrix $config "all" + $matrix.Length | Should -Be 8 + + $matrix[0].name | Should -Be "macOS1015_net461" + $matrix[0].parameters.operatingSystem | Should -Be "macOS-10.15" + $matrix[0].parameters.framework | Should -Be "net461" + $matrix[0].parameters.additionalArguments | Should -Be "" + + # Includes should get sorted along with base matrix + # The naming segment hierarchy is different because the includes + # get sorted differently, since parameter lengths are different + # (e.g. operatingSystem.Length = 1 vs. 3) + + $matrix[1].name | Should -Be "net461_enableWindowsFoo_windows2019" + $matrix[1].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[1].parameters.framework | Should -Be "net461" + $matrix[1].parameters.additionalArguments | Should -Be "--enableWindowsFoo" + + $matrix[2].name | Should -Be "net50_enableWindowsFoo_windows2019" + $matrix[2].parameters.framework | Should -Be "net50" + $matrix[2].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[2].parameters.additionalArguments | Should -Be "--enableWindowsFoo" + + $matrix[5].name | Should -Be "ubuntu1804_netcoreapp21" + $matrix[5].parameters.framework | Should -Be "netcoreapp2.1" + $matrix[5].parameters.operatingSystem | Should -Be "ubuntu-18.04" + $matrix[5].parameters.additionalArguments | Should -Be "" + + $matrix[7].name | Should -Be "windows2019_withFoo_netcoreapp21" + $matrix[7].parameters.framework | Should -Be "netcoreapp2.1" + $matrix[7].parameters.operatingSystem | Should -Be "windows-2019" + $matrix[7].parameters.additionalArguments | Should -Be "--enableFoo" + } +} diff --git a/eng/common/matrix/samples/matrix-test.yml b/eng/common/matrix/samples/matrix-test.yml new file mode 100644 index 00000000000..885cb4695aa --- /dev/null +++ b/eng/common/matrix/samples/matrix-test.yml @@ -0,0 +1,33 @@ +parameters: + - name: ProductMatrix + type: string + default: 'matrix.json' + +jobs: + - job: generate_matrix + steps: + - pwsh: | + eng/common/matrix/Create-JobMatrix.ps1 -ConfigPath ${{ parameters.ProductMatrix }} + name: generate_job_matrix + displayName: Generate Job Matrix + + - job: + dependsOn: generate_matrix + strategy: + maxParallel: 0 + matrix: $[ dependencies.generate_matrix.outputs['generate_job_matrix.matrix'] ] + steps: + - pwsh: | + Write-Output "SPARSE MATRIX JOB PARAMETERS" + Write-Output $(Agent.JobName) + Write-Output "-----------------" + Write-Output $(operatingSystem) + Write-Output $(framework) + try { + Write-Output $(additionalTestArguments) + } catch {} + displayName: Print matrix job variables + - pwsh: | + Write-Output "Success" + displayName: OS condition example + condition: and(succeededOrFailed(), contains(variables['operatingSystem'], 'windows')) diff --git a/eng/common/matrix/samples/matrix.json b/eng/common/matrix/samples/matrix.json new file mode 100644 index 00000000000..10d67730f0e --- /dev/null +++ b/eng/common/matrix/samples/matrix.json @@ -0,0 +1,30 @@ +{ + "displayNames": { + "/p:UseProjectReferenceToAzureClients=true": "UseProjectRef" + }, + "matrix": { + "operatingSystem": [ + "windows-2019", + "ubuntu-18.04", + "macOS-10.15" + ], + "framework": [ + "net461", + "netcoreapp2.1", + "net50" + ] + }, + "include": [ + { + "operatingSystem": "windows-2019", + "framework": "net461", + "additionalTestArguments": "/p:UseProjectReferenceToAzureClients=true" + } + ], + "exclude": [ + { + "operatingSystem": "windows-2019", + "framework": "net461" + } + ] +}