From f742191231975e4bbaea491570f4f639532427f8 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Mon, 11 Nov 2024 14:47:32 +0100 Subject: [PATCH 1/3] fix pipelines --- azure-pipelines/vs-insertion-experimental.yml | 34 ++++----- azure-pipelines/vs-insertion.yml | 72 +++++++++++-------- 2 files changed, 59 insertions(+), 47 deletions(-) diff --git a/azure-pipelines/vs-insertion-experimental.yml b/azure-pipelines/vs-insertion-experimental.yml index d74aad34d5f..6a1a2d6249f 100644 --- a/azure-pipelines/vs-insertion-experimental.yml +++ b/azure-pipelines/vs-insertion-experimental.yml @@ -1,13 +1,5 @@ trigger: none name: $(Date:yyyyMMdd).$(Rev:r) MSBuild Experimental VS Insertion -variables: - InsertConfigValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion) - InsertCustomScriptExecutionCommand: $(Pipeline.Workspace)\xsd\Update-MSBuildXsds.ps1 - InsertDescription: Insert MSBuild $(Build.BuildNumber) from the branch $(Build.SourceBranchName) at commit $(Build.SourceVersion). Corresponding package version is $(MSBuild_ExtApisPackageVersion) - InsertJsonValues: Microsoft.Build.vsman{$(MSBuild_ExtApisPackageVersion)+$(Build.SourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(Build.SourceBranchName)/$(Build.BuildNumber);Microsoft.Build.vsman,Microsoft.Build.Arm64.vsman{$(MSBuild_ExtApisPackageVersion)+$(Build.SourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(Build.SourceBranchName)/$(Build.BuildNumber);Microsoft.Build.Arm64.vsman,Microsoft.Build.UnGAC.vsman=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(Build.SourceBranchName)/$(Build.Build.Number);Microsoft.Build.UnGAC.vsman - InsertPayloadName: MSBuild $(Build.SourceBranchName) $(Build.BuildNumber) - TeamName: msbuild - TeamEmail: msbuild@microsoft.com resources: pipelines: - pipeline: 'MSBuild' @@ -28,10 +20,18 @@ extends: pool: name: VSEngSS-MicroBuild2022-1ES customBuildTags: - - ES365AIMigrationTooling-Release + - Experimental stages: - stage: Stage_1 displayName: Create VS PR + variables: + InsertConfigValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion) + InsertCustomScriptExecutionCommand: $(Pipeline.Workspace)\xsd\Update-MSBuildXsds.ps1 + InsertDescription: Insert MSBuild $(resources.pipeline.MSBuild.runName) from the branch $(resources.pipeline.MSBuild.sourceBranch) at commit $(resources.pipeline.MSBuild.sourceVersion). Corresponding package version is $(MSBuild_ExtApisPackageVersion) + InsertJsonValues: Microsoft.Build.vsman{$(MSBuild_ExtApisPackageVersion)+$(resources.pipeline.MSBuild.sourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(resources.pipeline.MSBuild.sourceBranch)/$(resources.pipeline.MSBuild.runName);Microsoft.Build.vsman,Microsoft.Build.Arm64.vsman{$(MSBuild_ExtApisPackageVersion)+$(resources.pipeline.MSBuild.sourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(resources.pipeline.MSBuild.sourceBranch)/$(resources.pipeline.MSBuild.runName);Microsoft.Build.Arm64.vsman,Microsoft.Build.UnGAC.vsman=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(resources.pipeline.MSBuild.sourceBranch)/$(resources.pipeline.MSBuild.runName);Microsoft.Build.UnGAC.vsman + InsertPayloadName: MSBuild $(resources.pipeline.MSBuild.sourceBranch) $(MSBuild_ExtApisPackageVersion) + TeamName: msbuild + TeamEmail: msbuild@microsoft.com jobs: - job: Job_1 displayName: Run on agent @@ -39,14 +39,6 @@ extends: timeoutInMinutes: 0 templateContext: inputs: - - input: pipelineArtifact - pipeline: 'MSBuild' - artifactName: 'BuildLogs_SourceBuild_Managed_Attempt1' - targetPath: '$(Pipeline.Workspace)/BuildLogs_SourceBuild_Managed_Attempt1' - - input: pipelineArtifact - pipeline: 'MSBuild' - artifactName: 'build_Windows_NT_SBOM' - targetPath: '$(Pipeline.Workspace)/build_Windows_NT_SBOM' - input: pipelineArtifact pipeline: 'MSBuild' artifactName: 'xsd' @@ -56,6 +48,14 @@ extends: artifactName: 'PackageArtifacts' targetPath: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' steps: + - task: Powershell@2 + name: RemoveSourceBuildPackage + displayName: Remove sourcebuild package + inputs: + targetType: inline + script: | + $folder = "$(Build.ArtifactStagingDirectory)/PackageArtifacts" + Get-ChildItem -Path $folder -Filter 'Microsoft.SourceBuild.Intermediate*.nupkg' | Remove-Item -Force - task: Powershell@2 name: Powershell_1 displayName: Munge ExternalAPIs package version diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index a7fafc8da41..086f22dab22 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -2,6 +2,17 @@ trigger: none name: $(Date:yyyyMMdd).$(Rev:r) +resources: + pipelines: + - pipeline: 'MSBuild' + project: 'DevDiv' + source: 'MSBuild' + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + schedules: - cron: '0 3 * * *' # Runs every day at 3AM UTC displayName: Daily VS insertion @@ -27,6 +38,15 @@ parameters: - rel/d17.6 - rel/d17.3 - rel/d17.0 + - name: DropRetentionDays + default: 183 + type: number + displayName: 'Drop Retention Days (do not set to < 90)' + - name: SourceBranchNameParam + default: auto + type: string + displayName: 'Source Branch Name (recommended to use `auto`)' + variables: # `auto` should work every time and selecting a branch in parameters is likely to fail due to incompatible versions in MSBuild and VS - name: AutoInsertTargetBranch @@ -65,17 +85,16 @@ variables: value: msbuild - name: TeamEmail value: msbuild@microsoft.com + - name: MSBuild_CI_BuildNumber + value: $(resources.pipeline.MSBuild.runName) + - name: MSBuild_CI_SourceVersion + value: $(resources.pipeline.MSBuild.sourceCommit) + - name: SourceBranchName + ${{ if not(eq(parameters.SourceBranchNameParam, 'auto')) }}: + value: ${{ parameters.SourceBranchNameParam }} + ${{ else }}: + value: $(Build.SourceBranchName) -resources: - pipelines: - - pipeline: 'MSBuild' - project: 'DevDiv' - source: 'MSBuild' - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: @@ -106,26 +125,19 @@ extends: name: MicroBuildRetainVstsDrops_2 displayName: Retain VSTS Drops inputs: - DropNames: Products/DevDiv/DotNet-msbuild-Trusted/$(Build.SourceBranchName)/$(Build.BuildNumber) + DropNames: Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber) AccessToken: $(System.AccessToken) DropServiceUri: https://devdiv.artifacts.visualstudio.com/DefaultCollection - # retain servicing indefinitely, main only for 183 days - ${{ if eq(variables['InsertTargetBranch'], 'main') }}: - DropRetentionDays: 183 + DropRetentionDays: ${{ parameters.DropRetentionDays }} - stage: VSInsertion displayName: VS insertion dependsOn: RetainBuild variables: InsertConfigValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion) InsertCustomScriptExecutionCommand: $(Pipeline.Workspace)\xsd\Update-MSBuildXsds.ps1 - InsertDescription: Insert MSBuild $(Build.BuildNumber) from the branch $(Build.SourceBranchName) at commit $(Build.SourceVersion). Corresponding package version is $(MSBuild_ExtApisPackageVersion) - InsertJsonValues: Microsoft.Build.vsman{$(MSBuild_ExtApisPackageVersion)+$(Build.SourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(Build.SourceBranchName)/$(Build.BuildNumber);Microsoft.Build.vsman,Microsoft.Build.Arm64.vsman{$(MSBuild_ExtApisPackageVersion)+$(Build.SourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(Build.SourceBranchName)/$(Build.BuildNumber);Microsoft.Build.Arm64.vsman,Microsoft.Build.UnGAC.vsman=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(Build.SourceBranchName)/$(Build.BuildNumber);Microsoft.Build.UnGAC.vsman - InsertPayloadName: MSBuild $(Build.SourceBranchName) $(Build.BuildNumber) - SymbolsAgentPath: $(Pipeline.Workspace)\$(Build.DefinitionName)\Symbols - SymbolsEmailContacts: raines - SymbolsFeatureName: MSBuild - SymbolsSymwebProject: DDE - SymbolsUncPath: '\\cpvsbuild\drops\MSBuild\symbols\$(Build.DefinitionName)\$(Build.BuildNumber)\symbols.archive' + InsertDescription: Insert MSBuild $(MSBuild_CI_BuildNumber) from the branch $(SourceBranchName) at commit $(MSBuild_CI_SourceVersion). Corresponding package version is $(MSBuild_ExtApisPackageVersion) + InsertJsonValues: Microsoft.Build.vsman{$(MSBuild_ExtApisPackageVersion)+$(MSBuild_CI_SourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber);Microsoft.Build.vsman,Microsoft.Build.Arm64.vsman{$(MSBuild_ExtApisPackageVersion)+$(MSBuild_CI_SourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber);Microsoft.Build.Arm64.vsman,Microsoft.Build.UnGAC.vsman=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber);Microsoft.Build.UnGAC.vsman + InsertPayloadName: MSBuild $(SourceBranchName) $(MSBuild_ExtApisPackageVersion) jobs: - job: PushPackages displayName: Push Packages @@ -133,14 +145,6 @@ extends: timeoutInMinutes: 0 templateContext: inputs: - - input: pipelineArtifact - pipeline: 'MSBuild' - artifactName: 'BuildLogs_SourceBuild_Managed_Attempt1' - targetPath: '$(Pipeline.Workspace)/BuildLogs_SourceBuild_Managed_Attempt1' - - input: pipelineArtifact - pipeline: 'MSBuild' - artifactName: 'build_Windows_NT_SBOM' - targetPath: '$(Pipeline.Workspace)/build_Windows_NT_SBOM' - input: pipelineArtifact pipeline: 'MSBuild' artifactName: 'xsd' @@ -150,6 +154,14 @@ extends: artifactName: 'PackageArtifacts' targetPath: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' steps: + - task: Powershell@2 + name: RemoveSourceBuildPackage + displayName: Remove sourcebuild package + inputs: + targetType: inline + script: | + $folder = "$(Build.ArtifactStagingDirectory)/PackageArtifacts" + Get-ChildItem -Path $folder -Filter 'Microsoft.SourceBuild.Intermediate*.nupkg' | Remove-Item -Force - task: Powershell@2 name: PwshMungeExternalAPIsPkgVersion displayName: Munge ExternalAPIs package version From 4b70fec6581ae8efdef4a333eeaa0c69d8316d8b Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Mon, 18 Nov 2024 13:50:07 +0100 Subject: [PATCH 2/3] polish pipelines --- azure-pipelines/vs-insertion-experimental.yml | 76 ++++++++++++++----- azure-pipelines/vs-insertion.yml | 73 +++++++++++------- 2 files changed, 104 insertions(+), 45 deletions(-) diff --git a/azure-pipelines/vs-insertion-experimental.yml b/azure-pipelines/vs-insertion-experimental.yml index 6a1a2d6249f..e8029d8fbad 100644 --- a/azure-pipelines/vs-insertion-experimental.yml +++ b/azure-pipelines/vs-insertion-experimental.yml @@ -1,5 +1,7 @@ +# Create an experimental VS insertion auto-triggered from exp/* branches to validate changes before merging to main trigger: none -name: $(Date:yyyyMMdd).$(Rev:r) MSBuild Experimental VS Insertion +name: $(Date:yyyyMMdd).$(Rev:r) + resources: pipelines: - pipeline: 'MSBuild' @@ -14,27 +16,48 @@ resources: type: git name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/release + +variables: + - name: TeamName + value: msbuild + - name: TeamEmail + value: msbtm@microsoft.com + - name: MSBuild_CI_BuildNumber + value: $(resources.pipeline.MSBuild.runName) + - name: MSBuild_CI_SourceVersion + value: $(resources.pipeline.MSBuild.sourceCommit) + - name: ArtifactPackagesPath + value: $(Build.ArtifactStagingDirectory)/PackageArtifacts + + - name: MSBuildPackagePattern + value: '$(ArtifactPackagesPath)/Microsoft.Build.*.nupkg' + - name: StringToolsPackagePattern + value: '$(ArtifactPackagesPath)/Microsoft.NET.StringTools*.nupkg' + - name: ExternalAPIsPackagePattern + value: '$(ArtifactPackagesPath)/VS.ExternalAPIs.*.nupkg' + extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: pool: name: VSEngSS-MicroBuild2022-1ES - customBuildTags: - - Experimental stages: - - stage: Stage_1 - displayName: Create VS PR + - stage: VSInsertion + displayName: VS insertion variables: + VSDropPath: https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber) + VSManVersionSuffix: $(MSBuild_ExtApisPackageVersion)+$(MSBuild_CI_SourceVersion) + StandardVSManPath: Microsoft.Build.vsman{$(VSManVersionSuffix)}=$(VSDropPath);Microsoft.Build.vsman + Arm64VSManPath: Microsoft.Build.Arm64.vsman{$(VSManVersionSuffix)}=$(VSDropPath);Microsoft.Build.Arm64.vsman + UnGACVSManPath: Microsoft.Build.UnGAC.vsman=$(VSDropPath);Microsoft.Build.UnGAC.vsman + InsertJsonValues: $(StandardVSManPath),$(Arm64VSManPath),$(UnGACVSManPath) InsertConfigValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion) InsertCustomScriptExecutionCommand: $(Pipeline.Workspace)\xsd\Update-MSBuildXsds.ps1 - InsertDescription: Insert MSBuild $(resources.pipeline.MSBuild.runName) from the branch $(resources.pipeline.MSBuild.sourceBranch) at commit $(resources.pipeline.MSBuild.sourceVersion). Corresponding package version is $(MSBuild_ExtApisPackageVersion) - InsertJsonValues: Microsoft.Build.vsman{$(MSBuild_ExtApisPackageVersion)+$(resources.pipeline.MSBuild.sourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(resources.pipeline.MSBuild.sourceBranch)/$(resources.pipeline.MSBuild.runName);Microsoft.Build.vsman,Microsoft.Build.Arm64.vsman{$(MSBuild_ExtApisPackageVersion)+$(resources.pipeline.MSBuild.sourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(resources.pipeline.MSBuild.sourceBranch)/$(resources.pipeline.MSBuild.runName);Microsoft.Build.Arm64.vsman,Microsoft.Build.UnGAC.vsman=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(resources.pipeline.MSBuild.sourceBranch)/$(resources.pipeline.MSBuild.runName);Microsoft.Build.UnGAC.vsman - InsertPayloadName: MSBuild $(resources.pipeline.MSBuild.sourceBranch) $(MSBuild_ExtApisPackageVersion) - TeamName: msbuild - TeamEmail: msbuild@microsoft.com + InsertDescription: Insert MSBuild $(MSBuild_CI_BuildNumber) from the branch $(SourceBranchName) at commit $(MSBuild_CI_SourceVersion). Corresponding package version is $(MSBuild_ExtApisPackageVersion) + InsertPayloadName: '[Experimental] [Skip-SymbolCheck] MSBuild $(SourceBranchName) $(MSBuild_ExtApisPackageVersion)' jobs: - - job: Job_1 - displayName: Run on agent + - job: PushPackagesAndInsert + displayName: Push Packages and Insert VS payload condition: succeeded() timeoutInMinutes: 0 templateContext: @@ -48,6 +71,7 @@ extends: artifactName: 'PackageArtifacts' targetPath: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' steps: + # the CI build creates a sourcebuild intermediate package that is not signed, remove it to avoid warning from Guardian - task: Powershell@2 name: RemoveSourceBuildPackage displayName: Remove sourcebuild package @@ -57,7 +81,19 @@ extends: $folder = "$(Build.ArtifactStagingDirectory)/PackageArtifacts" Get-ChildItem -Path $folder -Filter 'Microsoft.SourceBuild.Intermediate*.nupkg' | Remove-Item -Force - task: Powershell@2 - name: Powershell_1 + name: SetSourceBranch + displayName: Set source branch name + inputs: + targetType: inline + script: | + # Extract the last section after the last '/' + $fullBranch = "$(resources.pipeline.MSBuild.sourceBranch)" + $branchSegments = $fullBranch -split '/' + $branch = $branchSegments[-1] + Write-Host "Setting drops branch to '$branch'" + Write-Host "##vso[task.setvariable variable=SourceBranchName]$branch" + - task: Powershell@2 + name: PwshMungeExternalAPIsPkgVersion displayName: Munge ExternalAPIs package version inputs: targetType: inline @@ -76,7 +112,7 @@ extends: displayName: 'Push MSBuild CoreXT packages' inputs: packageParentPath: '$(Build.ArtifactStagingDirectory)' - packagesToPush: $(Build.ArtifactStagingDirectory)/PackageArtifacts/Microsoft.Build.*.nupkg;$(Build.ArtifactStagingDirectory)/PackageArtifacts/Microsoft.NET.StringTools.*.nupkg;$(Build.ArtifactStagingDirectory)/PackageArtifacts/VS.ExternalAPIs.*.nupkg + packagesToPush: $(MSBuildPackagePattern);$(StringToolsPackagePattern);$(ExternalAPIsPackagePattern) nuGetFeedType: internal publishVstsFeed: VS allowPackageConflicts: true @@ -84,11 +120,15 @@ extends: name: MicroBuildInsertVsPayload_4 displayName: Insert VS Payload inputs: - PackagePropsValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion);Microsoft.Build=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Framework=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Tasks.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Utilities.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.NET.StringTools=$(MicrosoftNETStringToolsPackageVersion) LinkWorkItemsToPR: true - TeamEmail: $(TeamEmail) TeamName: $(TeamName) - InsertionReviewers: MSBuild + TeamEmail: $(TeamEmail) TargetBranch: main - InsertionPayloadName: '[Experimental] [Skip-SymbolCheck] $(InsertPayloadName)' + InsertionPayloadName: $(InsertPayloadName) + PackagePropsValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion);Microsoft.Build=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Framework=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Tasks.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Utilities.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.NET.StringTools=$(MicrosoftNETStringToolsPackageVersion) + InsertionDescription: $(InsertDescription) + ComponentJsonValues: $(InsertJsonValues) + DefaultConfigValues: $(InsertConfigValues) + InsertionReviewers: MSBuild + CustomScriptExecutionCommand: $(InsertCustomScriptExecutionCommand) InsertionBuildPolicy: Request Perf DDRITs \ No newline at end of file diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 086f22dab22..03aa609b08b 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -42,10 +42,6 @@ parameters: default: 183 type: number displayName: 'Drop Retention Days (do not set to < 90)' - - name: SourceBranchNameParam - default: auto - type: string - displayName: 'Source Branch Name (recommended to use `auto`)' variables: # `auto` should work every time and selecting a branch in parameters is likely to fail due to incompatible versions in MSBuild and VS @@ -75,25 +71,23 @@ variables: value: ${{ parameters.TargetBranch }} ${{ else }}: value: $(AutoInsertTargetBranch) - - name: InsertPropsValues - # servicing branches until 17.12 also include Microsoft.Build.Engine and Microsoft.Build.Conversion.Core - ${{ if or(eq(variables['Build.SourceBranchName'], 'vs17.0'), eq(variables['Build.SourceBranchName'], 'vs17.3'), eq(variables['Build.SourceBranchName'], 'vs17.6'), eq(variables['Build.SourceBranchName'], 'vs17.8'), eq(variables['Build.SourceBranchName'], 'vs17.10'), eq(variables['Build.SourceBranchName'], 'vs17.11'), eq(variables['Build.SourceBranchName'], 'vs17.12')) }}: - value: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion);Microsoft.Build=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Conversion.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Engine=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Framework=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Tasks.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Utilities.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.NET.StringTools=$(MicrosoftNETStringToolsPackageVersion) - ${{ else }}: - value: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion);Microsoft.Build=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Framework=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Tasks.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Utilities.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.NET.StringTools=$(MicrosoftNETStringToolsPackageVersion) - name: TeamName value: msbuild - name: TeamEmail - value: msbuild@microsoft.com + value: msbtm@microsoft.com - name: MSBuild_CI_BuildNumber value: $(resources.pipeline.MSBuild.runName) - name: MSBuild_CI_SourceVersion value: $(resources.pipeline.MSBuild.sourceCommit) - - name: SourceBranchName - ${{ if not(eq(parameters.SourceBranchNameParam, 'auto')) }}: - value: ${{ parameters.SourceBranchNameParam }} - ${{ else }}: - value: $(Build.SourceBranchName) + - name: ArtifactPackagesPath + value: $(Build.ArtifactStagingDirectory)/PackageArtifacts + + - name: MSBuildPackagePattern + value: '$(ArtifactPackagesPath)/Microsoft.Build.*.nupkg' + - name: StringToolsPackagePattern + value: '$(ArtifactPackagesPath)/Microsoft.NET.StringTools*.nupkg' + - name: ExternalAPIsPackagePattern + value: '$(ArtifactPackagesPath)/VS.ExternalAPIs.*.nupkg' extends: template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates @@ -121,11 +115,23 @@ extends: Write-Error "InsertTargetBranch is not set, this means your're not inserting from main or a valid servicing branch." exit 1 } + # the insertion source branch corresponds to the branch on which the MSBuild pipeline ran + - task: PowerShell@2 + name: SetSourceBranchName + inputs: + targetType: inline + script: | + # Extract the last section after the last '/' + $fullBranch = "$(resources.pipeline.MSBuild.sourceBranch)" + $branchSegments = $fullBranch -split '/' + $branch = $branchSegments[-1] + Write-Host "Setting source branch to '$branch'" + Write-Host "##vso[task.setvariable variable=SourceBranchName;isoutput=true]$branch" - task: MicroBuildRetainVstsDrops@1 name: MicroBuildRetainVstsDrops_2 displayName: Retain VSTS Drops inputs: - DropNames: Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber) + DropNames: Products/DevDiv/DotNet-msbuild-Trusted/$(SetSourceBranchName.SourceBranchName)/$(MSBuild_CI_BuildNumber) AccessToken: $(System.AccessToken) DropServiceUri: https://devdiv.artifacts.visualstudio.com/DefaultCollection DropRetentionDays: ${{ parameters.DropRetentionDays }} @@ -133,14 +139,20 @@ extends: displayName: VS insertion dependsOn: RetainBuild variables: - InsertConfigValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion) - InsertCustomScriptExecutionCommand: $(Pipeline.Workspace)\xsd\Update-MSBuildXsds.ps1 - InsertDescription: Insert MSBuild $(MSBuild_CI_BuildNumber) from the branch $(SourceBranchName) at commit $(MSBuild_CI_SourceVersion). Corresponding package version is $(MSBuild_ExtApisPackageVersion) - InsertJsonValues: Microsoft.Build.vsman{$(MSBuild_ExtApisPackageVersion)+$(MSBuild_CI_SourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber);Microsoft.Build.vsman,Microsoft.Build.Arm64.vsman{$(MSBuild_ExtApisPackageVersion)+$(MSBuild_CI_SourceVersion)}=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber);Microsoft.Build.Arm64.vsman,Microsoft.Build.UnGAC.vsman=https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber);Microsoft.Build.UnGAC.vsman - InsertPayloadName: MSBuild $(SourceBranchName) $(MSBuild_ExtApisPackageVersion) + SourceBranchName: $[ stageDependencies.RetainBuild.Job_1.outputs['SetSourceBranchName.SourceBranchName'] ] + VSDropPath: https://vsdrop.corp.microsoft.com/file/v1/Products/DevDiv/DotNet-msbuild-Trusted/$(SourceBranchName)/$(MSBuild_CI_BuildNumber) + VSManVersionSuffix: $(MSBuild_ExtApisPackageVersion)+$(MSBuild_CI_SourceVersion) + StandardVSManPath: Microsoft.Build.vsman{$(VSManVersionSuffix)}=$(VSDropPath);Microsoft.Build.vsman + Arm64VSManPath: Microsoft.Build.Arm64.vsman{$(VSManVersionSuffix)}=$(VSDropPath);Microsoft.Build.Arm64.vsman + UnGACVSManPath: Microsoft.Build.UnGAC.vsman=$(VSDropPath);Microsoft.Build.UnGAC.vsman + InsertJsonValues: $(StandardVSManPath),$(Arm64VSManPath),$(UnGACVSManPath) + InsertConfigValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion) + InsertCustomScriptExecutionCommand: $(Pipeline.Workspace)\xsd\Update-MSBuildXsds.ps1 + InsertDescription: Insert MSBuild $(MSBuild_CI_BuildNumber) from the branch $(SourceBranchName) at commit $(MSBuild_CI_SourceVersion). Corresponding package version is $(MSBuild_ExtApisPackageVersion) + InsertPayloadName: 'MSBuild $(SourceBranchName) $(MSBuild_ExtApisPackageVersion)' jobs: - - job: PushPackages - displayName: Push Packages + - job: PushPackagesAndInsert + displayName: Push Packages and Insert VS payload condition: succeeded() timeoutInMinutes: 0 templateContext: @@ -154,6 +166,7 @@ extends: artifactName: 'PackageArtifacts' targetPath: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' steps: + # the CI build creates a sourcebuild intermediate package that is not signed, remove it to avoid warning from Guardian - task: Powershell@2 name: RemoveSourceBuildPackage displayName: Remove sourcebuild package @@ -182,7 +195,7 @@ extends: displayName: 'Push MSBuild CoreXT packages' inputs: packageParentPath: '$(Build.ArtifactStagingDirectory)' - packagesToPush: $(Build.ArtifactStagingDirectory)/PackageArtifacts/Microsoft.Build.*.nupkg;$(Build.ArtifactStagingDirectory)/PackageArtifacts/Microsoft.NET.StringTools.*.nupkg;$(Build.ArtifactStagingDirectory)/PackageArtifacts/VS.ExternalAPIs.*.nupkg + packagesToPush: $(MSBuildPackagePattern);$(StringToolsPackagePattern);$(ExternalAPIsPackagePattern) nuGetFeedType: internal publishVstsFeed: VS allowPackageConflicts: true @@ -194,7 +207,7 @@ extends: displayName: 'Push MSBuild packages to VSSDK' inputs: packageParentPath: '$(Build.ArtifactStagingDirectory)' - packagesToPush: $(Build.ArtifactStagingDirectory)/PackageArtifacts/Microsoft.Build.*.nupkg;$(Build.ArtifactStagingDirectory)/PackageArtifacts/Microsoft.NET.StringTools*.nupkg + packagesToPush: $(MSBuildPackagePattern);$(StringToolsPackagePattern) nuGetFeedType: external publishFeedCredentials: azure-public/vssdk allowPackageConflicts: true @@ -213,8 +226,14 @@ extends: TeamEmail: $(TeamEmail) TargetBranch: $(InsertTargetBranch) InsertionPayloadName: $(InsertPayloadName) - PackagePropsValues: $(InsertPropsValues) + # servicing branches until 17.12 also include Microsoft.Build.Engine and Microsoft.Build.Conversion.Core + ${{ if or(eq(variables['Build.SourceBranchName'], 'vs17.0'), eq(variables['Build.SourceBranchName'], 'vs17.3'), eq(variables['Build.SourceBranchName'], 'vs17.6'), eq(variables['Build.SourceBranchName'], 'vs17.8'), eq(variables['Build.SourceBranchName'], 'vs17.10'), eq(variables['Build.SourceBranchName'], 'vs17.11'), eq(variables['Build.SourceBranchName'], 'vs17.12')) }}: + PackagePropsValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion);Microsoft.Build=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Conversion.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Engine=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Framework=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Tasks.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Utilities.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.NET.StringTools=$(MicrosoftNETStringToolsPackageVersion) + ${{ else }}: + PackagePropsValues: VS.ExternalAPIs.MSBuild=$(MSBuild_ExtApisPackageVersion);Microsoft.Build=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Framework=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Tasks.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.Build.Utilities.Core=$(MicrosoftNETStringToolsPackageVersion);Microsoft.NET.StringTools=$(MicrosoftNETStringToolsPackageVersion) InsertionDescription: $(InsertDescription) + ComponentJsonValues: $(InsertJsonValues) + DefaultConfigValues: $(InsertConfigValues) InsertionReviewers: MSBuild,VS ProTools CustomScriptExecutionCommand: $(InsertCustomScriptExecutionCommand) AutoCompletePR: true From 53ba4e11b1c354a8e6e89007c024ae610900c89c Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Tue, 19 Nov 2024 10:42:29 +0100 Subject: [PATCH 3/3] address feedback - exclude pattern --- azure-pipelines/vs-insertion-experimental.yml | 13 ++++--------- azure-pipelines/vs-insertion.yml | 13 ++++--------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/azure-pipelines/vs-insertion-experimental.yml b/azure-pipelines/vs-insertion-experimental.yml index e8029d8fbad..4d39eb27c24 100644 --- a/azure-pipelines/vs-insertion-experimental.yml +++ b/azure-pipelines/vs-insertion-experimental.yml @@ -70,16 +70,11 @@ extends: pipeline: 'MSBuild' artifactName: 'PackageArtifacts' targetPath: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + # the CI build creates a sourcebuild intermediate package that is not signed, remove it to avoid warning from Guardian + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate*.nupkg steps: - # the CI build creates a sourcebuild intermediate package that is not signed, remove it to avoid warning from Guardian - - task: Powershell@2 - name: RemoveSourceBuildPackage - displayName: Remove sourcebuild package - inputs: - targetType: inline - script: | - $folder = "$(Build.ArtifactStagingDirectory)/PackageArtifacts" - Get-ChildItem -Path $folder -Filter 'Microsoft.SourceBuild.Intermediate*.nupkg' | Remove-Item -Force - task: Powershell@2 name: SetSourceBranch displayName: Set source branch name diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 03aa609b08b..7e1a27f3ed7 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -165,16 +165,11 @@ extends: pipeline: 'MSBuild' artifactName: 'PackageArtifacts' targetPath: '$(Build.ArtifactStagingDirectory)/PackageArtifacts' + # the CI build creates a sourcebuild intermediate package that is not signed, remove it to avoid warning from Guardian + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate*.nupkg steps: - # the CI build creates a sourcebuild intermediate package that is not signed, remove it to avoid warning from Guardian - - task: Powershell@2 - name: RemoveSourceBuildPackage - displayName: Remove sourcebuild package - inputs: - targetType: inline - script: | - $folder = "$(Build.ArtifactStagingDirectory)/PackageArtifacts" - Get-ChildItem -Path $folder -Filter 'Microsoft.SourceBuild.Intermediate*.nupkg' | Remove-Item -Force - task: Powershell@2 name: PwshMungeExternalAPIsPkgVersion displayName: Munge ExternalAPIs package version