diff --git a/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml b/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml index 32138d40a023..9da085371e29 100644 --- a/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml +++ b/src/SourceBuild/content/eng/pipelines/source-build-sdk-diff-tests.yml @@ -17,17 +17,19 @@ resources: trigger: branches: include: - - refs/heads/release/*.0.1xx-preview* - - refs/heads/release/9.0.1xx - - refs/heads/internal/release/*.0.1xx* + - refs/heads/release/*.0.1xx-preview* + - refs/heads/release/9.0.1xx + - refs/heads/internal/release/*.0.1xx* + + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release pr: none trigger: none -pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2004 - parameters: - name: dotnetDotnetRunId displayName: 'Specific dotnet-dotnet run ID number (e.g `2108850`)' @@ -40,46 +42,56 @@ variables: # GH access token for SB bot - BotAccount-dotnet-sb-bot-pat - group: DotNet-Source-Build-Bot-Secrets-MVP -jobs: -- template: templates/jobs/sdk-diff-tests.yml +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.centOSStreamName) }} - targetRid: ${{ variables.centOSStreamX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} - publishTestResultsPr: true + pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2004 + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: stage + jobs: + - template: templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.centOSStreamName) }} + targetRid: ${{ variables.centOSStreamX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + publishTestResultsPr: true -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.almaLinuxName) }} - targetRid: ${{ variables.almaLinuxX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.almaLinuxName) }} + targetRid: ${{ variables.almaLinuxX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Online_MsftSdk', variables.alpineLatestName) }} - targetRid: ${{ variables.alpineLatestX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Online_MsftSdk', variables.alpineLatestName) }} + targetRid: ${{ variables.alpineLatestX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.fedoraName) }} - targetRid: ${{ variables.fedoraX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.fedoraName) }} + targetRid: ${{ variables.fedoraX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}_Offline_MsftSdk', variables.ubuntuName) }} - targetRid: ${{ variables.ubuntuX64Rid }} - architecture: x64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}_Offline_MsftSdk', variables.ubuntuName) }} + targetRid: ${{ variables.ubuntuX64Rid }} + architecture: x64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} -- template: templates/jobs/sdk-diff-tests.yml - parameters: - buildName: ${{ format('{0}Arm64_Offline_MsftSdk', variables.ubuntuName) }} - targetRid: ${{ variables.ubuntuArm64Rid }} - architecture: arm64 - dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} + - template: templates/jobs/sdk-diff-tests.yml@self + parameters: + buildName: ${{ format('{0}Arm64_Offline_MsftSdk', variables.ubuntuName) }} + targetRid: ${{ variables.ubuntuArm64Rid }} + architecture: arm64 + dotnetDotnetRunId: ${{ parameters.dotnetDotnetRunId }} diff --git a/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml b/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml index 0aa82e5633fc..ff97460c0116 100644 --- a/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml +++ b/src/SourceBuild/content/eng/pipelines/templates/jobs/sdk-diff-tests.yml @@ -18,11 +18,29 @@ parameters: jobs: - job: ${{ parameters.buildName }}_${{ parameters.architecture }} timeoutInMinutes: 150 - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2004 variables: - template: ../variables/pipelines.yml + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + inputs: + - input: pipelineArtifact + displayName: '${{ parameters.displayName }}' + buildType: specific + buildVersionToDownload: specific + project: internal + pipeline: ${{ parameters.pipeline }} + buildId: ${{ parameters.buildId }} + artifactName: '' + patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux*-${{ parameters.architecture }}.tar.gz' + allowPartiallySucceededBuilds: true + allowFailedBuilds: true + downloadPath: $(Pipeline.Workspace)/Artifacts + checkDownloadedFiles: true steps: - script: | dotnet_dotnet_build='${{ replace(parameters.dotnetDotnetRunId, ' ', '') }}' @@ -55,11 +73,6 @@ jobs: env: AZURE_DEVOPS_EXT_PAT: $(System.AccessToken) - - template: ../steps/download-pipeline-artifact.yml - parameters: - patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux*-${{ parameters.architecture }}.tar.gz' - displayName: Download MSFT SDK - - template: ../steps/download-vmr-artifact.yml parameters: buildName: ${{ parameters.buildName }} @@ -147,12 +160,6 @@ jobs: continueOnError: true condition: succeededOrFailed() - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() - - task: PublishTestResults@2 displayName: Publish Test Results condition: succeededOrFailed() diff --git a/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml b/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml index 98d54c594eb1..e24dbd5ad6a0 100644 --- a/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml +++ b/src/SourceBuild/content/eng/pipelines/vmr-license-scan.yml @@ -15,7 +15,7 @@ schedules: pr: none # Always trigger a run when changes are made to the license test implementation or baselines -trigger: +trigger: branches: include: - main @@ -40,158 +40,173 @@ variables: - name: sdkRoot value: '$(Build.SourcesDirectory)/src/sdk' -jobs: -- job: Setup - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2004 - steps: - - script: | - vmrSrcDir="$(Build.SourcesDirectory)/src" - # Builds an Azure DevOps matrix definition. Each entry in the matrix is a path, - # allowing a job to be run for each src repo. - matrix="" - - # Trim leading/trailing spaces from the repo name - specificRepoName=$(echo "${{ parameters.specificRepoName }}" | awk '{$1=$1};1') - - # If the repo name is provided, only scan that repo. - if [ ! -z "$specificRepoName" ]; then - matrix="\"$specificRepoName\": { \"repoPath\": \"$vmrSrcDir/$specificRepoName\" }" - else - for dir in $vmrSrcDir/*/ - do - if [ ! -z "$matrix" ]; then - matrix="$matrix," - fi - repoName=$(basename $dir) - matrix="$matrix \"$repoName\": { \"repoPath\": \"$dir\" }" - done - fi - - matrix="{ $matrix }" - - echo "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" - name: GetMatrix - displayName: Get Matrix - -- job: LicenseScan - dependsOn: Setup - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2004 - timeoutInMinutes: 420 - strategy: - matrix: $[ dependencies.Setup.outputs['GetMatrix.matrix'] ] - steps: - - - script: | - source ./eng/common/tools.sh - InitializeDotNetCli true - displayName: Install .NET SDK - workingDirectory: $(Build.SourcesDirectory) - - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: public/dotnet-public-pypi - onlyAddExtraIndex: false - - - script: $(sdkRoot)/eng/install-scancode.sh - displayName: Install Scancode - - - script: > - $(Build.SourcesDirectory)/.dotnet/dotnet test - $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj - --filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.LicenseScanTests.ScanForLicenses" - --logger:'trx;LogFileName=$(Agent.JobName)_LicenseScan.trx' - --logger:'console;verbosity=detailed' - -c Release - -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog - -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log - -clp:v=m - /p:SmokeTestsLicenseScanPath=$(repoPath) - /p:SmokeTestsWarnOnLicenseScanDiffs=false - /p:TargetRid=linux-x64 - /p:PortableRid=linux-x64 - /p:SkipPrepareSdkArchive=true - displayName: Run Tests - workingDirectory: $(Build.SourcesDirectory) - - - script: | - set -x - targetFolder=$(Build.StagingDirectory)/BuildLogs/ - mkdir -p ${targetFolder} - cd "$(Build.SourcesDirectory)" - find artifacts/log/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/log/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; - echo "Updated:" - find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" - find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/TestResults/ -type f -name "Updated*.json" - find artifacts/TestResults/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \; - echo "Results:" - find artifacts/TestResults/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \; - echo "All:" - ls -R artifacts/TestResults/ - echo "BuildLogs:" - ls -R ${targetFolder} - displayName: Prepare BuildLogs staging directory - continueOnError: true - condition: succeededOrFailed() - - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() - - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - continueOnError: true - inputs: - testRunner: vSTest - testResultsFiles: '**/*.trx' - searchFolder: $(Build.SourcesDirectory)/artifacts/TestResults - mergeTestResults: true - publishRunAttachments: true - testRunTitle: $(Agent.JobName) - -- job: Publish_Test_Results_PR - dependsOn: LicenseScan - condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) - pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals 1es-ubuntu-2004 - variables: - - template: templates/variables/pipelines.yml - steps: - - - script: | - source ./eng/common/tools.sh - InitializeDotNetCli true - displayName: Install .NET SDK - workingDirectory: $(Build.SourcesDirectory) - - - template: templates/steps/download-pipeline-artifact.yml - parameters: - pipeline: $(SOURCE_BUILD_LICENSE_SCAN_PIPELINE_ID) - buildId: $(Build.BuildId) - artifact: '' - patterns: '**/Updated*' - displayName: Download Updated Test Files - - - script: | - find $(Pipeline.Workspace)/Artifacts -type f -exec mv {} $(Pipeline.Workspace)/Artifacts \; - displayName: Move Artifacts to root - - - template: templates/steps/create-baseline-update-pr.yml - parameters: - pipeline: license - repo: dotnet/sdk - originalFilesDirectory: src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseScanTests - updatedFilesDirectory: $(Pipeline.Workspace)/Artifacts - pullRequestTitle: Update Source-Build License Scan Baselines and Exclusions +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals 1es-ubuntu-2004 + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: stage + jobs: + - job: Setup + steps: + - script: | + vmrSrcDir="$(Build.SourcesDirectory)/src" + # Builds an Azure DevOps matrix definition. Each entry in the matrix is a path, + # allowing a job to be run for each src repo. + matrix="" + + # Trim leading/trailing spaces from the repo name + specificRepoName=$(echo "${{ parameters.specificRepoName }}" | awk '{$1=$1};1') + + # If the repo name is provided, only scan that repo. + if [ ! -z "$specificRepoName" ]; then + matrix="\"$specificRepoName\": { \"repoPath\": \"$vmrSrcDir/$specificRepoName\" }" + else + for dir in $vmrSrcDir/*/ + do + if [ ! -z "$matrix" ]; then + matrix="$matrix," + fi + repoName=$(basename $dir) + matrix="$matrix \"$repoName\": { \"repoPath\": \"$dir\" }" + done + fi + + matrix="{ $matrix }" + + echo "##vso[task.setvariable variable=matrix;isOutput=true]$matrix" + name: GetMatrix + displayName: Get Matrix + + - job: LicenseScan + dependsOn: Setup + timeoutInMinutes: 420 + strategy: + matrix: $[ dependencies.Setup.outputs['GetMatrix.matrix'] ] + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + steps: + + - script: | + source ./eng/common/tools.sh + InitializeDotNetCli true + displayName: Install .NET SDK + workingDirectory: $(Build.SourcesDirectory) + + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: public/dotnet-public-pypi + onlyAddExtraIndex: false + + - script: $(sdkRoot)/eng/install-scancode.sh + displayName: Install Scancode + + - script: > + $(Build.SourcesDirectory)/.dotnet/dotnet test + $(Build.SourcesDirectory)/test/Microsoft.DotNet.SourceBuild.SmokeTests/Microsoft.DotNet.SourceBuild.SmokeTests.csproj + --filter "FullyQualifiedName=Microsoft.DotNet.SourceBuild.SmokeTests.LicenseScanTests.ScanForLicenses" + --logger:'trx;LogFileName=$(Agent.JobName)_LicenseScan.trx' + --logger:'console;verbosity=detailed' + -c Release + -bl:$(Build.SourcesDirectory)/artifacts/log/Debug/BuildTests_$(date +"%m%d%H%M%S").binlog + -flp:LogFile=$(Build.SourcesDirectory)/artifacts/logs/BuildTests_$(date +"%m%d%H%M%S").log + -clp:v=m + /p:SmokeTestsLicenseScanPath=$(repoPath) + /p:SmokeTestsWarnOnLicenseScanDiffs=false + /p:TargetRid=linux-x64 + /p:PortableRid=linux-x64 + /p:SkipPrepareSdkArchive=true + displayName: Run Tests + workingDirectory: $(Build.SourcesDirectory) + + - script: | + set -x + targetFolder=$(Build.StagingDirectory)/BuildLogs/ + mkdir -p ${targetFolder} + cd "$(Build.SourcesDirectory)" + find artifacts/log/ -type f -name "BuildTests*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/log/ -type f -name "BuildTests*.log" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/TestResults/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/TestResults/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; + echo "Updated:" + find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" + find artifacts/TestResults/ -type f -name "UpdatedLicenseExclusions*.txt" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/TestResults/ -type f -name "Updated*.json" + find artifacts/TestResults/ -type f -name "Updated*.json" -exec cp {} --parents -t ${targetFolder} \; + echo "Results:" + find artifacts/TestResults/ -type f -name "scancode-results*.json" -exec cp {} --parents -t ${targetFolder} \; + echo "All:" + ls -R artifacts/TestResults/ + echo "BuildLogs:" + ls -R ${targetFolder} + displayName: Prepare BuildLogs staging directory + continueOnError: true + condition: succeededOrFailed() + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: '**/*.trx' + searchFolder: $(Build.SourcesDirectory)/artifacts/TestResults + mergeTestResults: true + publishRunAttachments: true + testRunTitle: $(Agent.JobName) + + - job: Publish_Test_Results_PR + dependsOn: LicenseScan + condition: or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release')) + variables: + - template: templates/variables/pipelines.yml@self + templateContext: + inputs: + - input: pipelineArtifact + displayName: '${{ parameters.displayName }}' + buildType: specific + buildVersionToDownload: specific + project: internal + pipeline: ${{ parameters.pipeline }} + buildId: ${{ parameters.buildId }} + artifactName: '' + patterns: '**/Updated*' + allowPartiallySucceededBuilds: true + allowFailedBuilds: true + downloadPath: $(Pipeline.Workspace)/Artifacts + checkDownloadedFiles: true + steps: + + - script: | + source ./eng/common/tools.sh + InitializeDotNetCli true + displayName: Install .NET SDK + workingDirectory: $(Build.SourcesDirectory) + + - script: | + find $(Pipeline.Workspace)/Artifacts -type f -exec mv {} $(Pipeline.Workspace)/Artifacts \; + displayName: Move Artifacts to root + + - template: templates/steps/create-baseline-update-pr.yml@self + parameters: + pipeline: license + repo: dotnet/sdk + originalFilesDirectory: src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/LicenseScanTests + updatedFilesDirectory: $(Pipeline.Workspace)/Artifacts + pullRequestTitle: Update Source-Build License Scan Baselines and Exclusions diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetWatchTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetWatchTests.cs index fd2c1125f89b..9ee348253d5a 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetWatchTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetWatchTests.cs @@ -13,7 +13,7 @@ public class DotNetWatchTests : SdkTests { public DotNetWatchTests(ITestOutputHelper outputHelper) : base(outputHelper) { } - [Fact] + // [Fact(Skip="https://github.com/dotnet/sdk/issues/42920")] public void WatchTests() { if (DotNetHelper.IsMonoRuntime) diff --git a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SymbolsTests.cs b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SymbolsTests.cs index f4b20cd9f8a4..728057c00af4 100644 --- a/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SymbolsTests.cs +++ b/src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SymbolsTests.cs @@ -24,7 +24,7 @@ public SymbolsTests(ITestOutputHelper outputHelper) : base(outputHelper) { } /// /// Verifies that all symbols have valid sourcelinks. /// - [Fact] + // [Fact(Skip="https://github.com/dotnet/sdk/issues/42920")] public void VerifySdkSymbols() { try