diff --git a/.azure/pipelines/ci.yml b/.azure/pipelines/ci.yml index cffbae0de313..d6a660b48bb2 100644 --- a/.azure/pipelines/ci.yml +++ b/.azure/pipelines/ci.yml @@ -25,7 +25,7 @@ variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: _BuildArgs value: /p:TeamName=$(_TeamName) - /p:OfficialBuildId=$(Build.BuildNumber) + /p:OfficialBuildId=$(Build.BuildNumber) - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - name: _BuildArgs value: '' @@ -58,10 +58,12 @@ jobs: # if they have already been signed. This results in slower builds due to re-submitting the same .nupkg many times for signing. # The sign settings have been configured to - - script: ./eng/scripts/cibuild.cmd + - script: ./build.cmd + -ci -arch x64 - -BuildNative - /p:DisableCodeSigning=true + -pack + -all + -buildNative /bl:artifacts/log/build.x64.binlog $(_BuildArgs) displayName: Build x64 @@ -69,64 +71,46 @@ jobs: # This is going to actually build x86 native assets. See https://github.com/aspnet/AspNetCore/issues/7196 # Build the x86 shared framework - # Set DisableSignCheck because we'll run sign check in an explicit step after installers build - - script: ./eng/scripts/cibuild.cmd + - script: ./build.cmd + -ci -arch x86 - -NoRestore - -BuildNative - /t:BuildSharedFx + -pack + -all + -buildNative + -noBuildJava /p:OnlyPackPlatformSpecificPackages=true - /p:DisableCodeSigning=true /bl:artifacts/log/build.x86.binlog $(_BuildArgs) displayName: Build x86 # This is in a separate build step with -forceCoreMsbuild to workaround MAX_PATH limitations - https://github.com/Microsoft/msbuild/issues/53 - - script: ./build.cmd + - script: .\src\SiteExtensions\build.cmd -ci - -sign - -forceCoreMsbuild - /p:DisableCodeSigning=true - -projects ./src/SiteExtensions/LoggingAggregate/src/Microsoft.AspNetCore.AzureAppServices.SiteExtension/Microsoft.AspNetCore.AzureAppServices.SiteExtension.csproj + -pack + -noBuildDeps $(_BuildArgs) displayName: Build SiteExtension - # Remove all task build output - - script: rmdir /s /q build\tasks\bin - displayName: Clear task build output - # This runs code-signing on all packages, zips, and jar files as defined in build/CodeSign.targets. If https://github.com/dotnet/arcade/issues/1957 is resolved, # consider running code-signing inline with the other previous steps. # Sign check is disabled because it is run in a separate step below, after installers are built. - script: ./build.cmd -ci + -noBuild + -noRestore -sign - -NoRestore - /t:CodeSign - /p:SignType=$(_SignType) - /p:DisableSignCheck=true /bl:artifacts/log/build.codesign.binlog $(_BuildArgs) displayName: Code sign packages # Windows installers bundle both x86 and x64 assets - - powershell: ./src/Installers/Windows/build.ps1 - -ci - /p:SignType=$(_SignType) - $(_BuildArgs) - displayName: Build Installers - - # Run sign check to verify everything was code signed. - script: ./build.cmd - -ci - -sign - -NoRestore - /t:SignCheck - /p:SignType=$(_SignType) - /bl:artifacts/log/build.signcheck.binlog - $(_BuildArgs) - displayName: Run sign check - condition: and(succeeded(), eq(variables['_SignType'], 'real')) + -ci + -sign + -buildInstallers + /bl:artifacts/log/installers.msbuild.binlog + $(_BuildArgs) + displayName: Build Installers artifacts: - name: Windows_Logs @@ -150,12 +134,13 @@ jobs: jobName: Windows_arm_build jobDisplayName: "Build: Windows ARM" agentOs: Windows - buildScript: ./eng/scripts/cibuild.cmd - buildArgs: -arch arm - -NoBuildNodeJS - -NoBuildJava + buildArgs: + -arch arm + -sign + -pack + -noBuildNodeJS + -noBuildJava /p:OnlyPackPlatformSpecificPackages=true - /p:SignType=$(_SignType) /bl:artifacts/log/build.win-arm.binlog $(_BuildArgs) installNodeJs: false @@ -177,8 +162,10 @@ jobs: jobName: MacOs_x64_build jobDisplayName: "Build: macOS" agentOs: macOs - buildScript: ./eng/scripts/cibuild.sh - buildArgs: --no-build-nodejs + buildArgs: + --pack + --all + --no-build-nodejs --no-build-java -p:OnlyPackPlatformSpecificPackages=true -bl:artifacts/log/build.macos.binlog @@ -206,24 +193,24 @@ jobs: agentOs: Linux installNodeJs: false steps: - - script: ./eng/scripts/cibuild.sh + - script: ./build.sh + --ci --arch x64 + --pack + --all --no-build-nodejs --no-build-java -p:OnlyPackPlatformSpecificPackages=true -bl:artifacts/log/build.linux-x64.binlog $(_BuildArgs) - displayName: Run cibuild.sh + displayName: Run build.sh - script: | - rm -rf .dotnet/ - git clean -xfd src/ + git clean -xfd src/**/obj/ ./dockerbuild.sh bionic \ - --ci --pack --all --no-build-nodejs --no-build-java \ + --ci \ --arch x64 \ - -e KOREBUILD_SKIP_INSTALL_NETFX=0 \ - --no-restore \ + --build-installers \ --no-build-deps \ - -t:BuildSharedFx \ -p:OnlyPackPlatformSpecificPackages=true \ -p:BuildRuntimeArchive=false \ -p:LinuxInstallerType=deb \ @@ -231,15 +218,12 @@ jobs: $(_BuildArgs) displayName: Build Debian installers - script: | - rm -rf .dotnet/ - git clean -xfd src/ + git clean -xfd src/**/obj/ ./dockerbuild.sh rhel \ - --ci --pack --all --no-build-nodejs --no-build-java \ + --ci \ --arch x64 \ - -e KOREBUILD_SKIP_INSTALL_NETFX=0 \ - --no-restore \ + --build-installers \ --no-build-deps \ - -t:BuildSharedFx \ -p:OnlyPackPlatformSpecificPackages=true \ -p:BuildRuntimeArchive=false \ -p:LinuxInstallerType=rpm \ @@ -266,8 +250,10 @@ jobs: jobName: Linux_arm_build jobDisplayName: "Build: Linux ARM" agentOs: Linux - buildScript: ./eng/scripts/cibuild.sh - buildArgs: --arch arm + buildArgs: + --arch arm + --pack + --all --no-build-nodejs --no-build-java -p:OnlyPackPlatformSpecificPackages=true @@ -294,8 +280,10 @@ jobs: jobName: Linux_arm64_build jobDisplayName: "Build: Linux ARM64" agentOs: Linux - buildScript: ./eng/scripts/cibuild.sh - buildArgs: --arch arm64 + buildArgs: + --arch arm64 + --all + --pack --no-build-nodejs --no-build-java -p:OnlyPackPlatformSpecificPackages=true @@ -323,12 +311,12 @@ jobs: jobDisplayName: "Build: Linux Musl x64" agentOs: Linux buildScript: ./dockerbuild.sh alpine - buildArgs: --ci - --pack - --all - -e KOREBUILD_SKIP_INSTALL_NETFX=0 + buildArgs: + --ci --arch x64 --os-name linux-musl + --pack + --all --no-build-nodejs --no-build-java -p:OnlyPackPlatformSpecificPackages=true @@ -356,12 +344,12 @@ jobs: jobDisplayName: "Build: Linux Musl ARM64" agentOs: Linux buildScript: ./dockerbuild.sh ubuntu-alpine37 - buildArgs: --ci - --pack - --all - -e KOREBUILD_SKIP_INSTALL_NETFX=0 + buildArgs: + --ci --arch arm64 --os-name linux-musl + --pack + --all --no-build-nodejs --no-build-java -p:OnlyPackPlatformSpecificPackages=true @@ -390,8 +378,7 @@ jobs: jobDisplayName: "Test: Windows Server 2016 x64" agentOs: Windows isTestingJob: true - buildScript: ./eng/scripts/cibuild.cmd - buildArgs: -test -BuildNative "/p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false /p:BuildSiteExtensions=false" + buildArgs: -all -pack -test -BuildNative "/p:SkipIISNewHandlerTests=true /p:SkipIISTests=true /p:SkipIISExpressTests=true /p:SkipIISNewShimTests=true /p:RunTemplateTests=false" beforeBuild: - powershell: "& ./src/Servers/IIS/tools/UpdateIISExpressCertificate.ps1; & ./src/Servers/IIS/tools/update_schema.ps1" displayName: Setup IISExpress test certificates and schema @@ -403,6 +390,9 @@ jobs: - name: Windows_Test_Logs path: artifacts/log/ publishOnError: true + - name: Windows_Test_Results + path: artifacts/TestResults/ + publishOnError: true - template: jobs/default-build.yml parameters: @@ -411,15 +401,20 @@ jobs: jobDisplayName: "Test: Templates - Windows Server 2016 x64" agentOs: Windows isTestingJob: true - buildScript: ./src/ProjectTemplates/build.cmd - buildArgs: -ci -test -NoRestore -NoBuild -NoBuilddeps "/p:RunTemplateTests=true /p:BuildSiteExtensions=false" - beforeBuild: - - powershell: ./eng/scripts/cibuild.cmd + steps: + - script: ./build.cmd -ci -all -pack displayName: Build Repo + - script: ./src/ProjectTemplates/build.cmd -ci -pack -NoRestore -NoBuilddeps "/p:RunTemplateTests=true /bl:artifacts/log/template.pack.binlog" + displayName: Pack Templates + - script: ./src/ProjectTemplates/build.cmd -ci -test -NoRestore -NoBuild -NoBuilddeps "/p:RunTemplateTests=true /bl:artifacts/log/template.test.binlog" + displayName: Test Templates artifacts: - name: Windows_Test_Templates_Logs path: artifacts/log/ publishOnError: true + - name: Windows_Test_Templates_Results + path: artifacts/TestResults/ + publishOnError: true - template: jobs/default-build.yml parameters: @@ -428,12 +423,15 @@ jobs: jobDisplayName: "Test: macOS 10.13" agentOs: macOs isTestingJob: true - buildScript: ./eng/scripts/cibuild.sh - buildArgs: --test + buildArgs: --all --test "/p:RunTemplateTests=false" beforeBuild: - bash: "./eng/scripts/install-nginx-mac.sh" displayName: Installing Nginx afterBuild: + - bash: ./build.sh --ci --pack --no-build --no-restore --no-build-deps "/bl:artifacts/log/packages.pack.binlog" + displayName: Pack Packages (for Template tests) + - bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog" + displayName: Pack Templates (for Template tests) - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true displayName: Run Flaky Tests continueOnError: true @@ -441,6 +439,9 @@ jobs: - name: MacOS_Test_Logs path: artifacts/log/ publishOnError: true + - name: MacOS_Test_Results + path: artifacts/TestResults/ + publishOnError: true - template: jobs/default-build.yml parameters: @@ -449,14 +450,17 @@ jobs: jobDisplayName: "Test: Ubuntu 16.04 x64" agentOs: Linux isTestingJob: true - buildScript: ./eng/scripts/cibuild.sh - buildArgs: --test + buildArgs: --all --test "/p:RunTemplateTests=false" beforeBuild: - bash: "./eng/scripts/install-nginx-linux.sh" displayName: Installing Nginx - bash: "echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p" displayName: Increase inotify limit afterBuild: + - bash: ./build.sh --ci --pack --no-build --no-restore --no-build-deps "/bl:artifacts/log/packages.pack.binlog" + displayName: Pack Packages (for Template tests) + - bash: ./src/ProjectTemplates/build.sh --ci --pack --no-restore --no-build-deps "/bl:artifacts/log/template.pack.binlog" + displayName: Pack Templates (for Template tests) - bash: ./build.sh --no-build --ci --test -p:RunFlakyTests=true displayName: Run Flaky Tests continueOnError: true @@ -464,3 +468,6 @@ jobs: - name: Linux_Test_Logs path: artifacts/log/ publishOnError: true + - name: Linux_Test_Results + path: artifacts/TestResults/ + publishOnError: true diff --git a/.azure/pipelines/helix-test.yml b/.azure/pipelines/helix-test.yml index 1b512109b1e4..4a7b845a2dcf 100644 --- a/.azure/pipelines/helix-test.yml +++ b/.azure/pipelines/helix-test.yml @@ -15,7 +15,9 @@ jobs: agentOs: Windows timeoutInMinutes: 240 steps: - - script: .\build.cmd -all -ci /p:BuildNative=true /t:Helix /p:IsHelixJob=true /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\SendToHelix.binlog + - script: .\restore.cmd -ci + displayName: Restore + - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true -bl displayName: Run build.cmd helix target env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops diff --git a/.azure/pipelines/jobs/default-build.yml b/.azure/pipelines/jobs/default-build.yml index 96586165ce21..26c116582f26 100644 --- a/.azure/pipelines/jobs/default-build.yml +++ b/.azure/pipelines/jobs/default-build.yml @@ -104,16 +104,12 @@ jobs: queue: BuildPool.Windows.10.Amd64.VS2019.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: NetCoreInternal-Pool - ${{ if ne(parameters.isTestingJob, true) }}: - # Visual Studio Build Tools - queue: BuildPool.Windows.10.Amd64.VS2019.BT - ${{ if eq(parameters.isTestingJob, true) }}: - # Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing - queue: BuildPool.Windows.10.Amd64.VS2019 + # Visual Studio Enterprise - contains some stuff, like SQL Server and IIS Express, that we use for testing + queue: BuildPool.Windows.10.Amd64.VS2019 variables: AgentOsName: ${{ parameters.agentOs }} ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping - DOTNET_HOME: $(Agent.BuildDirectory)/.dotnet + DOTNET_HOME: $(Build.SourcesDirectory)/.dotnet BuildScript: ${{ parameters.buildScript }} BuildScriptArgs: ${{ parameters.buildArgs }} BuildConfiguration: ${{ parameters.configuration }} @@ -150,7 +146,7 @@ jobs: displayName: Install JDK 11 - ${{ if eq(parameters.isTestingJob, true) }}: - powershell: | - Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(BuildDirectory)\obj\selenium\" + Write-Host "##vso[task.setvariable variable=SeleniumProcessTrackingFolder]$(BuildDirectory)\artifacts\tmp\selenium\" ./eng/scripts/InstallGoogleChrome.ps1 displayName: Install Chrome - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}: @@ -169,13 +165,13 @@ jobs: - ${{ if eq(parameters.steps, '')}}: - ${{ if eq(parameters.buildScript, '') }}: - ${{ if eq(parameters.agentOs, 'Windows') }}: - - script: .\$(BuildDirectory)\build.cmd -ci /p:SignType=$(_SignType) /p:Configuration=$(BuildConfiguration) $(BuildScriptArgs) + - script: .\$(BuildDirectory)\build.cmd -ci /p:DotNetSignType=$(_SignType) -Configuration $(BuildConfiguration) $(BuildScriptArgs) displayName: Run build.cmd - ${{ if ne(parameters.agentOs, 'Windows') }}: - - script: ./$(BuildDirectory)/build.sh -ci -p:Configuration=$(BuildConfiguration) $(BuildScriptArgs) + - script: ./$(BuildDirectory)/build.sh -ci -configuration $(BuildConfiguration) $(BuildScriptArgs) displayName: Run build.sh - ${{ if ne(parameters.buildScript, '') }}: - - script: $(BuildScript) /p:Configuration=$(BuildConfiguration) $(BuildScriptArgs) + - script: $(BuildScript) -Configuration $(BuildConfiguration) $(BuildScriptArgs) displayName: run $(BuildScript) - ${{ parameters.afterBuild }} @@ -208,33 +204,46 @@ jobs: artifactType: Container parallel: true - - task: PublishTestResults@2 - displayName: Publish test results - condition: always() - continueOnError: true - inputs: - testRunTitle: $(AgentOsName)-$(BuildConfiguration) - testRunner: vstest - testResultsFiles: '**/artifacts/**/*.trx' - mergeTestResults: true - buildConfiguration: $(BuildConfiguration) - buildPlatform: $(AgentOsName) - - task: PublishTestResults@2 - displayName: Publish js test results - condition: always() - inputs: - testRunner: junit - testResultsFiles: '**/artifacts/log/**/*.junit.xml' - buildConfiguration: $(BuildConfiguration) - buildPlatform: $(AgentOsName) - - task: PublishTestResults@2 - displayName: Publish Java test results - condition: always() - inputs: - testRunner: junit - testResultsFiles: '**/TEST-com.microsoft.signalr*.xml' - buildConfiguration: $(BuildConfiguration) - buildPlatform: $(AgentOsName) + - ${{ if eq(parameters.isTestingJob, true) }}: + - task: PublishTestResults@2 + displayName: Publish test results + condition: always() + continueOnError: true + inputs: + testRunTitle: $(AgentOsName)-$(BuildConfiguration) + testRunner: vstest + testResultsFiles: '**/artifacts/**/*.trx' + mergeTestResults: true + buildConfiguration: $(BuildConfiguration) + buildPlatform: $(AgentOsName) + - task: PublishTestResults@2 + displayName: Publish test results + condition: always() + continueOnError: true + inputs: + testRunTitle: $(AgentOsName)-$(BuildConfiguration) + testRunner: xunit + testResultsFiles: '**/artifacts/TestResults/**/*.xml' + mergeTestResults: true + buildConfiguration: $(BuildConfiguration) + buildPlatform: $(AgentOsName) + - task: PublishTestResults@2 + displayName: Publish js test results + condition: always() + inputs: + testRunner: junit + testResultsFiles: '**/artifacts/log/**/*.junit.xml' + buildConfiguration: $(BuildConfiguration) + buildPlatform: $(AgentOsName) + - task: PublishTestResults@2 + displayName: Publish Java test results + condition: always() + inputs: + testRunner: junit + testResultsFiles: '**/TEST-com.microsoft.signalr*.xml' + buildConfiguration: $(BuildConfiguration) + buildPlatform: $(AgentOsName) + - ${{ if and(eq(variables['System.TeamProject'], 'internal'), eq(parameters.agentOs, 'Windows')) }}: - task: MicroBuildCleanup@1 diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 3b4122bc7154..d2dac5f2dad2 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -5,13 +5,10 @@ about: Suggest an idea for this project ### Is your feature request related to a problem? Please describe. A clear and concise description of what the problem is. -Example. I'm am trying to do [...] but [...] +Example: I am trying to do [...] but [...] ### Describe the solution you'd like -A clear and concise description of what you want to happen. - -### Describe alternatives you've considered -A clear and concise description of any alternative solutions or features you've considered. +A clear and concise description of what you want to happen. Include any alternative solutions you've considered. ### Additional context Add any other context or screenshots about the feature request here. diff --git a/Directory.Build.props b/Directory.Build.props index a39b24039990..d7c5acd6b090 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,4 +1,9 @@  + + $(MSBuildThisFileDirectory) + https://github.com/aspnet/AspNetCore + git + @@ -18,10 +23,11 @@ false + + - Microsoft ASP.NET Core MicrosoftAspNetCore @@ -51,22 +57,23 @@ - $(MSBuildThisFileDirectory) - https://github.com/aspnet/AspNetCore - git $(MSBuildThisFileDirectory)src\Shared\ $(RepoRoot)src\submodules\googletest\ true - - true + + false Microsoft.AspNetCore.App Shared Framework for hosting of Microsoft ASP.NET Core applications. It is open source, cross-platform and is supported by Microsoft. We hope you enjoy using it! If you do, please consider joining the active community of developers that are contributing to the project on GitHub ($(RepositoryUrl)). We happily accept issues and PRs. + .NETCoreApp + netcoreapp$(AspNetCoreMajorMinorVersion) + ASP.NET Core $(AspNetCoreMajorMinorVersion) + Microsoft.AspNetCore.App.Ref aspnetcore-runtime aspnetcore-targeting-pack @@ -133,10 +140,9 @@ $(LocalDotNetRoot)packs\ - + - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/build/CodeSign.targets b/build/CodeSign.targets deleted file mode 100644 index 883898ee854a..000000000000 --- a/build/CodeSign.targets +++ /dev/null @@ -1,46 +0,0 @@ - - - - $(CodeSignDependsOn);CollectFileSignInfo - - - - - - - - - - - - - - $(ArtifactsDir)obj\RedistSharedFx.Layout\$(Configuration)\ - $(BaseRedistNetCorePath)$(TargetRuntimeIdentifier)\ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/build/SharedFx.targets b/build/SharedFx.targets deleted file mode 100644 index 5244b5e2a7cd..000000000000 --- a/build/SharedFx.targets +++ /dev/null @@ -1,41 +0,0 @@ - - - _BuildSharedFxProjects;RemoveSharedFrameworkOnlyRefsFromNuspec - $(BuildSharedFxDependsOn);CodeSign - - - - - - - - - - - - - - - - - - - <_RestoreGraphProjectInput>@(FxProjectToBuild) - - - - - - - - - - diff --git a/build/repo.targets b/build/repo.targets deleted file mode 100644 index df94a22d00f2..000000000000 --- a/build/repo.targets +++ /dev/null @@ -1,158 +0,0 @@ - - - - - - - - Prepare - - $(BuildDependsOn);Compile - $(BuildDependsOn);Package - $(BuildDependsOn);Test;Verify - true - true - - $(CleanDependsOn);CleanArtifacts - - $(RestoreDependsOn);InstallDotNet;RestoreProjects - - - $(CompileDependsOn);BuildProjects - - $(PackageDependsOn);PackProjects - $(PackageDependsOn);RemoveSharedFrameworkOnlyRefsFromNuspec;BuildSharedFx - - - $(TestDependsOn);Compile - $(TestDependsOn);TestProjects - - $(BuildDependsOn);GenerateBuildAssetManifest - - - $(CodeSignDependsOn);RemoveSharedFrameworkOnlyRefsFromNuspec - - - - - - - - - - - - - - - - - - - - - - - <_SharedFrameworkAndPackageRef Include="@(_ProjectReferenceProvider->WithMetadataValue('IsAspNetCoreApp','true')->WithMetadataValue('IsShippingPackage', 'true')->Distinct())" /> - <_SharedFrameworkRef Include="@(_ProjectReferenceProvider->WithMetadataValue('IsAspNetCoreApp','true')->WithMetadataValue('IsShippingPackage', 'false')->Distinct())" /> - <_ProjectReferenceProviderWithRefAssembly Include="@(_ProjectReferenceProvider->HasMetadata('ReferenceAssemblyProjectFileRelativePath'))" /> - <_ProjectReferenceProvider Remove="@(_ProjectReferenceProviderWithRefAssembly)" /> - - - - $(MSBuildThisFileDirectory)..\eng\ProjectReferences.props - - - - @(_ProjectReferenceProvider->'', '%0A ') - @(_ProjectReferenceProviderWithRefAssembly->'', '%0A ') - - -]]> - - - - - - - - $(MSBuildThisFileDirectory)..\eng\SharedFramework.Local.props - - and properties from each .csproj in this repository. ---> - - - - @(_SharedFrameworkAndPackageRef->'', '%0A ') - - - @(_SharedFrameworkRef->'', '%0A ') - - - ]]> - - - - - - - - - - - - - - <_BuildOutput Include="$(ArtifactsShippingPackagesDir)*.nupkg" - Exclude="$(ArtifactsShippingPackagesDir)*.symbols.nupkg" /> - - - - - - - - - - - - - - - - diff --git a/build/runtimes.props b/build/runtimes.props deleted file mode 100644 index d5cd1f77816f..000000000000 --- a/build/runtimes.props +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - diff --git a/build/tasks/FileUtilities.cs b/build/tasks/FileUtilities.cs deleted file mode 100644 index ecb20bfc0f4a..000000000000 --- a/build/tasks/FileUtilities.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Reflection; - -namespace Microsoft.DotNet.Build.Tasks -{ - internal static class FileUtilities - { - public static Version GetFileVersion(string sourcePath) - { - var fvi = FileVersionInfo.GetVersionInfo(sourcePath); - - return fvi != null - ? new Version(fvi.FileMajorPart, fvi.FileMinorPart, fvi.FileBuildPart, fvi.FilePrivatePart) - : null; - } - - private static readonly HashSet _assemblyExtensions = new HashSet(new[] { ".dll", ".exe", ".winmd" }, StringComparer.OrdinalIgnoreCase); - - public static Version TryGetAssemblyVersion(string sourcePath) - { - var extension = Path.GetExtension(sourcePath); - - return _assemblyExtensions.Contains(extension) - ? GetAssemblyVersion(sourcePath) - : null; - } - - private static Version GetAssemblyVersion(string sourcePath) - { - try - { - return AssemblyName.GetAssemblyName(sourcePath)?.Version; - } - catch (BadImageFormatException) - { - // If an .dll file cannot be read, it may be a native .dll which would not have an assembly version. - return null; - } - } - } -} diff --git a/build/tasks/RepoTasks.csproj b/build/tasks/RepoTasks.csproj deleted file mode 100644 index 24dc25d2eb96..000000000000 --- a/build/tasks/RepoTasks.csproj +++ /dev/null @@ -1,35 +0,0 @@ - - - - - netcoreapp3.0 - net472 - $(DefineConstants);BUILD_MSI_TASKS - false - embedded - true - - - - - - - - - - - - - $(WiXSdkPath)\Microsoft.Deployment.WindowsInstaller.dll - - - $(WiXSdkPath)\Microsoft.Deployment.WindowsInstaller.Package.dll - - - - - - - - - diff --git a/build/tasks/RepoTasks.tasks b/build/tasks/RepoTasks.tasks deleted file mode 100644 index 39f085fd04f9..000000000000 --- a/build/tasks/RepoTasks.tasks +++ /dev/null @@ -1,10 +0,0 @@ - - - <_RepoTaskAssembly>$(MSBuildThisFileDirectory)bin\publish\RepoTasks.dll - - - - - - - diff --git a/build/tasks/tasks.sln b/build/tasks/tasks.sln deleted file mode 100644 index 55921e71f948..000000000000 --- a/build/tasks/tasks.sln +++ /dev/null @@ -1,34 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.0.0 -MinimumVisualStudioVersion = 16.0.0.0 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RepoTasks", "RepoTasks.csproj", "{A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|x64.ActiveCfg = Debug|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|x64.Build.0 = Debug|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|x86.ActiveCfg = Debug|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Debug|x86.Build.0 = Debug|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|Any CPU.Build.0 = Release|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|x64.ActiveCfg = Release|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|x64.Build.0 = Release|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|x86.ActiveCfg = Release|Any CPU - {A114791F-35B7-4E5B-8E5B-9A91E0B6E4AE}.Release|x86.Build.0 = Release|Any CPU - EndGlobalSection -EndGlobal diff --git a/dockerbuild.sh b/dockerbuild.sh index 315a8a832aa1..10ce8e9d088b 100755 --- a/dockerbuild.sh +++ b/dockerbuild.sh @@ -29,7 +29,7 @@ __usage() { echo " -e, --env Additional environment variables to add to the build container" echo "" echo "Description:" - echo " This will run build.sh inside the dockerfile as defined in build/docker/\$image.Dockerfile." + echo " This will run build.sh inside the dockerfile as defined in eng/docker/\$image.Dockerfile." if [[ "${1:-}" != '--no-exit' ]]; then exit 2 @@ -101,7 +101,7 @@ if [ ! -z "$commit_hash" ]; then build_args[${#build_args[*]}]="-p:SourceRevisionId=$commit_hash" fi -dockerfile="$DIR/build/docker/$image.Dockerfile" +dockerfile="$DIR/eng/docker/$image.Dockerfile" tagname="aspnetcore-build-$image" # Use docker pull with retries to pre-pull the image need by the dockerfile @@ -130,8 +130,7 @@ docker build "$(dirname "$dockerfile")" \ docker run \ --rm \ -t \ - -e CI \ - -e TEAMCITY_VERSION \ + -e TF_BUILD \ -e BUILD_NUMBER \ -e BUILD_BUILDNUMBER \ -e BUILD_REPOSITORY_URI \ diff --git a/docs/BuildFromSource.md b/docs/BuildFromSource.md index 8c1506376198..d84fe2ab8c19 100644 --- a/docs/BuildFromSource.md +++ b/docs/BuildFromSource.md @@ -95,10 +95,6 @@ The cause of this problem is that the solution you are using does not include th ``` dotnet sln add C:\src\AspNetCore\src\Hosting\Abstractions\src\Microsoft.AspNetCore.Hosting.Abstractions.csproj ``` - Or you can use this script to automatically traverse the project reference graph, which then invokes `dotnet sln` for you: [eng/scripts/AddAllProjectRefsToSolution.ps1](/eng/scripts/AddAllProjectRefsToSolution.ps1). - ``` - ./eng/scripts/AddAllProjectRefsToSolution.ps1 -WorkingDir src/Mvc/ - ``` ## Building with Visual Studio Code diff --git a/docs/Helix.md b/docs/Helix.md index 3ac9c55f108a..c6859779811b 100644 --- a/docs/Helix.md +++ b/docs/Helix.md @@ -13,13 +13,13 @@ To run Helix tests for one particular test project: ``` cd src/MyCode/test -dotnet build /t:Helix +dotnet msbuild /t:Helix ``` To run tests for the entire repo, run: ``` -.\build.cmd /t:Helix +.\eng\scripts\TestHelix.ps1 ``` This will restore, and then publish all of the test projects including some bootstrapping scripts that will install the correct dotnet runtime/sdk before running the test assemblies on the helix machine, and upload the job to helix, it won't wait for the jobs to complete, but you can go to https://mc.dot.net/#/user/$(your user name)/builds. diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets new file mode 100644 index 000000000000..3857e1523950 --- /dev/null +++ b/eng/AfterSolutionBuild.targets @@ -0,0 +1,31 @@ + + + + + + + + + + <_BuildOutput Include="$(ArtifactsShippingPackagesDir)*.nupkg" + Exclude="$(ArtifactsShippingPackagesDir)*.symbols.nupkg" /> + + + + + + + + + + + + + diff --git a/build/repo.props b/eng/Build.props similarity index 81% rename from build/repo.props rename to eng/Build.props index bba7a1cc4344..eba7a2ac911a 100644 --- a/build/repo.props +++ b/eng/Build.props @@ -10,31 +10,6 @@ true - - - true - true - false - true - - false - - $(ArtifactsDir)packages\$(Configuration)\ - $(ArtifactsPackagesDir)Shipping\ - $(ArtifactsPackagesDir)NonShipping\ - $(ArtifactsDir)installers\$(Configuration)\ - $(ArtifactsDir)VSSetup\$(Configuration)\ - $(ArtifactsDir)log\ - - $(RepoRoot)eng\signcheck.exclusions.txt - - - true - - $(RepoRoot)src\Shared\ - true - - @@ -73,9 +48,9 @@ - + - + false @@ -108,9 +83,14 @@ - - + + + + + @@ -181,10 +161,4 @@ - - - - - - diff --git a/eng/CodeGen.proj b/eng/CodeGen.proj new file mode 100644 index 000000000000..d16f2875e5f5 --- /dev/null +++ b/eng/CodeGen.proj @@ -0,0 +1,88 @@ + + + + true + $([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', '..')) + + + + + + + + + + + + + + <_SharedFrameworkAndPackageRef Include="@(_ProjectReferenceProvider->WithMetadataValue('IsAspNetCoreApp','true')->WithMetadataValue('IsShippingPackage', 'true')->Distinct())" /> + <_SharedFrameworkRef Include="@(_ProjectReferenceProvider->WithMetadataValue('IsAspNetCoreApp','true')->WithMetadataValue('IsShippingPackage', 'false')->Distinct())" /> + <_ProjectReferenceProviderWithRefAssembly Include="@(_ProjectReferenceProvider->HasMetadata('ReferenceAssemblyProjectFileRelativePath'))" /> + <_ProjectReferenceProvider Remove="@(_ProjectReferenceProviderWithRefAssembly)" /> + + + + $(MSBuildThisFileDirectory)ProjectReferences.props + + + + @(_ProjectReferenceProvider->'', '%0A ') + @(_ProjectReferenceProviderWithRefAssembly->'', '%0A ') + + +]]> + + + + + + + + + $(MSBuildThisFileDirectory)SharedFramework.Local.props + + and properties from each .csproj in this repository. +--> + + + + @(_SharedFrameworkAndPackageRef->'', '%0A ') + + + @(_SharedFrameworkRef->'', '%0A ') + + + ]]> + + + + + + + + + + + + diff --git a/eng/FlakyTests.BeforeArcade.props b/eng/FlakyTests.BeforeArcade.props index 1cd94cc9cd8f..02f9fb6c5963 100644 --- a/eng/FlakyTests.BeforeArcade.props +++ b/eng/FlakyTests.BeforeArcade.props @@ -6,13 +6,13 @@ - + <_FlakyRunAdditionalArgs>$(_FlakyRunAdditionalArgs) -trait "Flaky:AzP:All=true" -trait "Flaky:AzP:OS:$(AGENT_OS)=true" <_NonFlakyRunAdditionalArgs>$(_NonFlakyRunAdditionalArgs) -notrait "Flaky:AzP:All=true" -notrait "Flaky:AzP:OS:$(AGENT_OS)=true" - $(_NonFlakyRunAdditionalArgs) - $(_FlakyRunAdditionalArgs) + $(_NonFlakyRunAdditionalArgs) $(TestRunnerAdditionalArguments) + $(_FlakyRunAdditionalArgs) $(TestRunnerAdditionalArguments) diff --git a/eng/NuGetPackageVerifier.json b/eng/NuGetPackageVerifier.json deleted file mode 100644 index 8c6f0eedfab9..000000000000 --- a/eng/NuGetPackageVerifier.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "adx": { - "rules": [ - "AdxVerificationCompositeRule" - ], - "packages": { - "dotnet-watch": { - "packageTypes": [ - "DotnetTool" - ] - }, - "dotnet-sql-cache": { - "packageTypes": [ - "DotnetTool" - ] - }, - "dotnet-user-secrets": { - "packageTypes": [ - "DotnetTool" - ] - }, - "dotnet-dev-certs": { - "packageTypes": [ - "DotnetTool" - ] - }, - "Microsoft.AspNetCore.DeveloperCertificates.XPlat": { - "Exclusions": { - "DOC_MISSING": { - "lib/netcoreapp3.0/Microsoft.AspNetCore.DeveloperCertificates.XPlat.dll": "Docs not required to shipoob package" - } - } - } - } - }, - "Default": { - "rules": [ - "DefaultCompositeRule" - ] - } -} diff --git a/eng/SharedFramework.External.props b/eng/SharedFramework.External.props index afd88d32a3c7..a6390c8a4d4d 100644 --- a/eng/SharedFramework.External.props +++ b/eng/SharedFramework.External.props @@ -91,6 +91,9 @@ This references are part of Microsoft.NETCore.App, so are listed here as references to be used during compilation only. --> + <_CompilationOnlyReference Include="Microsoft.Win32.Registry" /> + <_CompilationOnlyReference Include="System.Security.Cryptography.Cng" /> + <_CompilationOnlyReference Include="System.Security.Principal.Windows" /> <_CompilationOnlyReference Include="System.Buffers" /> <_CompilationOnlyReference Include="System.ComponentModel.Annotations" /> <_CompilationOnlyReference Include="System.Runtime.CompilerServices.Unsafe" /> diff --git a/eng/SharedFramework.Local.props b/eng/SharedFramework.Local.props index 065172aad2e5..798e1469f270 100644 --- a/eng/SharedFramework.Local.props +++ b/eng/SharedFramework.Local.props @@ -8,8 +8,15 @@ + + + + + + + @@ -20,11 +27,6 @@ - - - - - @@ -40,8 +42,6 @@ - - diff --git a/eng/Signing.props b/eng/Signing.props new file mode 100644 index 000000000000..26491b99d1ff --- /dev/null +++ b/eng/Signing.props @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(TargetOsName)-$(TargetArchitecture) + $(ArtifactsObjDir)RedistSharedFx.Layout\$(Configuration)\ + $(BaseRedistNetCorePath)$(TargetRuntimeIdentifier)\ + + + + + + + + + + + + + + + <_DotNetFilesToExclude Include="$(BaseRedistNetCorePath)win-x64\shared\Microsoft.NETCore.App\**\*.dll" CertificateName="None" /> + <_DotNetFilesToExclude Include="$(BaseRedistNetCorePath)win-x86\shared\Microsoft.NETCore.App\**\*.dll" CertificateName="None" /> + <_DotNetFilesToExclude Include="$(BaseRedistNetCorePath)win-arm\shared\Microsoft.NETCore.App\**\*.dll" CertificateName="None" /> + <_DotNetFilesToExclude Include="$(BaseRedistNetCorePath)win-x64\host\**\*.dll" CertificateName="None" /> + <_DotNetFilesToExclude Include="$(BaseRedistNetCorePath)win-x86\host\**\*.dll" CertificateName="None" /> + <_DotNetFilesToExclude Include="$(BaseRedistNetCorePath)win-arm\host\**\*.dll" CertificateName="None" /> + <_DotNetFilesToExclude Include="$(RedistNetCorePath)dotnet.exe" CertificateName="None" /> + + + + diff --git a/eng/Tools.props b/eng/Tools.props new file mode 100644 index 000000000000..46285b1448e0 --- /dev/null +++ b/eng/Tools.props @@ -0,0 +1,12 @@ + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 34038febe752..57caf7ff1853 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -9,25 +9,25 @@ --> - + https://github.com/aspnet/Blazor - fc20643ca2493c56180e4f5a00946116a3b6afdc + 8df334032840bda9221fe281890e88fb1de06845 - + https://github.com/aspnet/AspNetCore-Tooling - 271045bfd76b339ab1c8b2555507ad727b068aa3 + 5e7c4e5e382a7e486d9416b711fcd65bae25808b - + https://github.com/aspnet/AspNetCore-Tooling - 271045bfd76b339ab1c8b2555507ad727b068aa3 + 5e7c4e5e382a7e486d9416b711fcd65bae25808b - + https://github.com/aspnet/AspNetCore-Tooling - 271045bfd76b339ab1c8b2555507ad727b068aa3 + 5e7c4e5e382a7e486d9416b711fcd65bae25808b - + https://github.com/aspnet/AspNetCore-Tooling - 271045bfd76b339ab1c8b2555507ad727b068aa3 + 5e7c4e5e382a7e486d9416b711fcd65bae25808b https://github.com/aspnet/EntityFrameworkCore @@ -404,17 +404,17 @@ https://github.com/aspnet/Extensions 42b3a303ba27594637de04c78aaf9f1c7af6e303 - + https://github.com/dotnet/arcade - e6a5d5f970bb872451c6310ae34eda31041fb552 + 9946534da4f73e6242ca105f6798ab58119c9ab0 - + https://github.com/dotnet/arcade - e6a5d5f970bb872451c6310ae34eda31041fb552 + 9946534da4f73e6242ca105f6798ab58119c9ab0 - + https://github.com/dotnet/arcade - e6a5d5f970bb872451c6310ae34eda31041fb552 + 9946534da4f73e6242ca105f6798ab58119c9ab0 https://github.com/aspnet/Extensions diff --git a/eng/Versions.props b/eng/Versions.props index e34c41dc2702..407baf8c53cd 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -52,7 +52,7 @@ --> - 1.0.0-beta.19302.2 + 1.0.0-beta.19323.4 3.0.0-preview7-27812-08 3.0.0-preview7-27812-08 @@ -82,7 +82,7 @@ 3.0.0-preview7.19312.3 - 0.10.0-preview7.19310.1 + 0.10.0-preview7.19323.1 3.0.0-preview7.19312.4 3.0.0-preview7.19312.4 @@ -154,10 +154,10 @@ 3.0.0-preview7.19313.2 3.0.0-preview7.19313.2 - 3.0.0-preview7.19316.1 - 3.0.0-preview7.19316.1 - 3.0.0-preview7.19316.1 - 3.0.0-preview7.19316.1 + 3.0.0-preview7.19321.2 + 3.0.0-preview7.19321.2 + 3.0.0-preview7.19321.2 + 3.0.0-preview7.19321.2 - - - - $(BaseIntermediateOutputPath) - $(MSBuildProjectDir)\obj\ - - - - - <_ProjectExtensionsWereImported>true - $(WixInstallPath)\wix2010.targets - - @@ -44,4 +31,16 @@ false + + + + $(TestRunnerAdditionalArguments) -nocolor + + + + + + true + diff --git a/eng/common/PSScriptAnalyzerSettings.psd1 b/eng/common/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 000000000000..4c1ea7c98ea4 --- /dev/null +++ b/eng/common/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,11 @@ +@{ + IncludeRules=@('PSAvoidUsingCmdletAliases', + 'PSAvoidUsingWMICmdlet', + 'PSAvoidUsingPositionalParameters', + 'PSAvoidUsingInvokeExpression', + 'PSUseDeclaredVarsMoreThanAssignments', + 'PSUseCmdletCorrectly', + 'PSStandardDSCFunctionsInResource', + 'PSUseIdenticalMandatoryParametersForDSC', + 'PSUseIdenticalParametersForDSC') +} \ No newline at end of file diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 4cb2ce489b6c..feb58d14191c 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -134,7 +134,7 @@ try { } catch { Write-Host $_.ScriptStackTrace - Write-PipelineTaskError -Message $_ + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index ba16e991c7ec..ed70e0a86ebd 100755 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -157,15 +157,15 @@ fetch_tizen_pkgs() Inform "Initialize arm base" fetch_tizen_pkgs_init standard base Inform "fetch common packages" -fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel +fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic fetch_tizen_pkgs noarch linux-glibc-devel Inform "fetch coreclr packages" -fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel tizen-release lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl-devel krb5 krb5-devel libcurl libcurl-devel Inform "Initialize standard unified" fetch_tizen_pkgs_init standard unified Inform "fetch corefx packages" -fetch_tizen_pkgs armv7l gssdp gssdp-devel +fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 34d3d2ba1fe1..d7d5d7d5f449 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -181,7 +181,7 @@ if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then fi if [ -z "$__RootfsDir" ]; then - __RootfsDir="$__CrossDir/rootfs/$__BuildArch" + __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" fi if [ -d "$__RootfsDir" ]; then @@ -203,6 +203,7 @@ if [[ "$__LinuxCodeName" == "alpine" ]]; then -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ add $__AlpinePackages rm -r $__ApkToolsDir diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index dea7cdd903b1..8854d979f379 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -11,10 +11,10 @@ function InstallDarcCli ($darcVersion) { $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = Invoke-Expression "& `"$dotnet`" tool list -g" + $toolList = & "$dotnet" tool list -g if ($toolList -like "*$darcCliPackageName*") { - Invoke-Expression "& `"$dotnet`" tool uninstall $darcCliPackageName -g" + & "$dotnet" tool uninstall $darcCliPackageName -g } # If the user didn't explicitly specify the darc version, @@ -22,12 +22,12 @@ function InstallDarcCli ($darcVersion) { if (-not $darcVersion) { $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } - + $arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - Invoke-Expression "& `"$dotnet`" tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" + & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g } InstallDarcCli $darcVersion diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index a05b84f7987b..b056e4c1ac2a 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -25,7 +25,7 @@ function CheckExitCode ([string]$stage) try { Push-Location $PSScriptRoot - + Write-Host "Installing darc..." . .\darc-init.ps1 -darcVersion $darcVersion CheckExitCode "Running darc-init" @@ -40,9 +40,9 @@ try { $darcExe = "$env:USERPROFILE\.dotnet\tools" $darcExe = Resolve-Path "$darcExe\darc.exe" - + Create-Directory $outputFolder - + # Generate 3 graph descriptions: # 1. Flat with coherency information # 2. Graphviz (dot) file @@ -51,26 +51,26 @@ try { $graphVizImageFilePath = "$outputFolder\graph.png" $normalGraphFilePath = "$outputFolder\graph-full.txt" $flatGraphFilePath = "$outputFolder\graph-flat.txt" - $baseOptions = "get-dependency-graph --github-pat $gitHubPat --azdev-pat $azdoPat --password $barToken" - + $baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" ) + if ($includeToolset) { Write-Host "Toolsets will be included in the graph..." - $baseOptions += " --include-toolset" + $baseOptions += @( "--include-toolset" ) } Write-Host "Generating standard dependency graph..." - Invoke-Expression "& `"$darcExe`" $baseOptions --output-file $normalGraphFilePath" + & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath CheckExitCode "Generating normal dependency graph" Write-Host "Generating flat dependency graph and graphviz file..." - Invoke-Expression "& `"$darcExe`" $baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath" + & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath CheckExitCode "Generating flat and graphviz dependency graph" Write-Host "Generating graph image $graphVizFilePath" $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" - Invoke-Expression "& `"$dotFilePath`" -Tpng -o'$graphVizImageFilePath' `"$graphVizFilePath`"" + & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" CheckExitCode "Generating graphviz image" - + Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" } catch { diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index a4306bd37e14..9d18645f4553 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -79,28 +79,27 @@ try { $NativeTools.PSObject.Properties | ForEach-Object { $ToolName = $_.Name $ToolVersion = $_.Value - $LocalInstallerCommand = $InstallerPath - $LocalInstallerCommand += " -ToolName $ToolName" - $LocalInstallerCommand += " -InstallPath $InstallBin" - $LocalInstallerCommand += " -BaseUri $BaseUri" - $LocalInstallerCommand += " -CommonLibraryDirectory $EngCommonBaseDir" - $LocalInstallerCommand += " -Version $ToolVersion" + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } if ($Verbose) { - $LocalInstallerCommand += " -Verbose" + $LocalInstallerArguments += @{ Verbose = $True } } if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { if($Force) { - $LocalInstallerCommand += " -Force" + $LocalInstallerArguments += @{ Force = $True } } } if ($Clean) { - $LocalInstallerCommand += " -Clean" + $LocalInstallerArguments += @{ Clean = $True } } Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$LocalInstallerCommand'" - Invoke-Expression "$LocalInstallerCommand" + Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'" + & $InstallerPath @LocalInstallerArguments if ($LASTEXITCODE -Ne "0") { $errMsg = "$ToolName installation failed" if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 new file mode 100644 index 000000000000..8b8bafd6a896 --- /dev/null +++ b/eng/common/internal-feed-operations.ps1 @@ -0,0 +1,135 @@ +param( + [Parameter(Mandatory=$true)][string] $Operation, + [string] $AuthToken, + [string] $CommitSha, + [string] $RepoName, + [switch] $IsFeedPrivate +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\tools.ps1 + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified +# internal builds +function SetupCredProvider { + param( + [string] $AuthToken + ) + + # Install the Cred Provider NuGet plugin + Write-Host "Setting up Cred Provider NuGet plugin in the agent..." + Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' + + Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." + Invoke-WebRequest $url -OutFile installcredprovider.ps1 + + Write-Host "Installing plugin..." + .\installcredprovider.ps1 -Force + + Write-Host "Deleting local copy of 'installcredprovider.ps1'..." + Remove-Item .\installcredprovider.ps1 + + if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { + Write-Host "CredProvider plugin was not installed correctly!" + ExitWithExitCode 1 + } + else { + Write-Host "CredProvider plugin was installed correctly!" + } + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + $nugetConfigPath = "$RepoRoot\NuGet.config" + + if (-Not (Test-Path -Path $nugetConfigPath)) { + Write-Host "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + } + + $endpoints = New-Object System.Collections.ArrayList + $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} + + if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { + foreach ($stableRestoreResource in $nugetConfigPackageSources) { + $trimmedResource = ([string]$stableRestoreResource).Trim() + [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) + } + } + + if (($endpoints | Measure-Object).Count -gt 0) { + # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' + $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress + + # Create the environment variables the AzDo way + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ + 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' + 'issecret' = 'false' + } + + # We don't want sessions cached since we will be updating the endpoints quite frequently + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ + 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' + 'issecret' = 'false' + } + } + else + { + Write-Host "No internal endpoints found in NuGet.config" + } +} + +#Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + $dotnetTempDir = "$RepoRoot\dotnet" + $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + $dotnet = "$dotnetTempDir\dotnet.exe" + $restoreProjPath = "$PSScriptRoot\restore.proj" + + Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + '' | Out-File "$restoreProjPath" + + & $dotnet restore $restoreProjPath + + Write-Host "Arcade SDK restored!" + + if (Test-Path -Path $restoreProjPath) { + Remove-Item $restoreProjPath + } + + if (Test-Path -Path $dotnetTempDir) { + Remove-Item $dotnetTempDir -Recurse + } +} + +try { + Push-Location $PSScriptRoot + + if ($Operation -like "setup") { + SetupCredProvider $AuthToken + } + elseif ($Operation -like "install-restore") { + InstallDotNetSdkAndRestoreArcade + } + else { + Write-Host "Unknown operation '$Operation'!" + ExitWithExitCode 1 + } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} +finally { + Pop-Location +} diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh new file mode 100644 index 000000000000..1ff654d2ffcd --- /dev/null +++ b/eng/common/internal-feed-operations.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +set -e + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. +# This should ONLY be called from identified internal builds +function SetupCredProvider { + local authToken=$1 + + # Install the Cred Provider NuGet plugin + echo "Setting up Cred Provider NuGet plugin in the agent..."... + echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" + + echo "Writing the contents of 'installcredprovider.ps1' locally..." + local installcredproviderPath="installcredprovider.sh" + if command -v curl > /dev/null; then + curl $url > "$installcredproviderPath" + else + wget -q -O "$installcredproviderPath" "$url" + fi + + echo "Installing plugin..." + . "$installcredproviderPath" + + echo "Deleting local copy of 'installcredprovider.sh'..." + rm installcredprovider.sh + + if [ ! -d "$HOME/.nuget/plugins" ]; then + echo "CredProvider plugin was not installed correctly!" + ExitWithExitCode 1 + else + echo "CredProvider plugin was installed correctly!" + fi + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + local nugetConfigPath="$repo_root/NuGet.config" + + if [ ! "$nugetConfigPath" ]; then + echo "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + fi + + local endpoints='[' + local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` + local pattern="value=\"(.*)\"" + + for value in $nugetConfigPackageValues + do + if [[ $value =~ $pattern ]]; then + local endpoint="${BASH_REMATCH[1]}" + endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," + fi + done + + endpoints=${endpoints%?} + endpoints+=']' + + if [ ${#endpoints} -gt 2 ]; then + # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' + local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" + + echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" + echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" + else + echo "No internal endpoints found in NuGet.config" + fi +} + +# Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + local dotnetTempDir="$repo_root/dotnet" + local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + local restoreProjPath="$repo_root/eng/common/restore.proj" + + echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + echo "" > "$restoreProjPath" + + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + local res=`$dotnetTempDir/dotnet restore $restoreProjPath` + echo "Arcade SDK restored!" + + # Cleanup + if [ "$restoreProjPath" ]; then + rm "$restoreProjPath" + fi + + if [ "$dotnetTempDir" ]; then + rm -r $dotnetTempDir + fi +} + +source="${BASH_SOURCE[0]}" +operation='' +authToken='' +repoName='' + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | awk '{print tolower($0)}')" + case "$opt" in + --operation) + operation=$2 + shift + ;; + --authtoken) + authToken=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ "$operation" = "setup" ]; then + SetupCredProvider $authToken +elif [ "$operation" = "install-restore" ]; then + InstallDotNetSdkAndRestoreArcade +else + echo "Unknown operation '$operation'!" +fi diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index f286ae0cde2a..7a34c7e8a426 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -209,7 +209,7 @@ function New-ScriptShim { Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") } - Invoke-Expression "$ShimDirectory\WinShimmer\winshimmer.exe $ShimName $ToolFilePath $ShimDirectory" + & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory return $True } catch { diff --git a/eng/common/LoggingCommandFunctions.ps1 b/eng/common/pipeline-logging-functions.ps1 similarity index 68% rename from eng/common/LoggingCommandFunctions.ps1 rename to eng/common/pipeline-logging-functions.ps1 index c225eaecbf25..7b61376f8aab 100644 --- a/eng/common/LoggingCommandFunctions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -1,4 +1,4 @@ -# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 +# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. # NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 @@ -12,6 +12,93 @@ $script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%" # TODO: BUG: Escape % ??? # TODO: Add test to verify don't need to escape "=". +function Write-PipelineTelemetryError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Category, + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $PSBoundParameters.Remove("Category") | Out-Null + + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + $PSBoundParameters.Remove("Message") | Out-Null + $PSBoundParameters.Add("Message", $Message) + + Write-PipelineTaskError @PSBoundParameters +} + +function Write-PipelineTaskError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + if(!$ci) { + if($Type -eq 'error') { + Write-Host $Message -ForegroundColor Red + return + } + elseif ($Type -eq 'warning') { + Write-Host $Message -ForegroundColor Yellow + return + } + } + + if(($Type -ne 'error') -and ($Type -ne 'warning')) { + Write-Host $Message + return + } + if(-not $PSBoundParameters.ContainsKey('Type')) { + $PSBoundParameters.Add('Type', 'error') + } + Write-LogIssue @PSBoundParameters + } + + function Write-PipelineSetVariable { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput) + + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ + 'variable' = $Name + 'isSecret' = $Secret + 'isOutput' = 'true' + } -AsOutput:$AsOutput + } + } + + function Write-PipelinePrependPath { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [string]$Path, + [switch]$AsOutput) + if($ci) { + Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput + } + } + <######################################## # Private functions. ########################################> diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh new file mode 100644 index 000000000000..6098f9a5438a --- /dev/null +++ b/eng/common/pipeline-logging-functions.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +function Write-PipelineTelemetryError { + local telemetry_category='' + local function_args=() + local message='' + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -category|-c) + telemetry_category=$2 + shift + ;; + -*) + function_args+=("$1 $2") + shift + ;; + *) + message=$* + ;; + esac + shift + done + + if [[ "$ci" != true ]]; then + echo "$message" >&2 + return + fi + + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + + Write-PipelineTaskError $function_args +} + +function Write-PipelineTaskError { + if [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + + message_type="error" + sourcepath='' + linenumber='' + columnnumber='' + error_code='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -type|-t) + message_type=$2 + shift + ;; + -sourcepath|-s) + sourcepath=$2 + shift + ;; + -linenumber|-ln) + linenumber=$2 + shift + ;; + -columnnumber|-cn) + columnnumber=$2 + shift + ;; + -errcode|-e) + error_code=$2 + shift + ;; + *) + break + ;; + esac + + shift + done + + message="##vso[task.logissue" + + message="$message type=$message_type" + + if [ -n "$sourcepath" ]; then + message="$message;sourcepath=$sourcepath" + fi + + if [ -n "$linenumber" ]; then + message="$message;linenumber=$linenumber" + fi + + if [ -n "$columnnumber" ]; then + message="$message;columnnumber=$columnnumber" + fi + + if [ -n "$error_code" ]; then + message="$message;code=$error_code" + fi + + message="$message]$*" + echo "$message" +} + diff --git a/eng/common/post-build/dotnetsymbol-init.ps1 b/eng/common/post-build/dotnetsymbol-init.ps1 new file mode 100644 index 000000000000..e7659b98c8c3 --- /dev/null +++ b/eng/common/post-build/dotnetsymbol-init.ps1 @@ -0,0 +1,29 @@ +param ( + $dotnetsymbolVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function Installdotnetsymbol ($dotnetsymbolVersion) { + $dotnetsymbolPackageName = "dotnet-symbol" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + } + else { + Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity $verbosity --global + } +} + +Installdotnetsymbol $dotnetsymbolVersion diff --git a/eng/common/post-build/sourcelink-cli-init.ps1 b/eng/common/post-build/sourcelink-cli-init.ps1 new file mode 100644 index 000000000000..9eaa25b3b507 --- /dev/null +++ b/eng/common/post-build/sourcelink-cli-init.ps1 @@ -0,0 +1,29 @@ +param ( + $sourcelinkCliVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function InstallSourcelinkCli ($sourcelinkCliVersion) { + $sourcelinkCliPackageName = "sourcelink" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { + Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." + } + else { + Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity $verbosity --global + } +} + +InstallSourcelinkCli $sourcelinkCliVersion diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 new file mode 100644 index 000000000000..8abd684e9e57 --- /dev/null +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -0,0 +1,224 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages + [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + . $using:PSScriptRoot\..\tools.ps1 + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @(".dll", ".exe", ".pdb") + + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith(".resources.dll")) { + return + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" + $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" + $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" + + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + $Status = 0 + } + } + catch { + write-host $_ + $Status = 0 + } + } + + if ($Status -ne 200) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + } + catch { + + } + finally { + $zip.Dispose() + } + + if ($FailedFiles -eq 0) { + Write-Host "Passed." + } + else { + Write-PipelineTaskError "$PackagePath has broken SourceLink links." + } +} + +function ValidateSourceLinkLinks { + if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { + Write-PipelineTaskError "GHRepoName should be in the format / or -" + ExitWithExitCode 1 + } + else { + $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; + } + } + + if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string" + ExitWithExitCode 1 + } + + $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" + Write-Host $_ + ExitWithExitCode 1 + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + # Process each NuGet package in parallel + $Jobs = @() + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing SourceLink CLI..." + Get-Location + . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion + CheckExitCode "Running sourcelink-cli-init" + + Measure-Command { ValidateSourceLinkLinks } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 new file mode 100644 index 000000000000..69456854e04f --- /dev/null +++ b/eng/common/post-build/symbols-validation.ps1 @@ -0,0 +1,186 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function FirstMatchingSymbolDescriptionOrDefault { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + "\" + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") + + $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + + & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return "PDB" + } + elseif (Test-Path $NGenPdb) { + return "NGen PDB" + } + elseif (Test-Path $SODbg) { + return "DBG for SO" + } + elseif (Test-Path $DylibDwarf) { + return "Dwarf for Dylib" + } + elseif (Test-Path $SymbolPath) { + return "Module" + } + else { + return $null + } +} + +function CountMissingSymbols { + param( + [string] $PackagePath # Path to a NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for symbols + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") + + # How many files are missing symbol information + $MissingSymbols = 0 + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $PackageGuid = New-Guid + $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" + + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + + Get-ChildItem -Recurse $ExtractPath | + Where-Object {$RelevantExtensions -contains $_.Extension} | + ForEach-Object { + if ($_.FullName -Match "\\ref\\") { + Write-Host "`t Ignoring reference assembly file" $_.FullName + return + } + + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath + + Write-Host -NoNewLine "`t Checking file" $_.FullName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host "No symbols found on MSDL or SymWeb!" + } + else { + if ($SymbolsOnMSDL -eq $null) { + Write-Host "No symbols found on MSDL!" + } + else { + Write-Host "No symbols found on SymWeb!" + } + } + } + } + + Pop-Location + + return $MissingSymbols +} + +function CheckSymbolsAvailable { + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $FileName = $_.Name + + # These packages from Arcade-Services include some native libraries that + # our current symbol uploader can't handle. Below is a workaround until + # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + + Write-Host "Validating $FileName " + $Status = CountMissingSymbols "$InputPath\$FileName" + + if ($Status -ne 0) { + Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" + ExitWithExitCode $exitCode + } + + Write-Host + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing dotnet symbol ..." + Get-Location + . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion + CheckExitCode "Running dotnetsymbol-init" + + CheckSymbolsAvailable +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config new file mode 100644 index 000000000000..0c5451c11415 --- /dev/null +++ b/eng/common/sdl/NuGet.config @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 new file mode 100644 index 000000000000..0635f26fb637 --- /dev/null +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -0,0 +1,97 @@ +Param( + [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located + [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located + [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault + [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code + [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel="Standard" # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +#Replace repo names to the format of org/repo +if (!($Repository.contains('/'))) { + $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; +} +else{ + $RepoName = $Repository; +} + +if ($GuardianPackageName) { + $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) +} else { + $guardianCliLocation = $GuardianCliLocation +} + +$ValidPath = Test-Path $guardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" + +if ($TsaOnboard) { + if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified." + exit 1 + } +} + +if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} +if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel +} + +if ($UpdateBaseline) { + & (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Update baseline" +} + +if ($TsaPublish) { + if ($TsaBranchName -and $BuildNumber) { + if (-not $TsaRepositoryName) { + $TsaRepositoryName = "$($Repository)-$($BranchName)" + } + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified." + exit 1 + } +} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 new file mode 100644 index 000000000000..26e01c0673c1 --- /dev/null +++ b/eng/common/sdl/init-sdl.ps1 @@ -0,0 +1,48 @@ +Param( + [string] $GuardianCliLocation, + [string] $Repository, + [string] $BranchName="master", + [string] $WorkingDirectory, + [string] $AzureDevOpsAccessToken, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file +$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) +$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" +$zipFile = "$WorkingDirectory/gdn.zip" + +Add-Type -AssemblyName System.IO.Compression.FileSystem +$gdnFolder = (Join-Path $WorkingDirectory ".gdn") +Try +{ + # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead + Write-Host "Downloading gdn folder from internal config repostiory..." + Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile + if (Test-Path $gdnFolder) { + # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) + Remove-Item -Force -Recurse $gdnFolder + } + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) + Write-Host $gdnFolder +} Catch [System.Net.WebException] { + # if the folder does not exist, we'll do a guardian init and push it to the remote repository + Write-Host "Initializing Guardian..." + Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" + & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian init failed with exit code $LASTEXITCODE." + } + # We create the mainbaseline so it can be edited later + Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" + & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline + if ($LASTEXITCODE -ne 0) { + Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." + } + & $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder" +} \ No newline at end of file diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config new file mode 100644 index 000000000000..b054737df13e --- /dev/null +++ b/eng/common/sdl/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 new file mode 100644 index 000000000000..79c707d6d8a0 --- /dev/null +++ b/eng/common/sdl/push-gdn.ps1 @@ -0,0 +1,51 @@ +Param( + [string] $Repository, + [string] $BranchName="master", + [string] $GdnFolder, + [string] $AzureDevOpsAccessToken, + [string] $PushReason +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We create the temp directory where we'll store the sdl-config repository +$sdlDir = Join-Path $env:TEMP "sdl" +if (Test-Path $sdlDir) { + Remove-Item -Force -Recurse $sdlDir +} + +Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" +git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir +if ($LASTEXITCODE -ne 0) { + Write-Error "Git clone failed with exit code $LASTEXITCODE." +} +# We copy the .gdn folder from our local run into the git repository so it can be committed +$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" +if (Get-Command Robocopy) { + Robocopy /S $GdnFolder $sdlRepositoryFolder +} else { + rsync -r $GdnFolder $sdlRepositoryFolder +} +# cd to the sdl-config directory so we can run git there +Push-Location $sdlDir +# git add . --> git commit --> git push +Write-Host "git add ." +git add . +if ($LASTEXITCODE -ne 0) { + Write-Error "Git add failed with exit code $LASTEXITCODE." +} +Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" +git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" +if ($LASTEXITCODE -ne 0) { + Write-Error "Git commit failed with exit code $LASTEXITCODE." +} +Write-Host "git push" +git push +if ($LASTEXITCODE -ne 0) { + Write-Error "Git push failed with exit code $LASTEXITCODE." +} + +# Return to the original directory +Pop-Location \ No newline at end of file diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 new file mode 100644 index 000000000000..e6a86d03a210 --- /dev/null +++ b/eng/common/sdl/run-sdl.ps1 @@ -0,0 +1,65 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + [string[]] $ToolsList, + [string] $UpdateBaseline, + [string] $GuardianLoggerLevel="Standard" +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +$LASTEXITCODE = 0 + +# We store config files in the r directory of .gdn +Write-Host $ToolsList +$gdnConfigPath = Join-Path $GdnFolder "r" +$ValidPath = Test-Path $GuardianCliLocation + +if ($ValidPath -eq $False) +{ + Write-Host "Invalid Guardian CLI Location." + exit 1 +} + +foreach ($tool in $ToolsList) { + $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" + $config = $False + Write-Host $tool + # We have to manually configure tools that run on source to look at the source directory only + if ($tool -eq "credscan") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + if ($tool -eq "policheck") { + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `"" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + $config = $True + } + + Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config" + if ($config) { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } else { + & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE." + exit $LASTEXITCODE + } + } +} + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml new file mode 100644 index 000000000000..acb4c55d735f --- /dev/null +++ b/eng/common/templates/job/execute-sdl.yml @@ -0,0 +1,44 @@ +parameters: + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + continueOnError: false # optional: determines whether to continue the build if the step errors; + dependsOn: '' # Optional: dependencies of the job + +jobs: +- job: Run_SDL + dependsOn: ${{ parameters.dependsOn }} + displayName: Run SDL tool + variables: + - group: DotNet-VSTS-Bot + steps: + - checkout: self + clean: true + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: current + downloadType: specific files + matchingPattern: "**" + downloadPath: $(Build.SourcesDirectory)\artifacts + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + - task: NuGetCommand@2 + displayName: 'Install Guardian' + inputs: + restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + feedsToUse: config + nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config + externalFeedCredentials: GuardianConnect + restoreDirectory: $(Build.SourcesDirectory)\.packages + - ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.continueOnError }} + - ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 + -GuardianPackageName Microsoft.Guardian.Cli.0.3.2 + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 620bd3c62e78..ff7346163f43 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -59,6 +59,21 @@ jobs: /p:Configuration=$(_BuildConfig) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsInternalBuild) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + - task: PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml new file mode 100644 index 000000000000..c61eaa927d95 --- /dev/null +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -0,0 +1,145 @@ +parameters: + enableSymbolValidation: true + +stages: +- stage: Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Developer Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + buildType: current + artifactName: PDBArtifacts + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: + displayName: Publish to Static Feed + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet + /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' + /p:OverrideAssetsWithSameName=true + /p:PassIfExistingItemIdentical=true + /p:Configuration=Release + + +- stage: PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) --latest-location + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml new file mode 100644 index 000000000000..23725c6d620c --- /dev/null +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -0,0 +1,91 @@ +stages: +- stage: PVR_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Validation Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Publish to Static Feed + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet + /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' + /p:OverrideAssetsWithSameName=true + /p:PassIfExistingItemIdentical=true + /p:Configuration=Release + + +- stage: PVR_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) --latest-location + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml new file mode 100644 index 000000000000..97b48d97fece --- /dev/null +++ b/eng/common/templates/post-build/common-variables.yml @@ -0,0 +1,9 @@ +variables: + # .NET Core 3 Dev + PublicDevRelease_30_Channel_Id: 3 + + # .NET Tools - Validation + PublicValidationRelease_30_Channel_Id: 9 + + SourceLinkCLIVersion: 3.0.0 + SymbolToolVersion: 1.0.1 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml new file mode 100644 index 000000000000..2c411dd0098c --- /dev/null +++ b/eng/common/templates/post-build/post-build.yml @@ -0,0 +1,67 @@ +parameters: + enableSourceLinkValidation: true + enableSigningValidation: true + enableSymbolValidation: true + SDLValidationParameters: + enable: false + params: '' + +stages: +- stage: validate + dependsOn: build + displayName: Validate + jobs: + - ${{ if eq(parameters.enableSigningValidation, 'true') }}: + - job: + displayName: Signing Validation + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:Configuration=Release + + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: SourceLink Validation + variables: + - template: common-variables.yml + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + + - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + +- template: \eng\common\templates\post-build\channels\public-dev-release.yml + parameters: + enableSymbolValidation: ${{ parameters.enableSymbolValidation }} + +- template: \eng\common\templates\post-build\channels\public-validation-release.yml diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml new file mode 100644 index 000000000000..d00317003b79 --- /dev/null +++ b/eng/common/templates/post-build/promote-build.yml @@ -0,0 +1,28 @@ +parameters: + ChannelId: 0 + +jobs: +- job: + displayName: Promote Build + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: ChannelId + value: ${{ parameters.ChannelId }} + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Add Build to Channel + inputs: + targetType: inline + script: | + $headers = @{ + "Accept" = "application/json" + "Authorization" = "Bearer $(MaestroAccessToken)" + } + Invoke-RestMethod -Method Post -Headers $headers -Uri https://maestro-prod.westus2.cloudapp.azure.com/api/channels/$(ChannelId)/builds/$(BARBuildId)?api-version=2019-01-16 + enabled: false diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml new file mode 100644 index 000000000000..0eddd6cd3b77 --- /dev/null +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,37 @@ +jobs: +- job: setupMaestroVars + displayName: Setup Maestro Vars + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + script: | + # This is needed to make Write-PipelineSetVariable works in this context + $ci = $true + + . "$(Build.SourcesDirectory)/eng/common/tools.ps1" + + $Content = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + + $BarId = $Content | Select -Index 0 + + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsInternalBuild = $Content | Select -Index 2 + $IsStableBuild = $Content | Select -Index 3 + + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" + Write-PipelineSetVariable -Name 'IsInternalBuild' -Value $IsInternalBuild + Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 3983d719be3f..60741f039011 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -35,7 +35,7 @@ # Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json). [string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null } -# True to attempt using .NET Core already that meets requirements specified in global.json +# True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } @@ -76,7 +76,7 @@ function Exec-Process([string]$command, [string]$commandArgs) { $finished = $false try { - while (-not $process.WaitForExit(100)) { + while (-not $process.WaitForExit(100)) { # Non-blocking loop done to allow ctr-c interrupts } @@ -92,68 +92,6 @@ function Exec-Process([string]$command, [string]$commandArgs) { } } -function Write-PipelineTaskError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - if(!$ci) { - if($Type -eq 'error') { - Write-Error $Message - return - } - elseif ($Type -eq 'warning') { - Write-Warning $Message - return - } - } - - if(($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return - } - if(-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') - } - Write-LogIssue @PSBoundParameters -} - -function Write-PipelineSetVariable { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Name, - [string]$Value, - [switch]$Secret, - [switch]$AsOutput) - - if($ci) { - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ - 'variable' = $Name - 'issecret' = $Secret - } -AsOutput:$AsOutput - } -} - -function Write-PipelinePrependPath { - [CmdletBinding()] - param( - [Parameter(Mandatory=$true)] - [string]$Path, - [switch]$AsOutput) - if($ci) { - Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput - } -} - function InitializeDotNetCli([bool]$install) { if (Test-Path variable:global:_DotNetInstallDir) { return $global:_DotNetInstallDir @@ -196,7 +134,7 @@ function InitializeDotNetCli([bool]$install) { if ($install) { InstallDotNetSdk $dotnetRoot $dotnetSdkVersion } else { - Write-PipelineTaskError "Unable to find dotnet with SDK version '$dotnetSdkVersion'" + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" ExitWithExitCode 1 } } @@ -244,13 +182,13 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit & $installScript @installParameters if ($lastExitCode -ne 0) { - Write-PipelineTaskError -Message "Failed to install dotnet cli (exit code '$lastExitCode')." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')." ExitWithExitCode $lastExitCode } } # -# Locates Visual Studio MSBuild installation. +# Locates Visual Studio MSBuild installation. # The preference order for MSBuild to use is as follows: # # 1. MSBuild from an active VS command prompt @@ -267,7 +205,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } - $vsMinVersion = [Version]::new($vsMinVersionStr) + $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. if ($env:VSINSTALLDIR -ne $null) { @@ -316,7 +254,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { $env:VSINSTALLDIR = $vsInstallDir Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\") - + $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\" if (Test-Path $vsSdkInstallDir) { Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir @@ -351,13 +289,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { # Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json. # # The following properties of tools.vs are recognized: -# "version": "{major}.{minor}" +# "version": "{major}.{minor}" # Two part minimal VS version, e.g. "15.9", "16.0", etc. -# "components": ["componentId1", "componentId2", ...] +# "components": ["componentId1", "componentId2", ...] # Array of ids of workload components that must be available in the VS instance. # See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017 # -# Returns JSON describing the located VS instance (same format as returned by vswhere), +# Returns JSON describing the located VS instance (same format as returned by vswhere), # or $null if no instance meeting the requirements is found on the machine. # function LocateVisualStudio([object]$vsRequirements = $null){ @@ -377,8 +315,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild") - + $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") + if (Get-Member -InputObject $vsRequirements -Name "version") { $args += "-version" $args += $vsRequirements.version @@ -388,7 +326,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){ foreach ($component in $vsRequirements.components) { $args += "-requires" $args += $component - } + } } $vsInfo =& $vsWhereExe $args | ConvertFrom-Json @@ -418,7 +356,7 @@ function InitializeBuildTool() { if ($msbuildEngine -eq "dotnet") { if (!$dotnetRoot) { - Write-PipelineTaskError "/global.json must specify 'tools.dotnet'." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } @@ -427,13 +365,13 @@ function InitializeBuildTool() { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore } catch { - Write-PipelineTaskError $_ + Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ ExitWithExitCode 1 } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } } else { - Write-PipelineTaskError "Unexpected value of -msbuildEngine: '$msbuildEngine'." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 } @@ -445,12 +383,12 @@ function GetDefaultMSBuildEngine() { if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { return "vs" } - + if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { return "dotnet" } - Write-PipelineTaskError "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." ExitWithExitCode 1 } @@ -475,11 +413,13 @@ function GetSdkTaskProject([string]$taskName) { function InitializeNativeTools() { if (Get-Member -InputObject $GlobalJson -Name "native-tools") { - $nativeArgs="" + $nativeArgs= @{} if ($ci) { - $nativeArgs = "-InstallDirectory $ToolsDir" + $nativeArgs = @{ + InstallDirectory = "$ToolsDir" + } } - Invoke-Expression "& `"$PSScriptRoot/init-tools-native.ps1`" $nativeArgs" + & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs } } @@ -501,7 +441,7 @@ function InitializeToolset() { } if (-not $restore) { - Write-PipelineTaskError "Toolset version $toolsetVersion has not been restored." + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Toolset version $toolsetVersion has not been restored." ExitWithExitCode 1 } @@ -561,11 +501,13 @@ function MSBuild() { function MSBuild-Core() { if ($ci) { if (!$binaryLog) { - throw "Binary log must be enabled in CI build." + Write-PipelineTaskError -Message "Binary log must be enabled in CI build." + ExitWithExitCode 1 } if ($nodeReuse) { - throw "Node reuse must be disabled in CI build." + Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." + ExitWithExitCode 1 } } @@ -573,8 +515,8 @@ function MSBuild-Core() { $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" - if ($warnAsError) { - $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" + if ($warnAsError) { + $cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" } foreach ($arg in $args) { @@ -582,29 +524,29 @@ function MSBuild-Core() { $cmdArgs += " `"$arg`"" } } - + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-PipelineTaskError "Build failed." + Write-PipelineTaskError -Message "Build failed." $buildLog = GetMSBuildBinaryLogCommandLineArgument $args - if ($buildLog -ne $null) { - Write-Host "See log: $buildLog" -ForegroundColor DarkGray + if ($buildLog -ne $null) { + Write-Host "See log: $buildLog" -ForegroundColor DarkGray } ExitWithExitCode $exitCode } } -function GetMSBuildBinaryLogCommandLineArgument($arguments) { +function GetMSBuildBinaryLogCommandLineArgument($arguments) { foreach ($argument in $arguments) { if ($argument -ne $null) { $arg = $argument.Trim() if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { return $arg.Substring("/bl:".Length) - } - + } + if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { return $arg.Substring("/binaryLogger:".Length) } @@ -614,7 +556,7 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } -. $PSScriptRoot\LoggingCommandFunctions.ps1 +. $PSScriptRoot\pipeline-logging-functions.ps1 $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") diff --git a/eng/common/tools.sh b/eng/common/tools.sh index fd26f6fbb275..70d92cf85aa2 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -52,78 +52,6 @@ else use_global_nuget_cache=${use_global_nuget_cache:-true} fi -function EmitError { - if [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - - message_type="error" - sourcepath='' - linenumber='' - columnnumber='' - error_code='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" - case "$opt" in - -type|-t) - message_type=$2 - shift - ;; - -sourcepath|-s) - sourcepath=$2 - shift - ;; - -linenumber|-l) - linenumber=$2 - shift - ;; - -columnnumber|-col) - columnnumber=$2 - shift - ;; - -code|-c) - error_code=$2 - shift - ;; - *) - break - ;; - esac - - shift - done - - message='##vso[task.logissue' - - message="$message type=$message_type" - - if [ -n "$sourcepath" ]; then - message="$message;sourcepath=$sourcepath" - else - message="$message;sourcepath=${BASH_SOURCE[1]}" - fi - - if [ -n "$linenumber" ]; then - message="$message;linenumber=$linenumber" - else - message="$message;linenumber=${BASH_LINENO[0]}" - fi - - if [ -n "$columnnumber" ]; then - message="$message;columnnumber=$columnnumber" - fi - - if [ -n "$error_code" ]; then - message="$message;code=$error_code" - fi - - message="$message]$*" - - echo "$message" -} - # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -149,7 +77,7 @@ function ReadGlobalVersion { local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then - EmitError "Error: Cannot find \"$key\" in $global_json_file" + Write-PipelineTelemetryError -category 'InitializeTools' "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi @@ -210,7 +138,7 @@ function InitializeDotNetCli { if [[ "$install" == true ]]; then InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version" else - EmitError "Unable to find dotnet with SDK version '$dotnet_sdk_version'" + Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'" ExitWithExitCode 1 fi fi @@ -221,7 +149,7 @@ function InitializeDotNetCli { export PATH="$dotnet_root:$PATH" if [[ $ci == true ]]; then - # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build + # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build echo "##vso[task.prependpath]$dotnet_root" echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0" echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1" @@ -263,7 +191,7 @@ function InstallDotNet { fi bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? - EmitError "Failed to install dotnet SDK (exit code '$exit_code')." + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')." ExitWithExitCode $exit_code } } @@ -349,7 +277,7 @@ function InitializeToolset { fi if [[ "$restore" != true ]]; then - EmitError "Toolset version $toolsetVersion has not been restored." + Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored." ExitWithExitCode 2 fi @@ -366,7 +294,7 @@ function InitializeToolset { local toolset_build_proj=`cat "$toolset_location_file"` if [[ ! -a "$toolset_build_proj" ]]; then - EmitError "Invalid toolset path: $toolset_build_proj" + Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" ExitWithExitCode 3 fi @@ -404,12 +332,12 @@ function MSBuild { function MSBuild-Core { if [[ "$ci" == true ]]; then if [[ "$binary_log" != true ]]; then - EmitError "Binary log must be enabled in CI build." + Write-PipelineTaskError "Binary log must be enabled in CI build." ExitWithExitCode 1 fi if [[ "$node_reuse" == true ]]; then - EmitError "Node reuse must be disabled in CI build." + Write-PipelineTaskError "Node reuse must be disabled in CI build." ExitWithExitCode 1 fi fi @@ -423,7 +351,7 @@ function MSBuild-Core { "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { local exit_code=$? - EmitError "Build failed (exit code '$exit_code')." + Write-PipelineTaskError "Build failed (exit code '$exit_code')." ExitWithExitCode $exit_code } } @@ -431,6 +359,8 @@ function MSBuild-Core { ResolvePath "${BASH_SOURCE[0]}" _script_dir=`dirname "$_ResolvePath"` +. "$_script_dir/pipeline-logging-functions.sh" + eng_root=`cd -P "$_script_dir/.." && pwd` repo_root=`cd -P "$_script_dir/../.." && pwd` artifacts_dir="$repo_root/artifacts" @@ -460,4 +390,4 @@ mkdir -p "$log_dir" if [[ $ci == true ]]; then export TEMP="$temp_dir" export TMP="$temp_dir" -fi \ No newline at end of file +fi diff --git a/eng/configure-toolset.ps1 b/eng/configure-toolset.ps1 new file mode 100644 index 000000000000..eda160dff30b --- /dev/null +++ b/eng/configure-toolset.ps1 @@ -0,0 +1,5 @@ +# We can't use already installed dotnet cli since we need to install additional shared runtimes. +# We could potentially try to find an existing installation that has all the required runtimes, +# but it's unlikely one will be available. + +$script:useInstalledDotNetCli = $false diff --git a/eng/configure-toolset.sh b/eng/configure-toolset.sh new file mode 100644 index 000000000000..ea92acd6a37b --- /dev/null +++ b/eng/configure-toolset.sh @@ -0,0 +1,7 @@ +# We can't use already installed dotnet cli since we need to install additional shared runtimes. +# We could potentially try to find an existing installation that has all the required runtimes, +# but it's unlikely one will be available. + +if [ "${DotNetBuildFromSource:-false}" = false ]; then + use_installed_dotnet_cli="false" +fi diff --git a/build/docker/alpine.Dockerfile b/eng/docker/alpine.Dockerfile similarity index 100% rename from build/docker/alpine.Dockerfile rename to eng/docker/alpine.Dockerfile diff --git a/build/docker/bionic.Dockerfile b/eng/docker/bionic.Dockerfile similarity index 100% rename from build/docker/bionic.Dockerfile rename to eng/docker/bionic.Dockerfile diff --git a/build/docker/rhel.Dockerfile b/eng/docker/rhel.Dockerfile similarity index 100% rename from build/docker/rhel.Dockerfile rename to eng/docker/rhel.Dockerfile diff --git a/build/docker/ubuntu-alpine37.Dockerfile b/eng/docker/ubuntu-alpine37.Dockerfile similarity index 100% rename from build/docker/ubuntu-alpine37.Dockerfile rename to eng/docker/ubuntu-alpine37.Dockerfile diff --git a/eng/helix/helix.proj b/eng/helix/helix.proj index 7304dd629a67..cc0013e68c31 100644 --- a/eng/helix/helix.proj +++ b/eng/helix/helix.proj @@ -1,7 +1,15 @@ - + - - + + + + + + + @@ -16,7 +24,7 @@ 2 - + ci aspnetcore $(BUILD_BUILDNUMBER).$(SYSTEM_JOBATTEMPT) @@ -25,11 +33,11 @@ true - + dev $(USERNAME) $(USER) - $([System.DateTime]::Now.ToString('yyyyMMdd HH:mm')) + $([System.DateTime]::Now.ToString('yyyyMMddHHmm')) diff --git a/eng/scripts/AddAllProjectRefsToSolution.ps1 b/eng/scripts/AddAllProjectRefsToSolution.ps1 deleted file mode 100644 index 274c098fbabe..000000000000 --- a/eng/scripts/AddAllProjectRefsToSolution.ps1 +++ /dev/null @@ -1,60 +0,0 @@ -<# -.SYNOPSIS -This adds the complete closure of project references to a .sln file - -.EXAMPLE -Let's say you have a folder of projects in src/Banana/, and a file src/Banana/Banana.sln. -To traverse the ProjectReference graph to add all dependency projects, run this script: - - ./eng/scripts/AddAllProjectRefsToSolution.ps1 -WorkingDir ./src/Banana/ - -.EXAMPLE -If src/Banana/ has multiple .sln files, use the -sln parameter. - - ./eng/scripts/AddAllProjectRefsToSolution.ps1 -WorkingDir ./src/Banana/ -SolutionFile src/Banana/Solution1.sln -#> -[CmdletBinding(PositionalBinding = $false)] -param( - [string]$WorkingDir, - [Alias('sln')] - [string]$SolutionFile -) - -$ErrorActionPreference = 'Stop' -$repoRoot = Resolve-Path "$PSScriptRoot/../../" -$listFile = New-TemporaryFile - -if (-not $WorkingDir) { - $WorkingDir = Get-Location -} - -Push-Location $WorkingDir -try { - if (-not $SolutionFile) { - - $slnCount = Get-ChildItem *.sln | Measure - - if ($slnCount.count -eq 0) { - Write-Error "Could not find a solution in this directory. Specify one with -sln " - exit 1 - } - if ($slnCount.count -gt 1) { - Write-Error "Multiple solutions found in this directory. Specify which one to modify with -sln " - exit 1 - } - $SolutionFile = Get-ChildItem *.sln | select -first 1 - } - - & "$repoRoot\build.ps1" -projects "$(Get-Location)\**\*.*proj" /t:ShowProjectClosure "/p:ProjectsReferencedOutFile=$listFile" - - foreach ($proj in (Get-Content $listFile)) { - & dotnet sln $SolutionFile add $proj - if ($lastexitcode -ne 0) { - Write-Warning "Failed to add $proj to $SolutionFile" - } - } -} -finally { - Pop-Location - rm $listFile -ea ignore -} diff --git a/eng/scripts/CodeCheck.ps1 b/eng/scripts/CodeCheck.ps1 index a6c50e73d9de..12029d6cdad4 100644 --- a/eng/scripts/CodeCheck.ps1 +++ b/eng/scripts/CodeCheck.ps1 @@ -43,9 +43,11 @@ function LogError { try { if ($ci) { # Install dotnet.exe - & $repoRoot/build.ps1 -ci -norestore /t:InstallDotNet + & $repoRoot/restore.cmd -ci -NoBuildNodeJS } + . "$repoRoot/activate.ps1" + # # Duplicate .csproj files can cause issues with a shared build output folder # @@ -55,7 +57,7 @@ try { # Ignore duplicates in submodules. These should be isolated from the rest of the build. # Ignore duplicates in the .ref folder. This is expected. Get-ChildItem -Recurse "$repoRoot/src/*.*proj" ` - | ? { $_.FullName -notmatch 'submodules' } ` + | ? { $_.FullName -notmatch 'submodules' -and $_.FullName -notmatch 'node_modules' } ` | ? { (Split-Path -Leaf (Split-Path -Parent $_)) -ne 'ref' } ` | % { $fileName = [io.path]::GetFileNameWithoutExtension($_) @@ -122,10 +124,6 @@ try { -filepath "$repoRoot\eng\Versions.props" } - # - # Solutions - # - Write-Host "Checking that solutions are up to date" Get-ChildItem "$repoRoot/*.sln" -Recurse ` @@ -164,17 +162,13 @@ try { } Write-Host "Re-generating package baselines" - $dotnet = 'dotnet' - if ($ci) { - $dotnet = "$repoRoot/.dotnet/dotnet.exe" - } Invoke-Block { - & $dotnet run -p "$repoRoot/eng/tools/BaselineGenerator/" + & dotnet run -p "$repoRoot/eng/tools/BaselineGenerator/" } Write-Host "Re-generating Browser.JS files" Invoke-Block { - & $dotnet build "$repoRoot\src\Components\Browser.JS\Microsoft.AspNetCore.Components.Browser.JS.npmproj" + & dotnet build "$repoRoot\src\Components\Browser.JS\Microsoft.AspNetCore.Components.Browser.JS.npmproj" } Write-Host "Run git diff to check for pending changes" diff --git a/eng/scripts/GenerateProjectList.ps1 b/eng/scripts/GenerateProjectList.ps1 index adee357de409..47449b3d7ae8 100644 --- a/eng/scripts/GenerateProjectList.ps1 +++ b/eng/scripts/GenerateProjectList.ps1 @@ -5,4 +5,6 @@ $ErrorActionPreference = 'stop' $repoRoot = Resolve-Path "$PSScriptRoot/../.." -& "$repoRoot\build.ps1" -ci:$ci -NoRestore -all /t:GenerateProjectList +& "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" ` + /t:GenerateProjectList ` + /bl:artifacts/log/genprojlist.binlog diff --git a/eng/scripts/GenerateReferenceAssemblies.ps1 b/eng/scripts/GenerateReferenceAssemblies.ps1 index 78cb0243457d..fa58025c34ed 100644 --- a/eng/scripts/GenerateReferenceAssemblies.ps1 +++ b/eng/scripts/GenerateReferenceAssemblies.ps1 @@ -5,4 +5,6 @@ $ErrorActionPreference = 'stop' $repoRoot = Resolve-Path "$PSScriptRoot/../.." -& "$repoRoot\build.ps1" -ci:$ci -BuildManaged -NoBuildNodeJS /t:GenerateReferenceSources +& "$repoRoot\eng\common\msbuild.ps1" -ci:$ci "$repoRoot/eng/CodeGen.proj" ` + /t:GenerateReferenceSources ` + /bl:artifacts/log/genrefassemblies.binlog diff --git a/eng/scripts/KillProcesses.ps1 b/eng/scripts/KillProcesses.ps1 index 48681071cf47..153234abd9bd 100644 --- a/eng/scripts/KillProcesses.ps1 +++ b/eng/scripts/KillProcesses.ps1 @@ -45,14 +45,18 @@ _kill iisexpresstray.exe _kill w3wp.exe _kill msbuild.exe _kill vbcscompiler.exe -_kill git.exe _kill vctip.exe -_kill chrome.exe _kill h2spec.exe _kill WerFault.exe _killJavaInstances _killSeleniumTrackedProcesses +# Special case these. When testing with -ci locally, you typically don't actually want to kill your browser or git command line +if ($env:TF_BUILD) { + _kill chrome.exe + _kill git.exe +} + if (Get-Command iisreset -ErrorAction ignore) { iisreset /restart } diff --git a/eng/scripts/cibuild.cmd b/eng/scripts/cibuild.cmd deleted file mode 100644 index de8de8001410..000000000000 --- a/eng/scripts/cibuild.cmd +++ /dev/null @@ -1,6 +0,0 @@ -@ECHO OFF -SET RepoRoot=%~dp0..\.. -%RepoRoot%\build.cmd -ci -all -pack -sign %* -SET exit_code=%ERRORLEVEL% -ECHO build.cmd completed -EXIT /b %exit_code% diff --git a/eng/scripts/install-nginx-linux.sh b/eng/scripts/install-nginx-linux.sh index a245308a0c0d..b9ad6daf93fa 100755 --- a/eng/scripts/install-nginx-linux.sh +++ b/eng/scripts/install-nginx-linux.sh @@ -4,3 +4,4 @@ curl -sSL http://nginx.org/download/nginx-1.14.2.tar.gz | tar zxfv - -C /tmp && ./configure --prefix=$HOME/nginxinstall --with-http_ssl_module make make install +echo "##vso[task.prependpath]$HOME/nginxinstall/sbin" \ No newline at end of file diff --git a/eng/signcheck.exclusions.txt b/eng/signcheck.exclusions.txt deleted file mode 100644 index 93f17f63f593..000000000000 --- a/eng/signcheck.exclusions.txt +++ /dev/null @@ -1,8 +0,0 @@ -*/AppHostTemplate/apphost.exe;AspNetCoreRuntime.*.nupkg; Exclude the apphost because this is expected to be code-signed by customers after the SDK modifies it. -*/runtime.*.microsoft.netcore.dotnetapphost/*/apphost.exe;Microsoft.AspNetCore.AzureAppServices.SiteExtension.*.nupkg; Exclude the apphost because this is expected to be code-signed by customers after the SDK modifies it. -*.js;; Exclude all JavaScript files from codesigning because we don't expect these to run on Windows Script Host -*.binlog;; Exclude msbuild log files -*.symbols.nupkg;; Exclude NuGet symbols packages. These are not shipped to customers and should not be code signed. -;*.symbols.nupkg; Exclude everything inside NuGet symbols packages. These are not shipped to customers and should not be code signed. -;runtime.osx-x64.Microsoft.AspNetCore.App.*.nupkg; Exclude the contents of the MacOS runtime package because MacOS only supports codesigning for MacOS native binaries and apps -;runtime.linux-*.Microsoft.AspNetCore.App.*.nupkg; Exclude the contents of the Linux runtime packages because Linux doesn't support validating authenticode signatures diff --git a/eng/targets/Helix.targets b/eng/targets/Helix.targets index 1f671c0bd6f2..9322dab23918 100644 --- a/eng/targets/Helix.targets +++ b/eng/targets/Helix.targets @@ -13,12 +13,12 @@ This target is meant to be used when invoking helix tests on one project at a time. -Usage: dotnet build /t:Helix src/MyTestProject.csproj +Usage: dotnet msbuild /t:Helix src/MyTestProject.csproj --> + Properties="ProjectToBuild=$(MSBuildProjectFullPath)" /> diff --git a/eng/targets/Npm.Common.props b/eng/targets/Npm.Common.props index 3308ca9ee034..6f7de5c6523a 100644 --- a/eng/targets/Npm.Common.props +++ b/eng/targets/Npm.Common.props @@ -1,7 +1,7 @@ test - Release + Release Debug diff --git a/eng/targets/ReferenceAssembly.targets b/eng/targets/ReferenceAssembly.targets index 605c715eb771..dccc945227b8 100644 --- a/eng/targets/ReferenceAssembly.targets +++ b/eng/targets/ReferenceAssembly.targets @@ -40,6 +40,7 @@ + @@ -65,6 +66,7 @@ + diff --git a/eng/targets/ResolveReferences.targets b/eng/targets/ResolveReferences.targets index 848c95c8b2ec..684660fed175 100644 --- a/eng/targets/ResolveReferences.targets +++ b/eng/targets/ResolveReferences.targets @@ -103,10 +103,6 @@ - - - <_ReferenceTemp Include="@(Reference)" /> - - - <_ReferenceTemp Remove="@(_ReferenceTemp)" /> - ContentFiles;Build diff --git a/eng/tools/Directory.Build.props b/eng/tools/Directory.Build.props index f75adf7e4d62..771294cf6598 100644 --- a/eng/tools/Directory.Build.props +++ b/eng/tools/Directory.Build.props @@ -1,2 +1,9 @@  + + + + false + false + true + diff --git a/eng/tools/Directory.Build.targets b/eng/tools/Directory.Build.targets deleted file mode 100644 index f75adf7e4d62..000000000000 --- a/eng/tools/Directory.Build.targets +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/build/Maestro/Maestro.csproj b/eng/tools/Maestro/Maestro.csproj similarity index 100% rename from build/Maestro/Maestro.csproj rename to eng/tools/Maestro/Maestro.csproj diff --git a/eng/tools/RepoTasks/CreateFrameworkListFile.cs b/eng/tools/RepoTasks/CreateFrameworkListFile.cs new file mode 100644 index 000000000000..b0ab8c690a17 --- /dev/null +++ b/eng/tools/RepoTasks/CreateFrameworkListFile.cs @@ -0,0 +1,107 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace RepoTasks +{ + public class CreateFrameworkListFile : Task + { + /// + /// Files to extract basic information from and include in the list. + /// + [Required] + public ITaskItem[] Files { get; set; } + + [Required] + public string TargetFile { get; set; } + + /// + /// Extra attributes to place on the root node. + /// + /// %(Identity): Attribute name. + /// %(Value): Attribute value. + /// + public ITaskItem[] RootAttributes { get; set; } + + public override bool Execute() + { + XAttribute[] rootAttributes = RootAttributes + ?.Select(item => new XAttribute(item.ItemSpec, item.GetMetadata("Value"))) + .ToArray(); + + var frameworkManifest = new XElement("FileList", rootAttributes); + + var usedFileProfiles = new HashSet(); + + foreach (var f in Files + .Select(item => new + { + Item = item, + Filename = Path.GetFileName(item.ItemSpec), + TargetPath = item.GetMetadata("TargetPath"), + AssemblyName = FileUtilities.GetAssemblyName(item.ItemSpec), + FileVersion = FileUtilities.GetFileVersion(item.ItemSpec), + IsNative = item.GetMetadata("IsNativeImage") == "true", + IsSymbolFile = item.GetMetadata("IsSymbolFile") == "true" + }) + .Where(f => + !f.IsSymbolFile && + (f.Filename.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) || f.IsNative)) + .OrderBy(f => f.TargetPath, StringComparer.Ordinal) + .ThenBy(f => f.Filename, StringComparer.Ordinal)) + { + var element = new XElement( + "File", + new XAttribute("Type", f.IsNative ? "Native" : "Managed"), + new XAttribute( + "Path", + Path.Combine(f.TargetPath, f.Filename).Replace('\\', '/'))); + + if (f.AssemblyName != null) + { + byte[] publicKeyToken = f.AssemblyName.GetPublicKeyToken(); + string publicKeyTokenHex; + + if (publicKeyToken != null) + { + publicKeyTokenHex = BitConverter.ToString(publicKeyToken) + .ToLowerInvariant() + .Replace("-", ""); + } + else + { + Log.LogError($"No public key token found for assembly {f.Item.ItemSpec}"); + publicKeyTokenHex = ""; + } + + element.Add( + new XAttribute("AssemblyName", f.AssemblyName.Name), + new XAttribute("PublicKeyToken", publicKeyTokenHex), + new XAttribute("AssemblyVersion", f.AssemblyName.Version)); + } + else if (!f.IsNative) + { + // This file isn't managed and isn't native. Leave it off the list. + continue; + } + + element.Add(new XAttribute("FileVersion", f.FileVersion)); + + frameworkManifest.Add(element); + } + + Directory.CreateDirectory(Path.GetDirectoryName(TargetFile)); + File.WriteAllText(TargetFile, frameworkManifest.ToString()); + + return !Log.HasLoggedErrors; + } + } +} diff --git a/eng/tools/RepoTasks/FileUtilities.cs b/eng/tools/RepoTasks/FileUtilities.cs new file mode 100644 index 000000000000..b384cbe5e2c9 --- /dev/null +++ b/eng/tools/RepoTasks/FileUtilities.cs @@ -0,0 +1,48 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Reflection; + +namespace RepoTasks +{ + internal static partial class FileUtilities + { + private static readonly HashSet s_assemblyExtensions = new HashSet( + new[] { ".dll", ".exe", ".winmd" }, + StringComparer.OrdinalIgnoreCase); + + public static Version GetFileVersion(string sourcePath) + { + var fvi = FileVersionInfo.GetVersionInfo(sourcePath); + + if (fvi != null) + { + return new Version(fvi.FileMajorPart, fvi.FileMinorPart, fvi.FileBuildPart, fvi.FilePrivatePart); + } + + return null; + } + + public static AssemblyName GetAssemblyName(string path) + { + if (!s_assemblyExtensions.Contains(Path.GetExtension(path))) + { + return null; + } + + try + { + return AssemblyName.GetAssemblyName(path); + } + catch (BadImageFormatException) + { + // Not a valid assembly. + return null; + } + } + } +} \ No newline at end of file diff --git a/build/tasks/GenerateGuid.cs b/eng/tools/RepoTasks/GenerateGuid.cs similarity index 100% rename from build/tasks/GenerateGuid.cs rename to eng/tools/RepoTasks/GenerateGuid.cs diff --git a/build/tasks/GenerateSharedFrameworkDepsFile.cs b/eng/tools/RepoTasks/GenerateSharedFrameworkDepsFile.cs similarity index 97% rename from build/tasks/GenerateSharedFrameworkDepsFile.cs rename to eng/tools/RepoTasks/GenerateSharedFrameworkDepsFile.cs index a4022e749193..172f002d18fe 100644 --- a/build/tasks/GenerateSharedFrameworkDepsFile.cs +++ b/eng/tools/RepoTasks/GenerateSharedFrameworkDepsFile.cs @@ -10,7 +10,6 @@ using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -using Microsoft.DotNet.Build.Tasks; using Microsoft.Extensions.DependencyModel; namespace RepoTasks @@ -61,7 +60,7 @@ private void ExecuteCore() var filePath = reference.ItemSpec; var fileName = Path.GetFileName(filePath); var fileVersion = FileUtilities.GetFileVersion(filePath)?.ToString() ?? string.Empty; - var assemblyVersion = FileUtilities.TryGetAssemblyVersion(filePath); + var assemblyVersion = FileUtilities.GetAssemblyName(filePath)?.Version; if (assemblyVersion == null) { var nativeFile = new RuntimeFile(fileName, null, fileVersion); diff --git a/build/tasks/GetMsiProperty.cs b/eng/tools/RepoTasks/GetMsiProperty.cs similarity index 100% rename from build/tasks/GetMsiProperty.cs rename to eng/tools/RepoTasks/GetMsiProperty.cs diff --git a/build/tasks/RemoveSharedFrameworkDependencies.cs b/eng/tools/RepoTasks/RemoveSharedFrameworkDependencies.cs similarity index 97% rename from build/tasks/RemoveSharedFrameworkDependencies.cs rename to eng/tools/RepoTasks/RemoveSharedFrameworkDependencies.cs index 3066bb5a8936..1471c657dc9a 100644 --- a/build/tasks/RemoveSharedFrameworkDependencies.cs +++ b/eng/tools/RepoTasks/RemoveSharedFrameworkDependencies.cs @@ -98,6 +98,7 @@ private void FilterDependencies(string targetPath, ISet dependencyToRemo stream.Position = 0; stream.SetLength(0); rawNuspec.Save(stream); + Log.LogMessage(MessageImportance.High, "Added to {0}", fileName); } else { diff --git a/eng/tools/RepoTasks/RepoTasks.csproj b/eng/tools/RepoTasks/RepoTasks.csproj new file mode 100644 index 000000000000..f62865e20323 --- /dev/null +++ b/eng/tools/RepoTasks/RepoTasks.csproj @@ -0,0 +1,37 @@ + + + + netcoreapp3.0 + $(TargetFrameworks);net472 + $(DefineConstants);BUILD_MSI_TASKS + false + embedded + true + + + + + + + + + + + + + + + + + + + + + + $(WiXSdkPath)\Microsoft.Deployment.WindowsInstaller.dll + + + $(WiXSdkPath)\Microsoft.Deployment.WindowsInstaller.Package.dll + + + diff --git a/eng/tools/RepoTasks/RepoTasks.tasks b/eng/tools/RepoTasks/RepoTasks.tasks new file mode 100644 index 000000000000..b55f394e4c43 --- /dev/null +++ b/eng/tools/RepoTasks/RepoTasks.tasks @@ -0,0 +1,13 @@ + + + <_RepoTaskAssemblyFolder Condition="'$(MSBuildRuntimeType)' == 'core'">netcoreapp3.0 + <_RepoTaskAssemblyFolder Condition="'$(MSBuildRuntimeType)' != 'core'">net472 + <_RepoTaskAssembly>$(ArtifactsBinDir)RepoTasks\Release\$(_RepoTaskAssemblyFolder)\RepoTasks.dll + + + + + + + + diff --git a/build/tasks/Uuid.cs b/eng/tools/RepoTasks/Uuid.cs similarity index 100% rename from build/tasks/Uuid.cs rename to eng/tools/RepoTasks/Uuid.cs diff --git a/eng/tools/tools.sln b/eng/tools/tools.sln index 51e337b7e318..d6f192e38ea8 100644 --- a/eng/tools/tools.sln +++ b/eng/tools/tools.sln @@ -5,6 +5,8 @@ VisualStudioVersion = 16.0.0.0 MinimumVisualStudioVersion = 16.0.0.0 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaselineGenerator", "BaselineGenerator\BaselineGenerator.csproj", "{CF76A947-3A72-4824-87E6-BF029D84218B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RepoTasks", "RepoTasks\RepoTasks.csproj", "{E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -30,5 +32,17 @@ Global {CF76A947-3A72-4824-87E6-BF029D84218B}.Release|x64.Build.0 = Release|Any CPU {CF76A947-3A72-4824-87E6-BF029D84218B}.Release|x86.ActiveCfg = Release|Any CPU {CF76A947-3A72-4824-87E6-BF029D84218B}.Release|x86.Build.0 = Release|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Debug|x64.ActiveCfg = Debug|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Debug|x64.Build.0 = Debug|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Debug|x86.ActiveCfg = Debug|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Debug|x86.Build.0 = Debug|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Release|Any CPU.Build.0 = Release|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Release|x64.ActiveCfg = Release|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Release|x64.Build.0 = Release|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Release|x86.ActiveCfg = Release|Any CPU + {E418DA57-B2AA-43FC-9AD5-0E4D1AA993DE}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/global.json b/global.json index 97f7c9313f72..6828849fa253 100644 --- a/global.json +++ b/global.json @@ -4,11 +4,27 @@ }, "tools": { "dotnet": "3.0.100-preview5-011568", - "jdk": "11.0.3" + "runtimes": { + "dotnet/x64": [ + "$(MicrosoftNETCoreAppRuntimeVersion)" + ], + "dotnet/x86": [ + "$(MicrosoftNETCoreAppRuntimeVersion)" + ] + }, + "jdk": "11.0.3", + "vs": { + "version": "16.0", + "components": [ + "Microsoft.VisualStudio.Component.VC.ATL", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "Microsoft.VisualStudio.Component.Windows10SDK.17134" + ] + } }, "msbuild-sdks": { "Yarn.MSBuild": "1.15.2", - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19302.2", - "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19302.2" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19323.4", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19323.4" } } diff --git a/korebuild-lock.txt b/korebuild-lock.txt deleted file mode 100644 index cb6baecc02ad..000000000000 --- a/korebuild-lock.txt +++ /dev/null @@ -1,2 +0,0 @@ -version:3.0.0-build-20190606.1 -commithash:2229c5dff7ca6384e7c17148a6da1aaee161f44d diff --git a/korebuild.json b/korebuild.json deleted file mode 100644 index e189d4b1de04..000000000000 --- a/korebuild.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/aspnet/BuildTools/master/tools/korebuild.schema.json", - "channel": "master", - "msbuildType": "full", - "toolsets": { - "nodejs": { - "minVersion": "10.0", - "required": true - }, - "visualstudio": { - "required": [ - "Windows" - ], - "includePrerelease": true, - "versionRange": "[16.0, 17.0)", - "requiredWorkloads": [ - "Microsoft.VisualStudio.Component.VC.ATL", - "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", - "Microsoft.VisualStudio.Component.Windows10SDK.17134" - ] - } - } -} diff --git a/src/Analyzers/shared/FeatureDetection/Microsoft.AspNetCore.Analyzers.FeatureDetection.Sources.csproj b/src/Analyzers/shared/FeatureDetection/Microsoft.AspNetCore.Analyzers.FeatureDetection.Sources.csproj index 53f07e744a09..40e7bf1d97de 100644 --- a/src/Analyzers/shared/FeatureDetection/Microsoft.AspNetCore.Analyzers.FeatureDetection.Sources.csproj +++ b/src/Analyzers/shared/FeatureDetection/Microsoft.AspNetCore.Analyzers.FeatureDetection.Sources.csproj @@ -17,7 +17,7 @@ contentFiles true $(DefaultExcludeItems);$(BaseOutputPath);$(BaseIntermediateOutputPath); - $(NoWarn);NU5105;CS8021 + $(NoWarn);CS8021 false diff --git a/src/Antiforgery/test/DefaultAntiforgeryTokenStoreTest.cs b/src/Antiforgery/test/DefaultAntiforgeryTokenStoreTest.cs index 1ca1f57fc5f4..494b91c54009 100644 --- a/src/Antiforgery/test/DefaultAntiforgeryTokenStoreTest.cs +++ b/src/Antiforgery/test/DefaultAntiforgeryTokenStoreTest.cs @@ -5,8 +5,8 @@ using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.Internal; using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; using Moq; using Xunit; @@ -20,7 +20,7 @@ public class DefaultAntiforgeryTokenStoreTest public void GetCookieToken_CookieDoesNotExist_ReturnsNull() { // Arrange - var httpContext = GetHttpContext(new RequestCookieCollection()); + var httpContext = GetHttpContext(); var options = new AntiforgeryOptions { Cookie = { Name = _cookieName } @@ -79,7 +79,7 @@ public void GetCookieToken_CookieIsNotEmpty_ReturnsToken() public async Task GetRequestTokens_CookieIsEmpty_ReturnsNullTokens() { // Arrange - var httpContext = GetHttpContext(new RequestCookieCollection()); + var httpContext = GetHttpContext(); httpContext.Request.Form = FormCollection.Empty; var options = new AntiforgeryOptions @@ -407,18 +407,14 @@ public void SaveCookieToken_NonNullAntiforgeryOptionsConfigureCookieOptionsDomai private HttpContext GetHttpContext(string cookieName, string cookieValue) { - var cookies = new RequestCookieCollection(new Dictionary - { - { cookieName, cookieValue }, - }); - - return GetHttpContext(cookies); + var context = GetHttpContext(); + context.Request.Headers[HeaderNames.Cookie] = $"{cookieName}={cookieValue}"; + return context; } - private HttpContext GetHttpContext(IRequestCookieCollection cookies) + private HttpContext GetHttpContext() { var httpContext = new DefaultHttpContext(); - httpContext.Request.Cookies = cookies; return httpContext; } diff --git a/src/Components/Blazor/Blazor/ref/Microsoft.AspNetCore.Blazor.netstandard2.0.cs b/src/Components/Blazor/Blazor/ref/Microsoft.AspNetCore.Blazor.netstandard2.0.cs index ab316c370145..e7b60daa246c 100644 --- a/src/Components/Blazor/Blazor/ref/Microsoft.AspNetCore.Blazor.netstandard2.0.cs +++ b/src/Components/Blazor/Blazor/ref/Microsoft.AspNetCore.Blazor.netstandard2.0.cs @@ -61,7 +61,7 @@ public partial class WebAssemblyRenderer : Microsoft.AspNetCore.Components.Rende public WebAssemblyRenderer(System.IServiceProvider serviceProvider) : base (default(System.IServiceProvider)) { } public System.Threading.Tasks.Task AddComponentAsync(System.Type componentType, string domElementSelector) { throw null; } public System.Threading.Tasks.Task AddComponentAsync(string domElementSelector) where TComponent : Microsoft.AspNetCore.Components.IComponent { throw null; } - public override System.Threading.Tasks.Task DispatchEventAsync(int eventHandlerId, Microsoft.AspNetCore.Components.UIEventArgs eventArgs) { throw null; } + public override System.Threading.Tasks.Task DispatchEventAsync(int eventHandlerId, Microsoft.AspNetCore.Components.Rendering.EventFieldInfo eventFieldInfo, Microsoft.AspNetCore.Components.UIEventArgs eventArgs) { throw null; } protected override void Dispose(bool disposing) { } protected override void HandleException(System.Exception exception) { } protected override System.Threading.Tasks.Task UpdateDisplayAsync(in Microsoft.AspNetCore.Components.Rendering.RenderBatch batch) { throw null; } diff --git a/src/Components/Blazor/Blazor/src/Rendering/WebAssemblyRenderer.cs b/src/Components/Blazor/Blazor/src/Rendering/WebAssemblyRenderer.cs index 0e1e237b6bd2..4098bb5e411c 100644 --- a/src/Components/Blazor/Blazor/src/Rendering/WebAssemblyRenderer.cs +++ b/src/Components/Blazor/Blazor/src/Rendering/WebAssemblyRenderer.cs @@ -116,7 +116,7 @@ protected override void HandleException(Exception exception) } /// - public override Task DispatchEventAsync(int eventHandlerId, UIEventArgs eventArgs) + public override Task DispatchEventAsync(int eventHandlerId, EventFieldInfo eventFieldInfo, UIEventArgs eventArgs) { // Be sure we only run one event handler at once. Although they couldn't run // simultaneously anyway (there's only one thread), they could run nested on @@ -135,7 +135,7 @@ public override Task DispatchEventAsync(int eventHandlerId, UIEventArgs eventArg if (isDispatchingEvent) { - var info = new IncomingEventInfo(eventHandlerId, eventArgs); + var info = new IncomingEventInfo(eventHandlerId, eventFieldInfo, eventArgs); deferredIncomingEvents.Enqueue(info); return info.TaskCompletionSource.Task; } @@ -144,7 +144,7 @@ public override Task DispatchEventAsync(int eventHandlerId, UIEventArgs eventArg try { isDispatchingEvent = true; - return base.DispatchEventAsync(eventHandlerId, eventArgs); + return base.DispatchEventAsync(eventHandlerId, eventFieldInfo, eventArgs); } finally { @@ -168,7 +168,7 @@ private async Task ProcessNextDeferredEventAsync() try { - await DispatchEventAsync(info.EventHandlerId, info.EventArgs); + await DispatchEventAsync(info.EventHandlerId, info.EventFieldInfo, info.EventArgs); taskCompletionSource.SetResult(null); } catch (Exception ex) @@ -180,12 +180,14 @@ private async Task ProcessNextDeferredEventAsync() readonly struct IncomingEventInfo { public readonly int EventHandlerId; + public readonly EventFieldInfo EventFieldInfo; public readonly UIEventArgs EventArgs; public readonly TaskCompletionSource TaskCompletionSource; - public IncomingEventInfo(int eventHandlerId, UIEventArgs eventArgs) + public IncomingEventInfo(int eventHandlerId, EventFieldInfo eventFieldInfo, UIEventArgs eventArgs) { EventHandlerId = eventHandlerId; + EventFieldInfo = eventFieldInfo; EventArgs = eventArgs; TaskCompletionSource = new TaskCompletionSource(); } diff --git a/src/Components/Blazor/BlazorExtension/src/Microsoft.VisualStudio.BlazorExtension.csproj b/src/Components/Blazor/BlazorExtension/src/Microsoft.VisualStudio.BlazorExtension.csproj index 5fb085119857..5639616fed42 100644 --- a/src/Components/Blazor/BlazorExtension/src/Microsoft.VisualStudio.BlazorExtension.csproj +++ b/src/Components/Blazor/BlazorExtension/src/Microsoft.VisualStudio.BlazorExtension.csproj @@ -24,7 +24,7 @@ false true true - false + false true diff --git a/src/Components/Blazor/Build/src/ReferenceFromSource.props b/src/Components/Blazor/Build/src/ReferenceFromSource.props index 2eed23ef3f4a..ca6ee1eb5429 100644 --- a/src/Components/Blazor/Build/src/ReferenceFromSource.props +++ b/src/Components/Blazor/Build/src/ReferenceFromSource.props @@ -28,7 +28,7 @@ dotnet <_BlazorCliLocation>$(MSBuildThisFileDirectory)../../DevServer/src/bin/$(Configuration)/netcoreapp3.0/blazor-devserver.dll - exec "$(_BlazorCliLocation)" serve $(AdditionalRunArguments) + exec "$(_BlazorCliLocation)" serve "$(MSBuildProjectDirectory)/$(OutputPath)$(TargetFileName)" $(AdditionalRunArguments) diff --git a/src/Components/Blazor/Build/src/targets/All.targets b/src/Components/Blazor/Build/src/targets/All.targets index 2f61b5e4dff8..dd4fbf1b7a9a 100644 --- a/src/Components/Blazor/Build/src/targets/All.targets +++ b/src/Components/Blazor/Build/src/targets/All.targets @@ -29,4 +29,18 @@ + + + + $(GetCurrentProjectStaticWebAssetsDependsOn); + _ClearCurrentStaticWebAssetsForReferenceDiscovery + + + + + + + + + diff --git a/src/Components/Blazor/Build/test/ComponentRenderingRazorIntegrationTest.cs b/src/Components/Blazor/Build/test/ComponentRenderingRazorIntegrationTest.cs index 0d69f27e1d79..a754d4249d3e 100644 --- a/src/Components/Blazor/Build/test/ComponentRenderingRazorIntegrationTest.cs +++ b/src/Components/Blazor/Build/test/ComponentRenderingRazorIntegrationTest.cs @@ -445,7 +445,7 @@ public void Regression_784() // Act var component = CompileToComponent(@" -

+

@code { public string ParentBgColor { get; set; } = ""#FFFFFF""; diff --git a/src/Components/Blazor/Build/test/RenderingRazorIntegrationTest.cs b/src/Components/Blazor/Build/test/RenderingRazorIntegrationTest.cs index 815c5f16fe08..af4bffa2022c 100644 --- a/src/Components/Blazor/Build/test/RenderingRazorIntegrationTest.cs +++ b/src/Components/Blazor/Build/test/RenderingRazorIntegrationTest.cs @@ -483,15 +483,14 @@ public void SupportsEventHandlerWithString() { // Arrange var component = CompileToComponent(@" -", 0)); } [Fact] @@ -499,7 +498,7 @@ public void SupportsEventHandlerWithLambda() { // Arrange var component = CompileToComponent(@" -

+

- - + + -

@serializedValue

+

serializedValue

@if (serializedValue != null) { -

Serialized length: @serializedValue.Length chars

+

Serialized length: serializedValue.Length chars

} - - + + @if (numPeopleDeserialized > 0) { diff --git a/src/Components/Blazor/testassets/Microsoft.AspNetCore.Blazor.E2EPerformance/Pages/RenderList.razor b/src/Components/Blazor/testassets/Microsoft.AspNetCore.Blazor.E2EPerformance/Pages/RenderList.razor index 49d3b9898e9f..062d3d69fd15 100644 --- a/src/Components/Blazor/testassets/Microsoft.AspNetCore.Blazor.E2EPerformance/Pages/RenderList.razor +++ b/src/Components/Blazor/testassets/Microsoft.AspNetCore.Blazor.E2EPerformance/Pages/RenderList.razor @@ -3,9 +3,9 @@

Render List

-Number of items: - - +Number of items: + + @if (show) { diff --git a/src/Components/Blazor/testassets/StandaloneApp/Pages/Counter.razor b/src/Components/Blazor/testassets/StandaloneApp/Pages/Counter.razor index ea87f6be2d31..1f063003c84e 100644 --- a/src/Components/Blazor/testassets/StandaloneApp/Pages/Counter.razor +++ b/src/Components/Blazor/testassets/StandaloneApp/Pages/Counter.razor @@ -4,7 +4,7 @@

Current count: @currentCount

- + @code { int currentCount = 0; diff --git a/src/Components/Blazor/testassets/StandaloneApp/Shared/NavMenu.razor b/src/Components/Blazor/testassets/StandaloneApp/Shared/NavMenu.razor index 880f7be3896c..298bdd36f15a 100644 --- a/src/Components/Blazor/testassets/StandaloneApp/Shared/NavMenu.razor +++ b/src/Components/Blazor/testassets/StandaloneApp/Shared/NavMenu.razor @@ -1,11 +1,11 @@ -
+