Put artifacts in well-known locations

1. Shipping and non-shipping artifacts to Arcade default locations: -> artifacts/packages/<config>/Shipping -> artifacts/packages/<config>/NonShipping Add the local feed for the addition package directory to NuGet.config Remove the non-shipping packages list generation now that artifacts are grouped. 2. Genereated the intermediate PackageVersions* files in -> artifacts/obj/PackageVersions 3. Extract the repo symbol archives intermediates to -> artifacts/obj/Symbols/ 4. Sequence targets better and apply correct DependsOnTargets 5. Small clean-up Fixes dotnet/source-build#4104 Fixes dotnet/source-build#3696
dotnet · Feb 9, 2024 · 63d9377 · 63d9377
1 parent 4cd842e
commit 63d9377
Show file tree

Hide file tree

Showing 12 changed files with 159 additions and 212 deletions.
diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml
@@ -138,7 +138,6 @@ jobs:
           vmrPath: $(vmrPath)
           vmrBranch: ${{ parameters.vmrBranch }}
           targetRef: $(Build.SourceVersion) # Synchronize the current installer commit
-          architecture: ${{ parameters.architecture }}
 
   - ${{ if parameters.buildFromArchive }}:
     - script: |
@@ -399,7 +398,7 @@ jobs:
         publishRunAttachments: true
         testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName)
 
-  - publish: '$(sourcesPath)/artifacts/${{ parameters.architecture }}/Release/'
+  - publish: '$(sourcesPath)/artifacts/packages/Release/'
     artifact: $(Agent.JobName)_Artifacts
     displayName: Publish Artifacts
     condition: succeededOrFailed()

diff --git a/eng/pipelines/templates/steps/vmr-pull-updates.yml b/eng/pipelines/templates/steps/vmr-pull-updates.yml
@@ -17,11 +17,6 @@ parameters:
   type: string
   default: $(Agent.BuildDirectory)/vmr
 
-- name: architecture
-  displayName: Architecture used for build artifacts
-  type: string
-  default: 'x64'
-
 steps:
 - checkout: self
   displayName: Clone dotnet/installer
@@ -41,16 +36,6 @@ steps:
   displayName: Prepare branch ${{ parameters.vmrBranch }}
   workingDirectory: ${{ parameters.vmrPath }}
 
-- script: |
-    mkdir -p ${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release
-  displayName: Create artifacts folder (Unix)
-  condition: ne(variables['Agent.OS'], 'Windows_NT')
-
-- powershell: |
-    New-Item -ItemType Directory -Path ${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release
-  displayName: Create artifacts folder (Windows)
-  condition: eq(variables['Agent.OS'], 'Windows_NT')
-
 - script: |
     git config --global user.name "dotnet-maestro[bot]"
     git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com"

diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props
@@ -92,6 +92,9 @@
     <ArtifactsObjDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'obj'))</ArtifactsObjDir>
     <ArtifactsBinDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin'))</ArtifactsBinDir>
     <ArtifactsLogDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'log', '$(Configuration)'))</ArtifactsLogDir>
+    <ArtifactsPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(Configuration)'))</ArtifactsPackagesDir>
+    <ArtifactsNonShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'NonShipping'))</ArtifactsNonShippingPackagesDir>
+    <ArtifactsShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'Shipping'))</ArtifactsShippingPackagesDir>
     <RepositoryEngineeringDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'eng'))</RepositoryEngineeringDir>
 
     <!-- ProjectLayout.props -->
@@ -168,14 +171,7 @@
     <ArcadeBootstrapPackageDir Condition="'$(DotNetBuildSourceOnly)' != 'true'">$(NuGetPackageRoot)</ArcadeBootstrapPackageDir>
 
     <VSMSBuildSdkResolversDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers'))</VSMSBuildSdkResolversDir>
-
-    <!-- Shared output and intermediate output path folders that are architecture and configuration specific. -->
-    <SharedOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedOutputPath>
-    <SharedIntermediateOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedIntermediateOutputPath>
-
-    <SourceBuiltBlobFeedDir>$([MSBuild]::NormalizeDirectory('$(SharedIntermediateOutputPath)', 'blob-feed'))</SourceBuiltBlobFeedDir>
-    <SourceBuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'packages'))</SourceBuiltPackagesPath>
-    <SourceBuiltAssetsDir>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'assets'))</SourceBuiltAssetsDir>
+    <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
 
     <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
     <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>

diff --git a/src/SourceBuild/content/Directory.Build.targets b/src/SourceBuild/content/Directory.Build.targets
@@ -7,7 +7,7 @@
       <SdkFilenamePrefix>dotnet-sdk-</SdkFilenamePrefix>
     </PropertyGroup>
     <ItemGroup>
-      <SdkTarballItem Include="$(SourceBuiltAssetsDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsShippingPackagesDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
     </ItemGroup>
 
     <!--

diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj
@@ -19,21 +19,6 @@
              StopOnFirstFailure="true" />
   </Target>
 
-  <!-- Copies the output assets of the builds to the output path. -->
-  <Target Name="CopyBinariesToBinFolder"
-          AfterTargets="Build">
-    <ItemGroup>
-      <BinaryToCopy Include="$(SourceBuiltAssetsDir)*.*"
-                    Exclude="$(SourceBuiltAssetsDir)*.nupkg;
-                             $(SourceBuiltAssetsDir)*.requires_nupkg_signing" />
-    </ItemGroup>
-
-    <Copy SourceFiles="@(BinaryToCopy)"
-          DestinationFolder="$(SharedOutputPath)"
-          SkipUnchangedFiles="true"
-          Condition="'@(BinaryToCopy)'!=''" />
-  </Target>
-
   <Import Project="$(RepositoryEngineeringDir)build.sourcebuild.targets" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
 
 </Project>
diff --git a/src/SourceBuild/content/eng/build.sourcebuild.targets b/src/SourceBuild/content/eng/build.sourcebuild.targets
@@ -15,45 +15,17 @@
     <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
   </Target>
 
-  <Target Name="DiscoverSymbolsTarballs"
-          AfterTargets="Build">
-    <ItemGroup>
-      <SymbolsTarball Include="$(SharedOutputPath)Symbols.*$(ArchiveExtension)" />
-    </ItemGroup>
-  </Target>
-
-  <Target Name="ExtractSymbolsTarballs"
-          AfterTargets="Build"
-          DependsOnTargets="DiscoverSymbolsTarballs"
-          Outputs="%(SymbolsTarball.Identity)">
-
-    <PropertyGroup>
-      <Filename>$([System.IO.Path]::GetFileName('%(SymbolsTarball.Identity)'))</Filename>
-      <RepositoryName>$(Filename.Split('.')[1])</RepositoryName>
-      <UnifiedSymbolsLayout>$(ArtifactsTmpDir)Symbols</UnifiedSymbolsLayout>
-      <DestinationFolder>$(UnifiedSymbolsLayout)/$(RepositoryName)</DestinationFolder>
-    </PropertyGroup>
-
-    <MakeDir Directories="$(DestinationFolder)" />
-    <Exec Command="tar -xzf %(SymbolsTarball.Identity) -C $(DestinationFolder)"
-          WorkingDirectory="$(SymbolsRoot)" />
-
-    <Delete Files="%(SymbolsTarball.Identity)" />
-  </Target>
-
   <!-- After building, repackage symbols into a single tarball. -->
   <Target Name="RepackageSymbols"
           AfterTargets="Build"
-          DependsOnTargets="
-            DetermineSourceBuiltSdkVersion;
-            DiscoverSymbolsTarballs;
-            ExtractSymbolsTarballs">
+          DependsOnTargets="DetermineSourceBuiltSdkVersion">
     <PropertyGroup>
-      <UnifiedSymbolsTarball>$(SharedOutputPath)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
+      <UnifiedSymbolsTarball>$(ArtifactsShippingPackagesDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
     </PropertyGroup>
 
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(UnifiedSymbolsTarball)'))" />
     <Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
-          WorkingDirectory="$(UnifiedSymbolsLayout)" />
+          WorkingDirectory="$(IntermediateSymbolsRootDir)" />
 
     <Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />
   </Target>
@@ -64,22 +36,22 @@
           AfterTargets="Build"
           DependsOnTargets="RepackageSymbols">
     <ItemGroup>
-      <SdkTarballItem Include="$(SharedOutputPath)dotnet-sdk-*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsShippingPackagesDir)dotnet-sdk-*$(ArchiveExtension)" />
     </ItemGroup>
 
     <PropertyGroup>
-      <SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
-      <SdkSymbolsTarball>$(SharedOutputPath)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
-      <SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
+      <SdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</SdkSymbolsLayout>
+      <SdkSymbolsTarball>$(ArtifactsShippingPackagesDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
+      <SdkLayout>$(BaseIntermediateOutputPath)Sdk</SdkLayout>
       <SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
     </PropertyGroup>
 
     <MakeDir Directories="$(SdkLayout)" />
     <Exec Command="tar -xzf $(SdkTarball) -C $(SdkLayout)"
-          WorkingDirectory="$(SharedOutputPath)" />
+          WorkingDirectory="$(ArtifactsShippingPackagesDir)" />
 
     <CreateSdkSymbolsLayout SdkLayoutPath="$(SdkLayout)"
-                            AllSymbolsPath="$(UnifiedSymbolsLayout)"
+                            AllSymbolsPath="$(IntermediateSymbolsRootDir)"
                             SdkSymbolsLayoutPath="$(SdkSymbolsLayout)"
                             FailOnMissingPDBs="false" />
 
@@ -88,7 +60,7 @@
 
     <Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />
 
-    <RemoveDir Directories="$(UnifiedSymbolsLayout)" />
+    <RemoveDir Directories="$(IntermediateSymbolsRootDir)" />
     <RemoveDir Directories="$(SdkSymbolsLayout)" />
     <RemoveDir Directories="$(SdkLayout)" />
   </Target>
@@ -111,21 +83,16 @@
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)ReportPoisonUsage.complete" >
     <ItemGroup>
-      <FinalCliTarball Include="$(SharedOutputPath)**/*$(ArchiveExtension)" />
+      <FinalCliTarball Include="$(ArtifactsShippingPackagesDir)*$(ArchiveExtension)" />
     </ItemGroup>
 
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(FinalCliTarball) for poisoned files." />
 
-    <ItemGroup>
-      <NonShippingPackagesList Include="$(PackageListsDir)**/$(NonShippingPackagesListPrefix)*" />
-    </ItemGroup>
-
     <CheckForPoison FilesToCheck="@(FinalCliTarball)"
                     ProjectDirPath="$(RepoRoot)"
                     HashCatalogFilePath="$(PoisonReportDataFile)"
                     MarkerFileName="$(PoisonMarkerFile)"
-                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)"
-                    NonShippingPackagesListFiles="@(NonShippingPackagesList)" />
+                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
 
     <Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />
 
@@ -155,8 +122,8 @@
 
   <Target Name="RunSmokeTest">
     <ItemGroup>
-      <SdkTarballItem Include="$(SharedOutputPath)**/dotnet-sdk*$(ArchiveExtension)" />
-      <SourceBuiltArtifactsItem Include="$(SharedOutputPath)**/$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
+      <SdkTarballItem Include="$(ArtifactsShippingPackagesDir)dotnet-sdk*$(ArchiveExtension)" />
+      <SourceBuiltArtifactsItem Include="$(ArtifactsShippingPackagesDir)$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
     </ItemGroup>
 
     <PropertyGroup>
@@ -203,17 +170,18 @@
           DependsOnTargets="DetermineSourceBuiltSdkVersion"
           Condition="'@(SmokeTestsPrereqs->Count())' != '0'">
     <PropertyGroup>
-      <SmokeTestPrereqsTarballName>$(SharedOutputPath)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarballName>
+      <SmokeTestPrereqsTarball>$(ArtifactsShippingPackagesDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarball>
       <SmokeTestsPrereqPackagesDir>$(SmokeTestsArtifactsDir)prereq-packages/</SmokeTestsPrereqPackagesDir>
     </PropertyGroup>
 
     <Copy SourceFiles="@(SmokeTestsPrereqs)"
           DestinationFolder="$(SmokeTestsPrereqPackagesDir)" />
 
-    <Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarballName) ."
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SmokeTestPrereqsTarball)'))" />
+    <Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarball) ."
           WorkingDirectory="$(SmokeTestsPrereqPackagesDir)"/>
 
-    <Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarballName)'" />
+    <Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarball)'" />
   </Target>
 
   <Target Name="CreatePrebuiltsTarball"
@@ -236,13 +204,14 @@
           DependsOnTargets="DetermineSourceBuiltSdkVersion"
           Condition="'@(PrebuiltFile->Count())' != '0'">
     <PropertyGroup>
-      <TarballFilePath>$(SharedOutputPath)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</TarballFilePath>
+      <PrebuiltsTarball>$(ArtifactsShippingPackagesDir)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</PrebuiltsTarball>
       <TarballWorkingDir>$(ResultingPrebuiltPackagesDir)</TarballWorkingDir>
     </PropertyGroup>
 
-    <Exec Command="tar --numeric-owner -zcf $(TarballFilePath) -C $(TarballWorkingDir) ." />
+    <MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(PrebuiltsTarball)'))" />
+    <Exec Command="tar --numeric-owner -zcf $(PrebuiltsTarball) -C $(TarballWorkingDir) ." />
 
-    <Message Text="Tarball '$(TarballFilePath)' was successfully created from '$(TarballWorkingDir)'" Importance="High" />
+    <Message Text="Tarball '$(PrebuiltsTarball)' was successfully created from '$(TarballWorkingDir)'" Importance="High" />
   </Target>
 
   <Target Name="ErrorOnPrebuilts"

diff --git a/...ontent/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs b/...ontent/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs
@@ -66,11 +66,6 @@ public class CheckForPoison : Task
         /// </summary>
         public string OverrideTempPath { get; set; }
 
-        /// <summary>
-        /// Array of files containing lists of non-shipping packages
-        /// </summary>
-        public ITaskItem[] NonShippingPackagesListFiles { get; set; }
-
         private static readonly string[] ZipFileExtensions =
         {
             ".zip",
@@ -186,7 +181,6 @@ public override bool Execute()
         /// <returns>List of poisoned packages and files found and reasons for each</returns>
         internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
         {
-            IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
             IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
             var poisons = new List<PoisonedFileEntry>();
             var candidateQueue = new Queue<CandidateFileEntry>(initialCandidates.Select(candidate =>
@@ -209,12 +203,6 @@ internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> ini
                 {
                     Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}");
 
-                    // Skip non-shipping packages
-                    if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase))
-                    {
-                        continue;
-                    }
-
                     var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath));
                     PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue);
                     if (result != null)
@@ -237,21 +225,6 @@ internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> ini
             return poisons;
         }
 
-        private IEnumerable<string> GetAllNonShippingPackages()
-        {
-            if (NonShippingPackagesListFiles != null)
-            {
-                return NonShippingPackagesListFiles
-                    .SelectMany(item => File.ReadAllLines(item.ItemSpec))
-                    .Distinct()
-                    .ToList();
-            }
-            else
-            {
-                return Enumerable.Empty<string>();
-            }
-        }
-
         private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate)
         {
             // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as

diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props
@@ -17,10 +17,11 @@
     <ProjectDirectory>$([MSBuild]::NormalizeDirectory('$(SrcDir)', '$(RepositoryName)'))</ProjectDirectory>
 
     <!-- Paths to the version props files -->
-    <PackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).props</PackageVersionPropsPath>
-    <CurrentSourceBuiltPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Current.props</CurrentSourceBuiltPackageVersionPropsPath>
-    <PreviouslySourceBuiltPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Previous.props</PreviouslySourceBuiltPackageVersionPropsPath>
-    <SnapshotPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Snapshot.props</SnapshotPackageVersionPropsPath>
+    <PackageVersionsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'PackageVersions'))</PackageVersionsRootDir>
+    <PackageVersionPropsPath>$(PackageVersionsRootDir)PackageVersions.$(RepositoryName).props</PackageVersionPropsPath>
+    <CurrentSourceBuiltPackageVersionPropsPath>$(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Current.props</CurrentSourceBuiltPackageVersionPropsPath>
+    <PreviouslySourceBuiltPackageVersionPropsPath>$(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Previous.props</PreviouslySourceBuiltPackageVersionPropsPath>
+    <SnapshotPackageVersionPropsPath>$(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Snapshot.props</SnapshotPackageVersionPropsPath>
     <PackageVersionPropsFlowType>DependenciesOnly</PackageVersionPropsFlowType>
 
     <GlobalJsonFile Condition="'$(GlobalJsonFile)' == '' and Exists('$(ProjectDirectory)global.json')">$(ProjectDirectory)global.json</GlobalJsonFile>