Skip to content

Commit

Permalink
Put artifacts in well-known locations
Browse files Browse the repository at this point in the history
1. Shipping and non-shipping artifacts to Arcade default locations:
-> artifacts/packages/<config>/Shipping
-> artifacts/packages/<config>/NonShipping

Add the local feed for the addition package directory to NuGet.config
Remove the non-shipping packages list generation now that artifacts
are grouped.

2. Genereated the intermediate PackageVersions* files in
-> artifacts/obj/PackageVersions

3. Extract the repo symbol archives intermediates to
-> artifacts/obj/Symbols/

4. Sequence targets better and apply correct DependsOnTargets

5. Small clean-up

Fixes dotnet/source-build#4104
Fixes dotnet/source-build#3696
  • Loading branch information
ViktorHofer committed Feb 9, 2024
1 parent 4cd842e commit e6d76d5
Show file tree
Hide file tree
Showing 12 changed files with 169 additions and 220 deletions.
3 changes: 1 addition & 2 deletions eng/pipelines/templates/jobs/vmr-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,6 @@ jobs:
vmrPath: $(vmrPath)
vmrBranch: ${{ parameters.vmrBranch }}
targetRef: $(Build.SourceVersion) # Synchronize the current installer commit
architecture: ${{ parameters.architecture }}

- ${{ if parameters.buildFromArchive }}:
- script: |
Expand Down Expand Up @@ -399,7 +398,7 @@ jobs:
publishRunAttachments: true
testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName)

- publish: '$(sourcesPath)/artifacts/${{ parameters.architecture }}/Release/'
- publish: '$(sourcesPath)/artifacts/packages/Release/'
artifact: $(Agent.JobName)_Artifacts
displayName: Publish Artifacts
condition: succeededOrFailed()
Expand Down
15 changes: 0 additions & 15 deletions eng/pipelines/templates/steps/vmr-pull-updates.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@ parameters:
type: string
default: $(Agent.BuildDirectory)/vmr

- name: architecture
displayName: Architecture used for build artifacts
type: string
default: 'x64'

steps:
- checkout: self
displayName: Clone dotnet/installer
Expand All @@ -41,16 +36,6 @@ steps:
displayName: Prepare branch ${{ parameters.vmrBranch }}
workingDirectory: ${{ parameters.vmrPath }}

- script: |
mkdir -p ${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release
displayName: Create artifacts folder (Unix)
condition: ne(variables['Agent.OS'], 'Windows_NT')

- powershell: |
New-Item -ItemType Directory -Path ${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release
displayName: Create artifacts folder (Windows)
condition: eq(variables['Agent.OS'], 'Windows_NT')

- script: |
git config --global user.name "dotnet-maestro[bot]"
git config --global user.email "dotnet-maestro[bot]@users.noreply.github.com"
Expand Down
12 changes: 4 additions & 8 deletions src/SourceBuild/content/Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,9 @@
<ArtifactsObjDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'obj'))</ArtifactsObjDir>
<ArtifactsBinDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin'))</ArtifactsBinDir>
<ArtifactsLogDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'log', '$(Configuration)'))</ArtifactsLogDir>
<ArtifactsPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'packages', '$(Configuration)'))</ArtifactsPackagesDir>
<ArtifactsNonShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'NonShipping'))</ArtifactsNonShippingPackagesDir>
<ArtifactsShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsPackagesDir)', 'Shipping'))</ArtifactsShippingPackagesDir>
<RepositoryEngineeringDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'eng'))</RepositoryEngineeringDir>

<!-- ProjectLayout.props -->
Expand Down Expand Up @@ -168,14 +171,7 @@
<ArcadeBootstrapPackageDir Condition="'$(DotNetBuildSourceOnly)' != 'true'">$(NuGetPackageRoot)</ArcadeBootstrapPackageDir>

<VSMSBuildSdkResolversDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers'))</VSMSBuildSdkResolversDir>

<!-- Shared output and intermediate output path folders that are architecture and configuration specific. -->
<SharedOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedOutputPath>
<SharedIntermediateOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', '$(TargetArchitecture)', '$(Configuration)'))</SharedIntermediateOutputPath>

<SourceBuiltBlobFeedDir>$([MSBuild]::NormalizeDirectory('$(SharedIntermediateOutputPath)', 'blob-feed'))</SourceBuiltBlobFeedDir>
<SourceBuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'packages'))</SourceBuiltPackagesPath>
<SourceBuiltAssetsDir>$([MSBuild]::NormalizeDirectory('$(SourceBuiltBlobFeedDir)', 'assets'))</SourceBuiltAssetsDir>
<IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>

<PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
<PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>
Expand Down
2 changes: 1 addition & 1 deletion src/SourceBuild/content/Directory.Build.targets
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<SdkFilenamePrefix>dotnet-sdk-</SdkFilenamePrefix>
</PropertyGroup>
<ItemGroup>
<SdkTarballItem Include="$(SourceBuiltAssetsDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
<SdkTarballItem Include="$(ArtifactsShippingPackagesDir)$(SdkFilenamePrefix)*$(ArchiveExtension)" />
</ItemGroup>

<!--
Expand Down
15 changes: 0 additions & 15 deletions src/SourceBuild/content/build.proj
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,6 @@
StopOnFirstFailure="true" />
</Target>

<!-- Copies the output assets of the builds to the output path. -->
<Target Name="CopyBinariesToBinFolder"
AfterTargets="Build">
<ItemGroup>
<BinaryToCopy Include="$(SourceBuiltAssetsDir)*.*"
Exclude="$(SourceBuiltAssetsDir)*.nupkg;
$(SourceBuiltAssetsDir)*.requires_nupkg_signing" />
</ItemGroup>

<Copy SourceFiles="@(BinaryToCopy)"
DestinationFolder="$(SharedOutputPath)"
SkipUnchangedFiles="true"
Condition="'@(BinaryToCopy)'!=''" />
</Target>

<Import Project="$(RepositoryEngineeringDir)build.sourcebuild.targets" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />

</Project>
95 changes: 33 additions & 62 deletions src/SourceBuild/content/eng/build.sourcebuild.targets
Original file line number Diff line number Diff line change
Expand Up @@ -15,47 +15,21 @@
<MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
</Target>

<Target Name="DiscoverSymbolsTarballs"
AfterTargets="Build">
<ItemGroup>
<SymbolsTarball Include="$(SharedOutputPath)Symbols.*$(ArchiveExtension)" />
</ItemGroup>
</Target>

<Target Name="ExtractSymbolsTarballs"
AfterTargets="Build"
DependsOnTargets="DiscoverSymbolsTarballs"
Outputs="%(SymbolsTarball.Identity)">

<PropertyGroup>
<Filename>$([System.IO.Path]::GetFileName('%(SymbolsTarball.Identity)'))</Filename>
<RepositoryName>$(Filename.Split('.')[1])</RepositoryName>
<UnifiedSymbolsLayout>$(ArtifactsTmpDir)Symbols</UnifiedSymbolsLayout>
<DestinationFolder>$(UnifiedSymbolsLayout)/$(RepositoryName)</DestinationFolder>
</PropertyGroup>

<MakeDir Directories="$(DestinationFolder)" />
<Exec Command="tar -xzf %(SymbolsTarball.Identity) -C $(DestinationFolder)"
WorkingDirectory="$(SymbolsRoot)" />

<Delete Files="%(SymbolsTarball.Identity)" />
</Target>

<!-- After building, repackage symbols into a single tarball. -->
<Target Name="RepackageSymbols"
AfterTargets="Build"
DependsOnTargets="
DetermineSourceBuiltSdkVersion;
DiscoverSymbolsTarballs;
ExtractSymbolsTarballs">
DependsOnTargets="DetermineSourceBuiltSdkVersion">
<PropertyGroup>
<UnifiedSymbolsTarball>$(SharedOutputPath)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
<UnifiedSymbolsTarball>$(ArtifactsShippingPackagesDir)dotnet-symbols-all-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</UnifiedSymbolsTarball>
</PropertyGroup>

<MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(UnifiedSymbolsTarball)'))" />
<Exec Command="tar --numeric-owner -czf $(UnifiedSymbolsTarball) *"
WorkingDirectory="$(UnifiedSymbolsLayout)" />
WorkingDirectory="$(IntermediateSymbolsRootDir)" />

<Message Importance="High" Text="Packaged all symbols in '$(UnifiedSymbolsTarball)'" />

<RemoveDir Directories="$(IntermediateSymbolsRootDir)" />
</Target>

<!-- After building, create the sdk symbols tarball. -->
Expand All @@ -64,33 +38,33 @@
AfterTargets="Build"
DependsOnTargets="RepackageSymbols">
<ItemGroup>
<SdkTarballItem Include="$(SharedOutputPath)dotnet-sdk-*$(ArchiveExtension)" />
<SdkTarballItem Include="$(ArtifactsShippingPackagesDir)dotnet-sdk-*$(ArchiveExtension)" />
</ItemGroup>

<PropertyGroup>
<SdkSymbolsLayout>$(ArtifactsTmpDir)SdkSymbols</SdkSymbolsLayout>
<SdkSymbolsTarball>$(SharedOutputPath)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>
<SdkLayout>$(ArtifactsTmpDir)Sdk</SdkLayout>
<SdkSymbolsTarball>$(ArtifactsShippingPackagesDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension)</SdkSymbolsTarball>

<IntermediateSdkSymbolsLayout>$(BaseIntermediateOutputPath)SdkSymbols</IntermediateSdkSymbolsLayout>
<IntermediateSdkLayout>$(BaseIntermediateOutputPath)Sdk</IntermediateSdkLayout>
<SdkTarball>%(SdkTarballItem.Identity)</SdkTarball>
</PropertyGroup>

<MakeDir Directories="$(SdkLayout)" />
<Exec Command="tar -xzf $(SdkTarball) -C $(SdkLayout)"
WorkingDirectory="$(SharedOutputPath)" />
<MakeDir Directories="$(IntermediateSdkLayout)" />
<Exec Command="tar -xzf $(SdkTarball) -C $(IntermediateSdkLayout)" />

<CreateSdkSymbolsLayout SdkLayoutPath="$(SdkLayout)"
AllSymbolsPath="$(UnifiedSymbolsLayout)"
SdkSymbolsLayoutPath="$(SdkSymbolsLayout)"
<CreateSdkSymbolsLayout SdkLayoutPath="$(IntermediateSdkLayout)"
AllSymbolsPath="$(IntermediateSymbolsRootDir)"
SdkSymbolsLayoutPath="$(IntermediateSdkSymbolsLayout)"
FailOnMissingPDBs="false" />

<Exec Command="tar --numeric-owner -czf $(SdkSymbolsTarball) *"
WorkingDirectory="$(SdkSymbolsLayout)" />
WorkingDirectory="$(IntermediateSdkSymbolsLayout)" />

<Message Importance="High" Text="Packaged sdk symbols in '$(SdkSymbolsTarball)'" />

<RemoveDir Directories="$(UnifiedSymbolsLayout)" />
<RemoveDir Directories="$(SdkSymbolsLayout)" />
<RemoveDir Directories="$(SdkLayout)" />
<RemoveDir Directories="$(IntermediateSymbolsRootDir)" />
<RemoveDir Directories="$(IntermediateSdkSymbolsLayout)" />
<RemoveDir Directories="$(IntermediateSdkLayout)" />
</Target>

<!--
Expand All @@ -111,21 +85,16 @@
Inputs="$(MSBuildProjectFullPath)"
Outputs="$(BaseIntermediateOutputPath)ReportPoisonUsage.complete" >
<ItemGroup>
<FinalCliTarball Include="$(SharedOutputPath)**/*$(ArchiveExtension)" />
<FinalCliTarball Include="$(ArtifactsShippingPackagesDir)*$(ArchiveExtension)" />
</ItemGroup>

<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Checking @(FinalCliTarball) for poisoned files." />

<ItemGroup>
<NonShippingPackagesList Include="$(PackageListsDir)**/$(NonShippingPackagesListPrefix)*" />
</ItemGroup>

<CheckForPoison FilesToCheck="@(FinalCliTarball)"
ProjectDirPath="$(RepoRoot)"
HashCatalogFilePath="$(PoisonReportDataFile)"
MarkerFileName="$(PoisonMarkerFile)"
PoisonReportOutputFilePath="$(PoisonUsageReportFile)"
NonShippingPackagesListFiles="@(NonShippingPackagesList)" />
PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />

<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Done checking for poison." />

Expand Down Expand Up @@ -155,8 +124,8 @@

<Target Name="RunSmokeTest">
<ItemGroup>
<SdkTarballItem Include="$(SharedOutputPath)**/dotnet-sdk*$(ArchiveExtension)" />
<SourceBuiltArtifactsItem Include="$(SharedOutputPath)**/$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
<SdkTarballItem Include="$(ArtifactsShippingPackagesDir)dotnet-sdk*$(ArchiveExtension)" />
<SourceBuiltArtifactsItem Include="$(ArtifactsShippingPackagesDir)$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
</ItemGroup>

<PropertyGroup>
Expand Down Expand Up @@ -203,17 +172,18 @@
DependsOnTargets="DetermineSourceBuiltSdkVersion"
Condition="'@(SmokeTestsPrereqs->Count())' != '0'">
<PropertyGroup>
<SmokeTestPrereqsTarballName>$(SharedOutputPath)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarballName>
<SmokeTestPrereqsTarball>$(ArtifactsShippingPackagesDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarball>
<SmokeTestsPrereqPackagesDir>$(SmokeTestsArtifactsDir)prereq-packages/</SmokeTestsPrereqPackagesDir>
</PropertyGroup>

<Copy SourceFiles="@(SmokeTestsPrereqs)"
DestinationFolder="$(SmokeTestsPrereqPackagesDir)" />

<Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarballName) ."
<MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SmokeTestPrereqsTarball)'))" />
<Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarball) ."
WorkingDirectory="$(SmokeTestsPrereqPackagesDir)"/>

<Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarballName)'" />
<Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarball)'" />
</Target>

<Target Name="CreatePrebuiltsTarball"
Expand All @@ -236,13 +206,14 @@
DependsOnTargets="DetermineSourceBuiltSdkVersion"
Condition="'@(PrebuiltFile->Count())' != '0'">
<PropertyGroup>
<TarballFilePath>$(SharedOutputPath)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</TarballFilePath>
<TarballWorkingDir>$(ResultingPrebuiltPackagesDir)</TarballWorkingDir>
<PrebuiltsTarball>$(ArtifactsShippingPackagesDir)$(SourceBuiltPrebuiltsTarballName).$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</PrebuiltsTarball>
<PrebuiltsTarballWorkingDir>$(ResultingPrebuiltPackagesDir)</PrebuiltsTarballWorkingDir>
</PropertyGroup>

<Exec Command="tar --numeric-owner -zcf $(TarballFilePath) -C $(TarballWorkingDir) ." />
<MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(PrebuiltsTarball)'))" />
<Exec Command="tar --numeric-owner -zcf $(PrebuiltsTarball) -C $(PrebuiltsTarballWorkingDir) ." />

<Message Text="Tarball '$(TarballFilePath)' was successfully created from '$(TarballWorkingDir)'" Importance="High" />
<Message Text="Tarball '$(PrebuiltsTarball)' was successfully created from '$(PrebuiltsTarballWorkingDir)'" Importance="High" />
</Target>

<Target Name="ErrorOnPrebuilts"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,6 @@ public class CheckForPoison : Task
/// </summary>
public string OverrideTempPath { get; set; }

/// <summary>
/// Array of files containing lists of non-shipping packages
/// </summary>
public ITaskItem[] NonShippingPackagesListFiles { get; set; }

private static readonly string[] ZipFileExtensions =
{
".zip",
Expand Down Expand Up @@ -186,7 +181,6 @@ public override bool Execute()
/// <returns>List of poisoned packages and files found and reasons for each</returns>
internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> initialCandidates, string catalogedPackagesFilePath, string markerFileName)
{
IEnumerable<string> nonShippingPackages = GetAllNonShippingPackages();
IEnumerable<CatalogPackageEntry> catalogedPackages = ReadCatalog(catalogedPackagesFilePath);
var poisons = new List<PoisonedFileEntry>();
var candidateQueue = new Queue<CandidateFileEntry>(initialCandidates.Select(candidate =>
Expand All @@ -209,12 +203,6 @@ internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> ini
{
Log.LogMessage($"Zip or NuPkg file to check: {candidate.ExtractedPath}");

// Skip non-shipping packages
if (nonShippingPackages.Contains(Path.GetFileName(candidate.ExtractedPath), StringComparer.OrdinalIgnoreCase))
{
continue;
}

var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetFileNameWithoutExtension(candidate.ExtractedPath));
PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, candidate, markerFileName, tempCheckingDir, candidateQueue);
if (result != null)
Expand All @@ -237,21 +225,6 @@ internal IEnumerable<PoisonedFileEntry> GetPoisonedFiles(IEnumerable<string> ini
return poisons;
}

private IEnumerable<string> GetAllNonShippingPackages()
{
if (NonShippingPackagesListFiles != null)
{
return NonShippingPackagesListFiles
.SelectMany(item => File.ReadAllLines(item.ItemSpec))
.Distinct()
.ToList();
}
else
{
return Enumerable.Empty<string>();
}
}

private static PoisonedFileEntry CheckSingleFile(IEnumerable<CatalogPackageEntry> catalogedPackages, CandidateFileEntry candidate)
{
// skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as
Expand Down
9 changes: 5 additions & 4 deletions src/SourceBuild/content/repo-projects/Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@
<ProjectDirectory>$([MSBuild]::NormalizeDirectory('$(SrcDir)', '$(RepositoryName)'))</ProjectDirectory>

<!-- Paths to the version props files -->
<PackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).props</PackageVersionPropsPath>
<CurrentSourceBuiltPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Current.props</CurrentSourceBuiltPackageVersionPropsPath>
<PreviouslySourceBuiltPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Previous.props</PreviouslySourceBuiltPackageVersionPropsPath>
<SnapshotPackageVersionPropsPath>$(SharedIntermediateOutputPath)PackageVersions.$(RepositoryName).Snapshot.props</SnapshotPackageVersionPropsPath>
<PackageVersionsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'PackageVersions'))</PackageVersionsRootDir>
<PackageVersionPropsPath>$(PackageVersionsRootDir)PackageVersions.$(RepositoryName).props</PackageVersionPropsPath>
<CurrentSourceBuiltPackageVersionPropsPath>$(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Current.props</CurrentSourceBuiltPackageVersionPropsPath>
<PreviouslySourceBuiltPackageVersionPropsPath>$(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Previous.props</PreviouslySourceBuiltPackageVersionPropsPath>
<SnapshotPackageVersionPropsPath>$(PackageVersionsRootDir)PackageVersions.$(RepositoryName).Snapshot.props</SnapshotPackageVersionPropsPath>
<PackageVersionPropsFlowType>DependenciesOnly</PackageVersionPropsFlowType>

<GlobalJsonFile Condition="'$(GlobalJsonFile)' == '' and Exists('$(ProjectDirectory)global.json')">$(ProjectDirectory)global.json</GlobalJsonFile>
Expand Down
Loading

0 comments on commit e6d76d5

Please sign in to comment.