Fix publishing symbols (#5556)

* Disable Portable PDB conversion * Push packages to artifacts * Fix symbols issues * Added note about Microsoft.ML.dll * try out just packing * Return Build=false, but actually use configuration * Added missing TargetArchitecture * add back tests * Added missing flags
dotnet · Dec 17, 2020 · 6fe50e4 · 6fe50e4
1 parent 70ecfff
commit 6fe50e4
Show file tree

Hide file tree

Showing 9 changed files with 112 additions and 43 deletions.
diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml
@@ -197,5 +197,5 @@ jobs:
         artifactName: ${{ parameters.name }} $(_config_short)
         artifactType: container
     - ${{ if eq(parameters.nightlyBuild, 'false') }}:
-      - script: ${{ parameters.buildScript }} /p:Build=false -pack -ci  /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
-        displayName: Build Packages
+      - script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
+        displayName: Build Packages
diff --git a/build/publish.proj b/build/publish.proj
@@ -23,7 +23,7 @@
   <Target Name="SetupPublishSymbols">
     <PropertyGroup>
       <SymbolExpirationInDays Condition="'$(SymbolExpirationInDays)'=='' and '$(SymbolExpirationDate)'==''">180</SymbolExpirationInDays>
-      <ConvertPortablePdbsToWindowsPdbs>true</ConvertPortablePdbsToWindowsPdbs>
+      <ConvertPortablePdbsToWindowsPdbs>false</ConvertPortablePdbsToWindowsPdbs>
     </PropertyGroup>
 
     <ItemGroup>

diff --git a/build/vsts-ci.yml b/build/vsts-ci.yml
@@ -243,6 +243,16 @@ phases:
     displayName: sign packages
     continueOnError: false
 
+  # The generated .nupkgs and .snupkgs packages are published to Azure artifacts,
+  # in case we need to debug them. They're found under Release/Shipping
+  - task: PublishBuildArtifacts@1
+    displayName: Push packages and symbol packages to Azure Artifacts
+    inputs:
+      pathToPublish: $(Build.SourcesDirectory)/artifacts/packages
+      artifactName: PackageAssets
+      artifactType: container
+    continueOnError: true
+
   - task: NuGetAuthenticate@0
     inputs:
       nuGetServiceConnections: machinelearning-dnceng-public-feed # To allow publishing to a feed of another organization

diff --git a/src/Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.csproj b/src/Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.csproj
@@ -1,40 +1,41 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="$(RepoRoot)eng/pkg/Pack.props"/>
-
   <PropertyGroup>
     <TargetFrameworks>netstandard2.0;netcoreapp3.1</TargetFrameworks>
     <IncludeInPackage>Microsoft.ML.CpuMath</IncludeInPackage>
     <PackageDescription>Microsoft.ML.CpuMath contains optimized math routines for ML.NET.</PackageDescription>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <DefineConstants>$(DefineConstants);CPUMATH_INFRASTRUCTURE</DefineConstants>
+    <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
   </PropertyGroup>
-
   <ItemGroup>
     <Compile Include="..\Microsoft.ML.Core\Utilities\Contracts.cs" />
     <Compile Include="..\Microsoft.ML.Core\BestFriendAttribute.cs" />
     <Compile Include="..\Microsoft.ML.Core\PublicKey.cs" />
-
     <!-- Workaround https://github.com/dotnet/project-system/issues/935 -->
     <None Include="**/*.cs" />
   </ItemGroup>
-
   <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp3.1'">
     <Compile Remove="CpuMathUtils.netstandard.cs" />
     <Compile Remove="FactorizationMachine/FactorizationMachineInterface.netstandard.cs" />
   </ItemGroup>
-
   <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
     <Compile Remove="CpuMathUtils.netcoreapp.cs" />
     <Compile Remove="SseIntrinsics.cs" />
     <Compile Remove="AvxIntrinsics.cs" />
     <Compile Remove="FactorizationMachine/FactorizationMachineInterface.netcoreapp.cs" />
     <Compile Remove="FactorizationMachine/AvxIntrinsics.cs" />
-
     <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
   </ItemGroup>
-
   <ItemGroup>
     <Content Include="build\**\*" Pack="true" PackagePath="build" />
   </ItemGroup>
-
-</Project>
+  <Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
+    <ItemGroup Condition="'$(TargetFramework)' != 'netcoreapp3.1'">
+      <!--Include native PDBs-->
+      <!--The path needed to be hardcoded for this to work on our publishing CI-->
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\nativeassets\netstandard2.0\CpuMathNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\nativeassets\netstandard2.0\CpuMathNative.pdb" TargetPath="..\..\runtimes\win-x86\nativeassets\netstandard2.0" />
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\nativeassets\netstandard2.0\CpuMathNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\nativeassets\netstandard2.0\CpuMathNative.pdb" TargetPath="..\..\runtimes\win-x64\nativeassets\netstandard2.0" />
+    </ItemGroup>
+  </Target>
+</Project>
diff --git a/src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj b/src/Microsoft.ML.FastTree/Microsoft.ML.FastTree.csproj
@@ -1,17 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="$(RepoRoot)eng/pkg/Pack.props" />
-
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
     <IncludeInPackage>Microsoft.ML.FastTree</IncludeInPackage>
     <PackageDescription>ML.NET component for FastTree</PackageDescription>
     <DefineConstants>$(DefineConstants);USE_FASTTREENATIVE;NO_STORE;CORECLR</DefineConstants>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
   </PropertyGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
-
     <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" >
       <PrivateAssets>all</PrivateAssets>
     </ProjectReference>
@@ -22,9 +20,15 @@
       <PrivateAssets>all</PrivateAssets>
     </ProjectReference>
   </ItemGroup>
-
   <ItemGroup>
     <Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
   </ItemGroup>
-
-</Project>
+  <Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
+    <ItemGroup>
+      <!--Include native PDBs-->
+      <!--The path needed to be hardcoded for this to work on our publishing CI-->
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\FastTreeNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\FastTreeNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\FastTreeNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\FastTreeNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
+    </ItemGroup>
+  </Target>
+</Project>
diff --git a/src/Microsoft.ML.Mkl.Components/Microsoft.ML.Mkl.Components.csproj b/src/Microsoft.ML.Mkl.Components/Microsoft.ML.Mkl.Components.csproj
@@ -1,17 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="$(RepoRoot)eng/pkg/Pack.props" />
-
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
     <IncludeInPackage>Microsoft.ML.Mkl.Components</IncludeInPackage>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <PackageDescription>ML.NET additional learners making use of Intel Mkl.</PackageDescription>
+    <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
   </PropertyGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
     <ProjectReference Include="..\Microsoft.ML.Mkl.Redist\Microsoft.ML.Mkl.Redist.csproj" />
-
     <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" >
       <PrivateAssets>all</PrivateAssets>
     </ProjectReference>
@@ -22,11 +20,9 @@
       <PrivateAssets>all</PrivateAssets>
     </ProjectReference>
   </ItemGroup>
-
   <ItemGroup>
     <Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
   </ItemGroup>
-
   <ItemGroup>
     <!-- The package reference is in there, so that the build can download the MlNetMklDeps package at build time.
     The binaries of this package, are extraceted, and repackaged in the Microsoft.ML.Mkl.Redist; that is the one that has a true
@@ -35,5 +31,12 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
   </ItemGroup>
-
-</Project>
+  <Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
+    <ItemGroup>
+      <!--Include native PDBs-->
+      <!--The path needed to be hardcoded for this to work on our publishing CI-->
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\SymSgdNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\SymSgdNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\SymSgdNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\SymSgdNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
+    </ItemGroup>
+  </Target>
+</Project>
diff --git a/src/Microsoft.ML.Mkl.Redist/Microsoft.ML.Mkl.Redist.csproj b/src/Microsoft.ML.Mkl.Redist/Microsoft.ML.Mkl.Redist.csproj
@@ -1,23 +1,28 @@
 <Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
   <Import Project="$(RepoRoot)eng/pkg/Pack.props" />
-
   <PropertyGroup>
     <Authors>Intel</Authors>
-    <IncludeSymbols>false</IncludeSymbols>
-    <IncludeBuildOutput>false</IncludeBuildOutput>
+    <IncludeSymbols>true</IncludeSymbols>
+    <IncludeBuildOutput>true</IncludeBuildOutput>
     <TargetFramework>netstandard2.0</TargetFramework>
     <PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
     <PackageDescription>$(MSBuildProjectName) contains the MKL library redistributed as a NuGet package.</PackageDescription>
     <PackageTags>$(PackageTags) MLNET MKL</PackageTags>
+    <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
   </PropertyGroup>
-
   <PropertyGroup>
     <IncludeMLNetNotices>false</IncludeMLNetNotices>
   </PropertyGroup>
-
   <ItemGroup>
     <Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
     <Content Include="$(RepoRoot)eng\pkg\_._" Pack="true" PackagePath="lib\netstandard2.0\" />
     <Content Include="$(NUGET_PACKAGES)\mlnetmkldeps\$(MlNetMklDepsPackageVersion)\LICENSE.txt" Pack="true" PackagePath="" />
   </ItemGroup>
-</Project>
+  <Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
+    <ItemGroup>
+      <!--Include native PDBs-->
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\MklProxyNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\MklProxyNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\MklProxyNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\MklProxyNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
+    </ItemGroup>
+  </Target>
+</Project>
diff --git a/src/Microsoft.ML.Recommender/Microsoft.ML.Recommender.csproj b/src/Microsoft.ML.Recommender/Microsoft.ML.Recommender.csproj
@@ -1,27 +1,31 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <Import Project="$(RepoRoot)eng/pkg/Pack.props" />
-
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
     <IncludeInPackage>Microsoft.ML.Recommender</IncludeInPackage>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <PackageDescription>LIBMF, the core computation library for matrix factorization in ML.NET</PackageDescription>
+    <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
   </PropertyGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.ML\Microsoft.ML.csproj" />
-
     <ProjectReference Include="..\Microsoft.ML.Core\Microsoft.ML.Core.csproj" >
       <PrivateAssets>all</PrivateAssets>
     </ProjectReference>
     <ProjectReference Include="..\Microsoft.ML.Data\Microsoft.ML.Data.csproj" >
       <PrivateAssets>all</PrivateAssets>
     </ProjectReference>
   </ItemGroup>
-
   <ItemGroup>
     <Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
     <Content Include="$(RepoRoot)src\Native\MatrixFactorizationNative\libmf\COPYRIGHT" Pack="true" PackagePath="" />
   </ItemGroup>
-
-</Project>
+  <Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
+    <ItemGroup>
+      <!--Include native PDBs-->
+      <!--The path needed to be hardcoded for this to work on our publishing CI-->
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\MatrixFactorizationNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\MatrixFactorizationNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\MatrixFactorizationNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\MatrixFactorizationNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
+    </ItemGroup>
+  </Target>
+</Project>
diff --git a/src/Microsoft.ML/Microsoft.ML.csproj b/src/Microsoft.ML/Microsoft.ML.csproj
@@ -1,27 +1,69 @@
 <Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
   <Import Project="$(RepoRoot)eng/pkg/Pack.props"/>
-
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
-    <IncludeBuildOutput>false</IncludeBuildOutput>
+    <!-- To include the DLL and PDBs of the underlying projects
+    We need to remove the IncludeBuildOutput=false. This also generates a
+    Microsoft.ML.dll in the nuget package, and a Microsoft.ML.pdb in the .snupkg
+    But these are to be ignored, as they don't contain anything meaningful-->
+    <!-- <IncludeBuildOutput>false</IncludeBuildOutput> -->
     <NoWarn>$(NoWarn);NU5127;NU5128</NoWarn>
     <PackageDescription>ML.NET is a cross-platform open-source machine learning framework which makes machine learning accessible to .NET developers.</PackageDescription>
+    <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
   </PropertyGroup>
-
   <ItemGroup>
     <ProjectReference Include="../Microsoft.ML.DataView/Microsoft.ML.DataView.csproj" />
     <ProjectReference Include="../Microsoft.ML.CpuMath/Microsoft.ML.CpuMath.csproj" />
-
+    <ProjectReference Include="../Microsoft.ML.Core/Microsoft.ML.Core.csproj">
+      <IncludeInNuget>true</IncludeInNuget>
+      <PrivateAssets>all</PrivateAssets>
+    </ProjectReference>
+    <ProjectReference Include="../Microsoft.ML.Data/Microsoft.ML.Data.csproj">
+      <IncludeInNuget>true</IncludeInNuget>
+      <PrivateAssets>all</PrivateAssets>
+    </ProjectReference>
+    <ProjectReference Include="../Microsoft.ML.KMeansClustering/Microsoft.ML.KMeansClustering.csproj">
+      <IncludeInNuget>true</IncludeInNuget>
+      <PrivateAssets>all</PrivateAssets>
+    </ProjectReference>
+    <ProjectReference Include="../Microsoft.ML.PCA/Microsoft.ML.PCA.csproj">
+      <IncludeInNuget>true</IncludeInNuget>
+      <PrivateAssets>all</PrivateAssets>
+    </ProjectReference>
+    <ProjectReference Include="../Microsoft.ML.StandardTrainers/Microsoft.ML.StandardTrainers.csproj">
+      <IncludeInNuget>true</IncludeInNuget>
+      <PrivateAssets>all</PrivateAssets>
+    </ProjectReference>
+    <ProjectReference Include="../Microsoft.ML.Transforms/Microsoft.ML.Transforms.csproj">
+      <IncludeInNuget>true</IncludeInNuget>
+      <PrivateAssets>all</PrivateAssets>
+    </ProjectReference>
+    <NativeAssemblyReference Include="LdaNative" />
     <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonPackageVersion)" />
     <PackageReference Include="System.Reflection.Emit.Lightweight" Version="$(SystemReflectionEmitLightweightPackageVersion)" />
     <PackageReference Include="System.CodeDom" Version="$(SystemCodeDomPackageVersion)" />
     <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
     <PackageReference Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
     <PackageReference Include="System.Threading.Channels" Version="$(SystemThreadingChannelsPackageVersion)" />
   </ItemGroup>
-
   <ItemGroup>
     <Content Include="$(RepoRoot)eng\pkg\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
     <Content Include="build\**\*" Pack="true" PackagePath="build" />
   </ItemGroup>
-</Project>
+<PropertyGroup>
+  <!--Include Project References output-->
+  <TargetsForTfmSpecificBuildOutput>$(TargetsForTfmSpecificBuildOutput);CopyProjectReferencesToPackage</TargetsForTfmSpecificBuildOutput>
+</PropertyGroup>
+  <Target DependsOnTargets="ResolveReferences" Name="CopyProjectReferencesToPackage">
+    <ItemGroup>
+      <!--Include DLLs of Project References-->
+      <BuildOutputInPackage Include="@(ReferenceCopyLocalPaths->WithMetadataValue('ReferenceSourceTarget', 'ProjectReference')->WithMetadataValue('IncludeInNuget','true'))"/>
+      <!--Include PDBs of Project References-->
+      <BuildOutputInPackage Include="@(ReferenceCopyLocalPaths->WithMetadataValue('ReferenceSourceTarget', 'ProjectReference')->WithMetadataValue('IncludeInNuget','true')->Replace('.dll', '.pdb'))"/>
+      <!--Include PDBs for Native binaries-->
+      <!--The path needed to be hardcoded for this to work on our publishing CI-->
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\LdaNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\LdaNative.pdb" TargetPath="..\..\runtimes\win-x86\native"/>
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\LdaNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\LdaNative.pdb" TargetPath="..\..\runtimes\win-x64\native"/>
+    </ItemGroup>
+  </Target>
+</Project>