Merge pull request #16 from nguerrera/embed-source-in-pdb

Support reading embedded source

src/DeployCoreClrTestRuntime/project.json

@@ -17,6 +17,7 @@
     "System.Diagnostics.Tools": "4.0.1",
     "System.Dynamic.Runtime": "4.0.11",
     "System.Globalization": "4.0.11",
+    "System.IO.Compression": "4.1.0",
     "System.IO.FileSystem": "4.0.1",
     "System.IO.FileSystem.Primitives": "4.0.1",
     "System.IO.FileSystem.Watcher": "4.0.0",

src/Microsoft.DiaSymReader.Converter.Tests/Microsoft.DiaSymReader.Converter.UnitTests.csproj

@@ -37,6 +37,9 @@
   <ItemGroup>
     <Compile Include="PdbConverterTests.cs" />
   </ItemGroup>
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
   <ImportGroup>
     <Import Project="..\..\build\Targets\Imports.targets" />
     <Import Project="..\..\build\Toolset\XunitProjectRunAction.targets" />

src/Microsoft.DiaSymReader.PortablePdb.Tests/EmbeddedSourceTests.cs

@@ -0,0 +1,157 @@
+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using Roslyn.Test.Utilities;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.IO.Compression;
+using System.Reflection.Metadata;
+using Xunit;
+
+namespace Microsoft.DiaSymReader.PortablePdb.UnitTests
+{
+    using static SymTestHelpers;
+
+    public class EmbeddedSourceTests
+    {
+        [Fact]
+        public void EmbeddedSource_Portable()
+        {
+            EmbeddedSource(TestResources.EmbeddedSource.PortableDllAndPdb);
+        }
+
+        // TODO: Once native PDB support is implemented, add /embed to EmbeddedSource.cmd 
+        // native command line, run it to regenerate native dll and PDB, and unskip this test.
+        [Fact(Skip = "Native support not yet implemented.")]
+        public void EmbeddedSource_Native()
+        {
+            EmbeddedSource(TestResources.EmbeddedSource.DllAndPdb);
+        }
+
+        private void EmbeddedSource(KeyValuePair<byte[], byte[]> dllAndPdb)
+        {
+            ISymUnmanagedReader symReader = CreateSymReaderFromResource(dllAndPdb);
+
+            foreach (string file in new[] { @"C:\EmbeddedSource.cs", @"C:\EmbeddedSourceSmall.cs" })
+            {
+                bool hasEmbeddedSource;
+                int length, bytesRead;
+                ISymUnmanagedDocument doc;
+                Assert.Equal(HResult.S_OK, symReader.GetDocument(file, default(Guid), default(Guid), default(Guid), out doc));
+                Assert.Equal(HResult.S_OK, doc.HasEmbeddedSource(out hasEmbeddedSource));
+                Assert.Equal(HResult.S_OK, doc.GetSourceLength(out length));
+                Assert.True(hasEmbeddedSource);
+
+                var blob = new byte[length];
+                Assert.Equal(HResult.S_OK, doc.GetSourceRange(0, 0, int.MaxValue, int.MaxValue, length, out bytesRead, blob));
+                Assert.Equal(length, bytesRead);
+
+                Assert.True(length >= sizeof(int));
+
+                byte[] expectedContent;
+                int uncompressedSize = BitConverter.ToInt32(blob, 0);
+
+                if (uncompressedSize == 0)
+                {
+                    Assert.Equal(@"C:\EmbeddedSourceSmall.cs", file);
+                    expectedContent = TestResources.EmbeddedSource.CSSmall;
+                }
+                else
+                {
+                    Assert.Equal(@"C:\EmbeddedSource.cs", file);
+                    expectedContent = TestResources.EmbeddedSource.CS;
+                }
+
+                AssertEx.Equal(expectedContent, Decode(blob, uncompressedSize));
+            }
+        }
+
+        [Fact]
+        public void NoEmbeddedSource_Portable()
+        {
+            NoEmbeddedSource(TestResources.Documents.PortableDllAndPdb);
+        }
+
+        [Fact]
+        public void NoEmbeddedSource_Native()
+        {
+            NoEmbeddedSource(TestResources.Documents.DllAndPdb);
+        }
+
+        private void NoEmbeddedSource(KeyValuePair<byte[], byte[]> dllAndPdb)
+        {
+            ISymUnmanagedReader symReader = CreateSymReaderFromResource(dllAndPdb);
+
+            ISymUnmanagedDocument doc;
+            Assert.Equal(HResult.S_OK, symReader.GetDocument(@"C:\Documents.cs", default(Guid), default(Guid), default(Guid), out doc));
+
+            bool hasEmbeddedSource;
+            Assert.Equal(HResult.S_OK, doc.HasEmbeddedSource(out hasEmbeddedSource));
+            Assert.False(hasEmbeddedSource);
+
+            int length;
+            Assert.Equal(HResult.S_OK, doc.GetSourceLength(out length));
+            Assert.Equal(0, length);
+
+            Assert.Equal(HResult.S_FALSE, doc.GetSourceRange(0, 0, -1, -1, 0, out length, null));
+            Assert.Equal(HResult.S_FALSE, doc.GetSourceRange(0, 0, int.MaxValue, int.MaxValue, 0, out length, null));
+        }
+
+        [Fact]
+        public void BadArgs_Portable()
+        {
+            BadArgs(TestResources.EmbeddedSource.PortableDllAndPdb);
+        }
+
+        [Fact]
+        public void BadArgs_Native()
+        {
+            BadArgs(TestResources.EmbeddedSource.DllAndPdb);
+        }
+
+        private void BadArgs(KeyValuePair<byte[], byte[]> dllAndPdb)
+        {
+            ISymUnmanagedReader symReader = CreateSymReaderFromResource(dllAndPdb);
+
+            ISymUnmanagedDocument doc;
+            Assert.Equal(HResult.S_OK, symReader.GetDocument(@"C:\EmbeddedSource.cs", default(Guid), default(Guid), default(Guid), out doc));
+
+            int count;
+            Assert.Equal(HResult.E_INVALIDARG, doc.GetSourceRange(-1, 0, int.MaxValue, int.MaxValue, 0, out count, null));
+            Assert.Equal(HResult.E_INVALIDARG, doc.GetSourceRange(0, -1, int.MaxValue, int.MaxValue, 0, out count, null));
+            Assert.Equal(HResult.E_INVALIDARG, doc.GetSourceRange(0, 0, 1, int.MaxValue, 0, out count, null));
+            Assert.Equal(HResult.E_INVALIDARG, doc.GetSourceRange(0, 0, int.MaxValue, 1, 0, out count, null));
+            Assert.Equal(HResult.E_INVALIDARG, doc.GetSourceRange(0, 0, int.MaxValue, int.MaxValue, 1, out count, null));
+
+            // negative bufferLength test does not apply to native as it uses uint arguments.
+            if (dllAndPdb.Equals(TestResources.EmbeddedSource.PortableDllAndPdb))
+            {
+                Assert.Equal(HResult.E_INVALIDARG, doc.GetSourceRange(0, 0, int.MaxValue, int.MaxValue, -1, out count, new byte[1]));
+            }
+        }
+
+        private byte[] Decode(byte[] blob, int uncompressedSize)
+        {
+            Assert.True(uncompressedSize >= 0);
+
+            if (uncompressedSize == 0)
+            {
+                byte[] content = new byte[blob.Length - sizeof(int)];
+                Array.Copy(blob, sizeof(int), content, 0, content.Length);
+                return content;
+            }
+            else
+            {
+                var compressed = new MemoryStream(blob, sizeof(int), blob.Length - sizeof(int));
+                using (var decompressor = new DeflateStream(compressed, CompressionMode.Decompress))
+                using (var decompressed = new MemoryStream(uncompressedSize))
+                {
+                    decompressor.CopyTo(decompressed);
+                    Assert.Equal(uncompressedSize, decompressed.Length);
+                    return decompressed.ToArray();
+                }
+            }
+        }
+    }
+}

src/Microsoft.DiaSymReader.PortablePdb.Tests/Microsoft.DiaSymReader.PortablePdb.UnitTests.csproj

@@ -77,10 +77,29 @@
     <EmbeddedResource Include="Resources\MiscEmbedded.dll">
       <LogicalName>MiscEmbedded.dll</LogicalName>
     </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.cs">
+      <LogicalName>EmbeddedSource.cs</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSourceSmall.cs">
+      <LogicalName>EmbeddedSourceSmall.cs</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.dll">
+      <LogicalName>EmbeddedSource.dll</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.pdb">
+      <LogicalName>EmbeddedSource.pdb</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.dllx">
+      <LogicalName>EmbeddedSource.dllx</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.pdbx">
+      <LogicalName>EmbeddedSource.pdbx</LogicalName>
+    </EmbeddedResource>
   </ItemGroup>
   <ItemGroup>
     <Content Include="Resources\Documents.cmd" />
     <Content Include="Resources\Documents.cs" />
+    <Content Include="Resources\EmbeddedSource.cmd" />
     <Content Include="Resources\Scopes.cmd" />
     <Content Include="Resources\Scopes.cs" />
     <Content Include="Resources\Async.cmd" />
@@ -91,6 +110,7 @@
     <Content Include="Resources\MiscEmbedded.cs" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="EmbeddedSourceTests.cs" />
     <Compile Include="MethodMapTests.cs" />
     <Compile Include="ResourceLoader.cs" />
     <Compile Include="SymBinderTests.cs" />

src/Microsoft.DiaSymReader.PortablePdb.Tests/Resources/EmbeddedSource.cmd

@@ -0,0 +1,7 @@
+csc /target:library /debug:portable /optimize- /deterministic /pathmap:%~dp0=C:\ /embed EmbeddedSource.cs EmbeddedSourceSmall.cs
+copy /y EmbeddedSource.pdb EmbeddedSource.pdbx
+copy /y EmbeddedSource.dll EmbeddedSource.dllx
+
+@REM -- TODO: Native support not there yet, hence no /embed, add it when implemented and unskip EmbeddedSource_Native test.
+csc /target:library /debug+ /optimize- /deterministic /pathmap:%~dp0=C:\ EmbeddedSource.cs EmbeddedSourceSmall.cs
+

src/Microsoft.DiaSymReader.PortablePdb.Tests/Resources/EmbeddedSource.cs

@@ -0,0 +1,19 @@
+// should be higher than compression threshold (200 chars)
+
+using System;
+
+namespace Test
+{
+    public static class SomeCode
+    {
+        public static int SomeMethod(int value)
+        {
+            if (value < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(value));
+            }
+
+            return checked(value + 42);
+        }
+    }
+}

src/Microsoft.DiaSymReader.PortablePdb.Tests/Resources/EmbeddedSourceSmall.cs

@@ -0,0 +1,4 @@
+// should be less than compression threshold (200 chars)
+public class Small
+{
+}

src/Microsoft.DiaSymReader.PortablePdb.Tests/TestResources.cs

@@ -1,7 +1,5 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
 using System.Collections.Generic;
 
 namespace TestResources
@@ -76,4 +74,28 @@ public static class MiscEmbedded
         private static byte[] s_dll;
         public static byte[] Dll => ResourceLoader.GetOrCreateResource(ref s_dll, nameof(MiscEmbedded) + ".dll");
     }
+
+    public static class EmbeddedSource
+    {
+        private static byte[] s_portableDll;
+        public static byte[] PortableDll => ResourceLoader.GetOrCreateResource(ref s_portableDll, nameof(EmbeddedSource) + ".dllx");
+
+        private static byte[] s_portablePdb;
+        public static byte[] PortablePdb => ResourceLoader.GetOrCreateResource(ref s_portablePdb, nameof(EmbeddedSource) + ".pdbx");
+
+        private static byte[] s_dll;
+        public static byte[] Dll => ResourceLoader.GetOrCreateResource(ref s_dll, nameof(EmbeddedSource) + ".dll");
+
+        private static byte[] s_pdb;
+        public static byte[] Pdb => ResourceLoader.GetOrCreateResource(ref s_pdb, nameof(EmbeddedSource) + ".pdb");
+
+        private static byte[] s_cs;
+        public static byte[] CS => ResourceLoader.GetOrCreateResource(ref s_cs, nameof(EmbeddedSource) + ".cs");
+
+        private static byte[] s_csSmall;
+        public static byte[] CSSmall => ResourceLoader.GetOrCreateResource(ref s_csSmall, nameof(EmbeddedSource) + "Small.cs");
+
+        public static KeyValuePair<byte[], byte[]> PortableDllAndPdb => new KeyValuePair<byte[], byte[]>(PortableDll, PortablePdb);
+        public static KeyValuePair<byte[], byte[]> DllAndPdb => new KeyValuePair<byte[], byte[]>(Dll, Pdb);
+    }
 }

src/Microsoft.DiaSymReader.PortablePdb.Tests/project.json

@@ -6,6 +6,7 @@
     "System.Runtime": "4.1.0",
     "System.Runtime.Extensions": "4.1.0",
     "System.Runtime.InteropServices": "4.1.0",
+    "System.IO.Compression": "4.1.0",
     "System.IO.FileSystem": "4.0.1",
     "Microsoft.DiaSymReader.Native": "1.5.0-beta1",
     "xunit": "2.1.0",

src/Microsoft.DiaSymReader.PortablePdb/SymDocument.cs

@@ -122,9 +122,8 @@ public int GetLanguageVendor(ref Guid vendor)
 
         public int GetSourceLength(out int length)
         {
-            // SymReader doesn't support embedded source.
-            length = 0;
-            return HResult.E_NOTIMPL;
+            length = GetEmbeddedSourceBlobReader().Length;
+            return HResult.S_OK;
         }
 
         public int GetSourceRange(
@@ -136,9 +135,45 @@ public int GetSourceRange(
             out int count,
             [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 4), Out]byte[] source)
         {
-            // SymReader doesn't support embedded source.
             count = 0;
-            return HResult.E_NOTIMPL;
+
+            // This function used to return E_NOTIMPL in all implementations. When filling in the
+            // implementation, it was deemed not very useful and impractical to actually decompress
+            // and decode text and interpret lines and columns here. A convention was adopted that 
+            // (0, 0, >= int.MaxValue, >= int.MaxValue) is a maximal range indicating that all
+            // source bytes should be returned. Anything else is rejected as invalid. This matches 
+            // the new native behavior.
+            if (startLine != 0 || 
+                startColumn != 0 ||
+                unchecked((uint)endLine) < int.MaxValue ||
+                unchecked((uint)endColumn) < int.MaxValue)
+            {
+                return HResult.E_INVALIDARG;
+            }
+
+            if (bufferLength < 0)
+            {
+                return HResult.E_INVALIDARG;
+            }
+
+            if (source == null && bufferLength > 0)
+            {
+                return HResult.E_INVALIDARG;
+            }
+
+            BlobReader reader = GetEmbeddedSourceBlobReader();
+            if (reader.Length == 0)
+            {
+                return HResult.S_FALSE;
+            }
+
+            // There is not currently a mechanism for reading from BlobReader to existing byte[]
+            // https://github.com/dotnet/corefx/issues/8004 tracks adding that API to corefx and
+            // this should be updated to use that when it's fixed. In the meantime, we are forced
+            // to make an extra copy here.
+            count = Math.Min(bufferLength, reader.Length);
+            Array.Copy(reader.ReadBytes(count), source, count);
+            return HResult.S_OK;
         }
 
         public int GetUrl(
@@ -152,9 +187,14 @@ public int GetUrl(
 
         public int HasEmbeddedSource(out bool value)
         {
-            // SymReader doesn't support embedded source.
-            value = false;
-            return HResult.E_NOTIMPL;
+            value = GetEmbeddedSourceBlobReader().Length > 0;
+            return HResult.S_OK;
+        }
+
+        private BlobReader GetEmbeddedSourceBlobReader()
+        {
+            BlobHandle blobHandle = SymReader.MetadataReader.GetCustomDebugInformation(Handle, MetadataUtilities.EmbeddedSourceId);
+            return blobHandle.IsNil ? default(BlobReader) : SymReader.MetadataReader.GetBlobReader(blobHandle);
         }
     }
 }

src/Microsoft.DiaSymReader.PortablePdb/Utilities/MetadataUtilities.cs

@@ -17,6 +17,7 @@ internal static class MetadataUtilities
         // Custom Attribute kinds:
         public static readonly Guid MethodSteppingInformationBlobId = new Guid("54FD2AC5-E925-401A-9C2A-F94F171072F8");
         public static readonly Guid VbDefaultNamespaceId = new Guid("58b2eab6-209f-4e4e-a22c-b2d0f910c782");
+        public static readonly Guid EmbeddedSourceId = new Guid("0E8A571B-6926-466E-B4AD-8AB04611F5FE");
 
         internal static int GetTypeDefOrRefOrSpecCodedIndex(EntityHandle typeHandle)
         {

src/Microsoft.DiaSymReader.PortablePdb/project.json

@@ -1,7 +1,7 @@
 {
   "dependencies": {
     "System.Collections.Immutable": "1.2.0",
-    "System.Reflection.Metadata": "1.4.1-beta-24322-03",
+    "System.Reflection.Metadata": "1.4.1-beta-24403-05",
     "Microsoft.DiaSymReader": "1.1.0-beta1-60625-03",
     "System.Collections": "4.0.11",
     "System.Diagnostics.Debug": "4.0.11",