Support reading embedded source

src/DeployCoreClrTestRuntime/project.json

@@ -17,6 +17,7 @@
     "System.Diagnostics.Tools": "4.0.1",
     "System.Dynamic.Runtime": "4.0.11",
     "System.Globalization": "4.0.11",
+    "System.IO.Compression": "4.1.0",
     "System.IO.FileSystem": "4.0.1",
     "System.IO.FileSystem.Primitives": "4.0.1",
     "System.IO.FileSystem.Watcher": "4.0.0",

src/Microsoft.DiaSymReader.PortablePdb.Tests/EmbeddedSourceTests.cs

@@ -0,0 +1,134 @@
+// Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+using Roslyn.Test.Utilities;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.IO.Compression;
+using System.Reflection.Metadata;
+using Xunit;
+
+namespace Microsoft.DiaSymReader.PortablePdb.UnitTests
+{
+    using static SymTestHelpers;
+
+    public class EmbeddedSourceTests
+    {
+        [Fact]
+        public void EmbeddedSource_Portable()
+        {
+            EmbeddedSource(TestResources.EmbeddedSource.PortableDllAndPdb);
+        }
+
+        [Fact]
+        public void EmbeddedSource_Native()
+        {
+            // TODO: We need an update to Microsoft.DiaSymReader.Native before we can read embedded 
+            // source from native PDBs. Uncomment this and remove the E_NOTIMPL tests below (deliberately
+            // put in place to make tests fail when we get update).
+            /*
+            EmbeddedSource(TestResources.EmbeddedSource.DllAndPdb);
+            */
+
+            var symReader = CreateSymReaderFromResource(TestResources.EmbeddedSource.DllAndPdb);
+
+            ISymUnmanagedDocument doc;
+            Assert.Equal(HResult.S_OK, symReader.GetDocument(@"C:\EmbeddedSource.cs", default(Guid), default(Guid), default(Guid), out doc));
+
+            bool hasEmbeddedSource;
+            Assert.Equal(HResult.E_NOTIMPL, doc.HasEmbeddedSource(out hasEmbeddedSource));
+
+            int length;
+            Assert.Equal(HResult.E_NOTIMPL, doc.GetSourceLength(out length));
+
+            byte[] source = null;
+            Assert.Equal(HResult.E_NOTIMPL, doc.GetSourceRange(0, 0, int.MaxValue, int.MaxValue, 0, out length, source));
+        }
+
+        private void EmbeddedSource(KeyValuePair<byte[], byte[]> dllAndPdb)
+        {
+            var symReader = CreateSymReaderFromResource(dllAndPdb);
+
+            foreach (string file in new[] { @"C:\EmbeddedSource.cs", @"C:\EmbeddedSourceSmall.cs"})
+            {
+                bool hasEmbeddedSource;
+                int length, bytesRead;
+                ISymUnmanagedDocument doc;
+                Assert.Equal(HResult.S_OK, symReader.GetDocument(file, default(Guid), default(Guid), default(Guid), out doc));
+                Assert.Equal(HResult.S_OK, doc.HasEmbeddedSource(out hasEmbeddedSource));
+                Assert.Equal(HResult.S_OK, doc.GetSourceLength(out length));
+                Assert.True(hasEmbeddedSource);
+
+                bool isCompressed;
+                var content = new byte[length];
+                Assert.Equal(HResult.S_OK, doc.GetSourceRange(0, 0, int.MaxValue, int.MaxValue, length, out bytesRead, content));
+                Assert.Equal(length, bytesRead);
+
+                if (content[0] == 0x1F && content[1] == 0x8B) // gzip signature
+                {
+                    isCompressed = true;
+                    Assert.Equal(@"C:\EmbeddedSource.cs", file);
+                }
+                else
+                {
+                    isCompressed = false;
+                    Assert.Equal(@"C:\EmbeddedSourceSmall.cs", file);
+                }
+
+                if (isCompressed)
+                {
+                    content = Decompress(content);
+                }
+
+                if (dllAndPdb.Equals(TestResources.EmbeddedSource.PortableDllAndPdb))
+                {
+                    CheckPortableFormatHeader(dllAndPdb.Value, file, isCompressed);
+                }
+
+                byte[] expectedContent = isCompressed ? TestResources.EmbeddedSource.CS : TestResources.EmbeddedSource.CSSmall;
+                AssertEx.Equal(expectedContent, content);
+            }
+        }
+
+        private byte[] Decompress(byte[] content)
+        {
+            using (var compressed = new MemoryStream(content))
+            using (var decompressor = new GZipStream(compressed, CompressionMode.Decompress))
+            using (var decompressed = new MemoryStream())
+            {
+                decompressor.CopyTo(decompressed);
+                return decompressed.ToArray();
+            }
+        }
+
+        // Portable PDB has a leading format indicator that is not exposed via the COM API. Check it for consistency.
+        private static void CheckPortableFormatHeader(byte[] portablePdb, string file, bool isCompressed)
+        {
+            using (var provider = MetadataReaderProvider.FromPortablePdbImage(ImmutableArray.Create(portablePdb)))
+            {
+                var portablePdbReader = provider.GetMetadataReader();
+                var document = GetDocumentHandle(portablePdbReader, file);
+                var customDebugInfo = portablePdbReader.GetCustomDebugInformation(document, MetadataUtilities.EmbeddedSourceId);
+                var blobReader = portablePdbReader.GetBlobReader(customDebugInfo);
+                Assert.Equal(isCompressed ? 1 : 0, blobReader.ReadUInt16());
+            }
+        }
+
+        private static DocumentHandle GetDocumentHandle(MetadataReader portablePdbReader, string file)
+        {
+            foreach (var handle in portablePdbReader.Documents)
+            {
+                var document = portablePdbReader.GetDocument(handle);
+                if (portablePdbReader.StringComparer.Equals(document.Name, file))
+                {
+                    return handle;
+                }
+            }
+
+            Assert.False(true, "Document not found.");
+            throw null;
+        }
+    }
+}
+

src/Microsoft.DiaSymReader.PortablePdb.Tests/Microsoft.DiaSymReader.PortablePdb.UnitTests.csproj

@@ -68,13 +68,32 @@
     <EmbeddedResource Include="Resources\MethodBoundaries.pdbx">
       <LogicalName>MethodBoundaries.pdbx</LogicalName>
     </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.cs">
+      <LogicalName>EmbeddedSource.cs</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSourceSmall.cs">
+      <LogicalName>EmbeddedSourceSmall.cs</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.dll">
+      <LogicalName>EmbeddedSource.dll</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.pdb">
+      <LogicalName>EmbeddedSource.pdb</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.dllx">
+      <LogicalName>EmbeddedSource.dllx</LogicalName>
+    </EmbeddedResource>
+    <EmbeddedResource Include="Resources\EmbeddedSource.pdbx">
+      <LogicalName>EmbeddedSource.pdbx</LogicalName>
+    </EmbeddedResource>
     <Content Include="Resources\Documents.cmd" />
     <Content Include="Resources\Scopes.cmd" />
     <Content Include="Resources\MethodBoundaries.cmd" />
     <Content Include="Resources\Async.cmd" />
     <Content Include="Resources\Scopes.cs" />
     <Content Include="Resources\Async.cs" />
     <Content Include="Resources\Documents.cs" />
+    <Compile Include="EmbeddedSourceTests.cs" />
     <Compile Include="MethodMapTests.cs" />
     <Compile Include="ResourceLoader.cs" />
     <Content Include="Resources\MethodBoundaries.cs" />
@@ -89,10 +108,11 @@
     <Compile Include="SymReaderTests.cs" />
   </ItemGroup>
   <ItemGroup>
-    <None Include="project.json" />    
+    <None Include="project.json" />
     <None Include="Microsoft.DiaSymReader.PortablePdb.UnitTests.xunit.runner.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <Content Include="Resources\EmbeddedSource.cmd" />
   </ItemGroup>
   <ImportGroup>
     <Import Project="..\..\build\Targets\Imports.targets" />

src/Microsoft.DiaSymReader.PortablePdb.Tests/Resources/EmbeddedSource.cmd

@@ -0,0 +1,13 @@
+@REM -- TODO: Compiled with WIP writer and preliminary command-line.
+@setlocal
+@set CSC=D:\src\roslyn\binaries\debug\csc.exe
+
+pushd
+
+%CSC% /target:library /debug:portable /optimize- /deterministic /pathmap:%~dp0=C:\ /embedsourceinpdb EmbeddedSource.cs EmbeddedSourceSmall.cs
+copy /y EmbeddedSource.pdb EmbeddedSource.pdbx
+copy /y EmbeddedSource.dll EmbeddedSource.dllx
+
+%CSC% /target:library /debug+ /optimize- /deterministic /pathmap:%~dp0=C:\ /embedsourceinpdb EmbeddedSource.cs EmbeddedSourceSmall.cs
+
+

src/Microsoft.DiaSymReader.PortablePdb.Tests/Resources/EmbeddedSource.cs

@@ -0,0 +1,19 @@
+// should be higher than compression threshold (200 chars)
+
+using System;
+
+namespace Test
+{
+    public static class SomeCode
+    {
+        public static int SomeMethod(int value)
+        {
+            if (value < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(value));
+            }
+
+            return checked(value + 42);
+        }
+    }
+}

src/Microsoft.DiaSymReader.PortablePdb.Tests/Resources/EmbeddedSourceSmall.cs

@@ -0,0 +1,5 @@
+// should be less than compression threshold (200 chars)
+public class Small
+{
+    public Small() {}
+}

src/Microsoft.DiaSymReader.PortablePdb.Tests/TestResources.cs

@@ -1,7 +1,5 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
 using System.Collections.Generic;
 
 namespace TestResources
@@ -70,4 +68,28 @@ public static class MethodBoundaries
         public static KeyValuePair<byte[], byte[]> PortableDllAndPdb => new KeyValuePair<byte[], byte[]>(PortableDll, PortablePdb);
         public static KeyValuePair<byte[], byte[]> DllAndPdb => new KeyValuePair<byte[], byte[]>(Dll, Pdb);
     }
+
+    public static class EmbeddedSource
+    {
+        private static byte[] s_portableDll;
+        public static byte[] PortableDll => ResourceLoader.GetOrCreateResource(ref s_portableDll, nameof(EmbeddedSource) + ".dllx");
+
+        private static byte[] s_portablePdb;
+        public static byte[] PortablePdb => ResourceLoader.GetOrCreateResource(ref s_portablePdb, nameof(EmbeddedSource) + ".pdbx");
+
+        private static byte[] s_dll;
+        public static byte[] Dll => ResourceLoader.GetOrCreateResource(ref s_dll, nameof(EmbeddedSource) + ".dll");
+
+        private static byte[] s_pdb;
+        public static byte[] Pdb => ResourceLoader.GetOrCreateResource(ref s_pdb, nameof(EmbeddedSource) + ".pdb");
+
+        private static byte[] s_cs;
+        public static byte[] CS => ResourceLoader.GetOrCreateResource(ref s_cs, nameof(EmbeddedSource) + ".cs");
+
+        private static byte[] s_csSmall;
+        public static byte[] CSSmall => ResourceLoader.GetOrCreateResource(ref s_csSmall, nameof(EmbeddedSource) + "Small.cs");
+
+        public static KeyValuePair<byte[], byte[]> PortableDllAndPdb => new KeyValuePair<byte[], byte[]>(PortableDll, PortablePdb);
+        public static KeyValuePair<byte[], byte[]> DllAndPdb => new KeyValuePair<byte[], byte[]>(Dll, Pdb);
+    }
 }

src/Microsoft.DiaSymReader.PortablePdb.Tests/project.json

@@ -6,6 +6,7 @@
     "System.Runtime": "4.1.0",
     "System.Runtime.Extensions": "4.1.0",
     "System.Runtime.InteropServices": "4.1.0",
+    "System.IO.Compression": "4.1.0",
     "System.IO.FileSystem": "4.0.1",
     "Microsoft.DiaSymReader.Native": "1.4.0",
     "xunit": "2.1.0",

src/Microsoft.DiaSymReader.PortablePdb/SymDocument.cs

@@ -122,9 +122,17 @@ public int GetLanguageVendor(ref Guid vendor)
 
         public int GetSourceLength(out int length)
         {
-            // SymReader doesn't support embedded source.
             length = 0;
-            return HResult.E_NOTIMPL;
+
+            BlobReader reader;
+            int hr = GetEmbeddedSourceBlobReader(out reader);
+            if (hr != HResult.S_OK)
+            {
+                return hr;
+            }
+
+            length = reader.RemainingBytes;
+            return HResult.S_OK;
         }
 
         public int GetSourceRange(
@@ -136,9 +144,50 @@ public int GetSourceRange(
             out int count,
             [In, MarshalAs(UnmanagedType.LPArray, SizeParamIndex = 4), Out]byte[] source)
         {
-            // SymReader doesn't support embedded source.
             count = 0;
-            return HResult.E_NOTIMPL;
+
+            // This function used to return E_NOTIMPL in all implementations. When filling in the
+            // implementation, it was deemed not very useful and impractical to actually decompress
+            // and decode text and interpret lines and columns here. A convention was adopted that 
+            // (0, 0, >= int.MaxValue, >= int.MaxValue) is a maximal range indicating that all
+            // source bytes should be returned. Anything else is rejected as invalid. This matches 
+            // the new native behavior.
+            if (startLine != 0 || 
+                startColumn != 0 ||
+                unchecked((uint)endLine) < int.MaxValue ||
+                unchecked((uint)endColumn) < int.MaxValue)
+            {
+                return HResult.E_INVALIDARG;
+            }
+
+            if (bufferLength < 0)
+            {
+                return HResult.E_INVALIDARG;
+            }
+
+            if (source == null && bufferLength > 0)
+            {
+                return HResult.E_INVALIDARG;
+            }
+
+            BlobReader reader;
+            int hr = GetEmbeddedSourceBlobReader(out reader);
+            if (hr != HResult.S_OK)
+            {
+                return hr;
+            }
+
+            count = Math.Min(bufferLength, reader.RemainingBytes);
+            if (count > 0)
+            {
+                // There is not currently a mechanism for reading from BlobReader to existing byte[]
+                // https://github.com/dotnet/corefx/issues/8004 tracks adding that API to corefx and
+                // this should be updated to use that when it's fixed. In the meantime, we are forced
+                // to make an extra copy here.
+                Array.Copy(reader.ReadBytes(count), source, count);
+            }
+
+            return HResult.S_OK;
         }
 
         public int GetUrl(
@@ -152,9 +201,40 @@ public int GetUrl(
 
         public int HasEmbeddedSource(out bool value)
         {
-            // SymReader doesn't support embedded source.
-            value = false;
-            return HResult.E_NOTIMPL;
+            int hr;
+            int length;
+            hr = GetSourceLength(out length);
+            value = length > 0;
+            return hr == HResult.S_FALSE ? HResult.S_OK : hr;
+        }
+
+        private int GetEmbeddedSourceBlobReader(out BlobReader reader)
+        {
+            BlobHandle blobHandle = SymReader.MetadataReader.GetCustomDebugInformation(Handle, MetadataUtilities.EmbeddedSourceId);
+            if (blobHandle.IsNil)
+            {
+                reader = default(BlobReader);
+                return HResult.S_FALSE;
+            }
+
+            reader = SymReader.MetadataReader.GetBlobReader(blobHandle);
+            if (reader.RemainingBytes < sizeof(ushort))
+            {
+                return HResult.E_UNEXPECTED;
+            }
+
+            var format = reader.ReadUInt16();
+            switch (format)
+            {
+                // Note that consumer of native PDB format does not have this leading format indicator.
+                // and native API consumer is expected to sniff for GZIP signature.
+                case 0: // Uncompressed
+                case 1: // GZIP compressed
+                    return HResult.S_OK;
+
+                default:
+                    return HResult.E_UNEXPECTED;
+            }
         }
     }
 }