diff --git a/build/Targets/Dependencies.props b/build/Targets/Dependencies.props index c6281ad11b042..2687bdedb8af5 100644 --- a/build/Targets/Dependencies.props +++ b/build/Targets/Dependencies.props @@ -23,6 +23,7 @@ 4.0.11 4.0.11 4.1.0 + 4.1.0 4.0.1 4.0.0 4.0.1 diff --git a/src/Compilers/CSharp/Portable/CSharpResources.Designer.cs b/src/Compilers/CSharp/Portable/CSharpResources.Designer.cs index c76af4e0a8a92..2f3222e9dc310 100644 --- a/src/Compilers/CSharp/Portable/CSharpResources.Designer.cs +++ b/src/Compilers/CSharp/Portable/CSharpResources.Designer.cs @@ -2320,6 +2320,15 @@ internal static string ERR_CannotDeconstructDynamic { } } + /// + /// Looks up a localized string similar to /embed switch is only supported when emitting Portable PDB (/debug:portable or /debug:embedded).. + /// + internal static string ERR_CannotEmbedWithoutPdb { + get { + return ResourceManager.GetString("ERR_CannotEmbedWithoutPdb", resourceCulture); + } + } + /// /// Looks up a localized string similar to Cannot pass null for friend assembly name. /// diff --git a/src/Compilers/CSharp/Portable/CSharpResources.resx b/src/Compilers/CSharp/Portable/CSharpResources.resx index 202eeed7b53eb..c1938bd9be965 100644 --- a/src/Compilers/CSharp/Portable/CSharpResources.resx +++ b/src/Compilers/CSharp/Portable/CSharpResources.resx @@ -4407,6 +4407,8 @@ To remove the warning, you can use /reference instead (set the Embed Interop Typ /additionalfile:<file list> Additional files that don't directly affect code generation but may be used by analyzers for producing errors or warnings. + /embed Embed all source files in the PDB. + /embed:<file list> Embed specfic files the PDB - RESOURCES - /win32res:<file> Specify a Win32 resource file (.res) @@ -4936,4 +4938,7 @@ To remove the warning, you can use /reference instead (set the Embed Interop Typ /sourcelink switch is only supported when emitting Portable PDB (/debug:portable or /debug:embedded must be specified). - + + /embed switch is only supported when emitting Portable PDB (/debug:portable or /debug:embedded). + + \ No newline at end of file diff --git a/src/Compilers/CSharp/Portable/CommandLine/CSharpCommandLineParser.cs b/src/Compilers/CSharp/Portable/CommandLine/CSharpCommandLineParser.cs index 9045c62394752..fd7397d37ded0 100644 --- a/src/Compilers/CSharp/Portable/CommandLine/CSharpCommandLineParser.cs +++ b/src/Compilers/CSharp/Portable/CommandLine/CSharpCommandLineParser.cs @@ -85,7 +85,9 @@ internal sealed override CommandLineArguments CommonParse(IEnumerable ar List managedResources = new List(); List sourceFiles = new List(); List additionalFiles = new List(); + List embeddedFiles = new List(); bool sourceFilesSpecified = false; + bool embedAllSourceFiles = false; bool resourcesOrModulesSpecified = false; Encoding codepage = null; var checksumAlgorithm = SourceHashAlgorithm.Sha1; @@ -1097,7 +1099,17 @@ internal sealed override CommandLineArguments CommonParse(IEnumerable ar continue; } - additionalFiles.AddRange(ParseAdditionalFileArgument(value, baseDirectory, diagnostics)); + additionalFiles.AddRange(ParseSeparatedFileArgument(value, baseDirectory, diagnostics)); + continue; + + case "embed": + if (string.IsNullOrEmpty(value)) + { + embedAllSourceFiles = true; + continue; + } + + embeddedFiles.AddRange(ParseSeparatedFileArgument(value, baseDirectory, diagnostics)); continue; } } @@ -1161,11 +1173,26 @@ internal sealed override CommandLineArguments CommonParse(IEnumerable ar if (sourceLink != null) { - if (!emitPdb || debugInformationFormat != DebugInformationFormat.PortablePdb && debugInformationFormat != DebugInformationFormat.Embedded) + if (!emitPdb || !debugInformationFormat.IsPortable()) { AddDiagnostic(diagnostics, ErrorCode.ERR_SourceLinkRequiresPortablePdb); } } + + if (embedAllSourceFiles) + { + embeddedFiles.AddRange(sourceFiles); + } + + if (embeddedFiles.Count > 0) + { + // Restricted to portable PDBs for now, but the IsPortable condition should be removed + // and the error message adjusted accordingly when native PDB support is added. + if (!emitPdb || !debugInformationFormat.IsPortable()) + { + AddDiagnostic(diagnostics, ErrorCode.ERR_CannotEmbedWithoutPdb); + } + } var parsedFeatures = CompilerOptionParseUtilities.ParseFeatures(features); @@ -1273,7 +1300,8 @@ internal sealed override CommandLineArguments CommonParse(IEnumerable ar PrintFullPaths = printFullPaths, ShouldIncludeErrorEndLocation = errorEndLocation, PreferredUILang = preferredUILang, - ReportAnalyzer = reportAnalyzer + ReportAnalyzer = reportAnalyzer, + EmbeddedFiles = embeddedFiles.AsImmutable() }; } diff --git a/src/Compilers/CSharp/Portable/CommandLine/CSharpCompiler.cs b/src/Compilers/CSharp/Portable/CommandLine/CSharpCompiler.cs index 7aff32b0e9140..8be675c3fd436 100644 --- a/src/Compilers/CSharp/Portable/CommandLine/CSharpCompiler.cs +++ b/src/Compilers/CSharp/Portable/CommandLine/CSharpCompiler.cs @@ -8,9 +8,11 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Collections; using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; +using Microsoft.CodeAnalysis.CSharp.Syntax; namespace Microsoft.CodeAnalysis.CSharp { @@ -151,7 +153,7 @@ private SyntaxTree ParseFile( out string normalizedFilePath) { var fileReadDiagnostics = new List(); - var content = ReadFileContent(file, fileReadDiagnostics, out normalizedFilePath); + var content = TryReadFileContent(file, fileReadDiagnostics, out normalizedFilePath); if (content == null) { @@ -277,5 +279,37 @@ protected override ImmutableArray ResolveAnalyzersFromArgume { return Arguments.ResolveAnalyzersFromArguments(LanguageNames.CSharp, diagnostics, messageProvider, AssemblyLoader); } + + protected override void ResolveEmbeddedFilesFromExternalSourceDirectives( + SyntaxTree tree, + SourceReferenceResolver resolver, + OrderedSet embeddedFiles, + IList diagnostics) + { + foreach (LineDirectiveTriviaSyntax directive in tree.GetRoot().GetDirectives( + d => d.IsActive && !d.HasErrors && d.Kind() == SyntaxKind.LineDirectiveTrivia)) + { + string path = (string)directive.File.Value; + if (path == null) + { + continue; + } + + string resolvedPath = resolver.ResolveReference(path, tree.FilePath); + if (resolvedPath == null) + { + diagnostics.Add( + MessageProvider.CreateDiagnostic( + (int)ErrorCode.ERR_NoSourceFile, + directive.File.GetLocation(), + path, + CSharpResources.CouldNotFindFile)); + + continue; + } + + embeddedFiles.Add(resolvedPath); + } + } } } diff --git a/src/Compilers/CSharp/Portable/Compilation/CSharpCompilation.cs b/src/Compilers/CSharp/Portable/Compilation/CSharpCompilation.cs index d5a7b769305a4..e8173b4ea4132 100644 --- a/src/Compilers/CSharp/Portable/Compilation/CSharpCompilation.cs +++ b/src/Compilers/CSharp/Portable/Compilation/CSharpCompilation.cs @@ -2222,6 +2222,7 @@ internal override CommonPEModuleBuilder CreateModuleBuilder( EmitOptions emitOptions, IMethodSymbol debugEntryPoint, Stream sourceLinkStream, + IEnumerable embeddedTexts, IEnumerable manifestResources, CompilationTestData testData, DiagnosticBag diagnostics, @@ -2269,6 +2270,11 @@ internal override CommonPEModuleBuilder CreateModuleBuilder( moduleBeingBuilt.SourceLinkStreamOpt = sourceLinkStream; + if (embeddedTexts != null) + { + moduleBeingBuilt.EmbeddedTexts = embeddedTexts; + } + // testData is only passed when running tests. if (testData != null) { @@ -2314,7 +2320,8 @@ internal override bool CompileMethods( } else { - if ((emittingPdb || emitOptions.EmitDynamicAnalysisData) && !StartSourceChecksumCalculation(moduleBeingBuilt.DebugDocumentsBuilder, diagnostics)) + if ((emittingPdb || emitOptions.EmitDynamicAnalysisData) && + !StartSourceChecksumCalculation(moduleBeingBuilt.DebugDocumentsBuilder, moduleBeingBuilt.EmbeddedTexts, diagnostics)) { return false; } @@ -2491,11 +2498,11 @@ internal override void AddDebugSourceDocumentsForChecksumDirectives( continue; } - var checksumAndAlgorithm = existingDoc.ChecksumAndAlgorithm; - if (ChecksumMatches(checksumText, checksumAndAlgorithm.Item1)) + var sourceInfo = existingDoc.GetSourceInfo(); + if (ChecksumMatches(checksumText, sourceInfo.Checksum)) { var guid = Guid.Parse(checksumDirective.Guid.ValueText); - if (guid == checksumAndAlgorithm.Item2) + if (guid == sourceInfo.ChecksumAlgorithmId) { // all parts match, nothing to do continue; diff --git a/src/Compilers/CSharp/Portable/Errors/ErrorCode.cs b/src/Compilers/CSharp/Portable/Errors/ErrorCode.cs index 0888c4e311ca1..00ed27157ec75 100644 --- a/src/Compilers/CSharp/Portable/Errors/ErrorCode.cs +++ b/src/Compilers/CSharp/Portable/Errors/ErrorCode.cs @@ -1061,7 +1061,8 @@ internal enum ErrorCode ERR_InvalidDebugInformationFormat = 2042, ERR_LegacyObjectIdSyntax = 2043, ERR_SourceLinkRequiresPortablePdb = 2044, - // unused 2045-2999 + ERR_CannotEmbedWithoutPdb = 2045, + // unused 2046-2999 WRN_CLS_NoVarArgs = 3000, WRN_CLS_BadArgType = 3001, // Requires SymbolDistinguisher. WRN_CLS_BadReturnType = 3002, diff --git a/src/Compilers/CSharp/Portable/Errors/MessageProvider.cs b/src/Compilers/CSharp/Portable/Errors/MessageProvider.cs index b4cc46c0722d4..c9b9f98fef664 100644 --- a/src/Compilers/CSharp/Portable/Errors/MessageProvider.cs +++ b/src/Compilers/CSharp/Portable/Errors/MessageProvider.cs @@ -95,6 +95,11 @@ public override Diagnostic CreateDiagnostic(int code, Location location, params return new CSDiagnostic(info, location); } + public override Diagnostic CreateDiagnostic(DiagnosticInfo info) + { + return new CSDiagnostic(info, Location.None); + } + public override string GetErrorDisplayString(ISymbol symbol) { // show extra info for assembly if possible such as version, public key token etc. diff --git a/src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs b/src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs index f0a84ca0d6b47..01f203c0694da 100644 --- a/src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs +++ b/src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs @@ -1690,6 +1690,168 @@ public void SourceLink_EndToEnd_Portable() CleanupAllGeneratedFiles(src.Path); } + [Fact] + public void Embed() + { + var parsedArgs = DefaultParse(new[] { "a.cs "}, _baseDirectory); + parsedArgs.Errors.Verify(); + Assert.Empty(parsedArgs.EmbeddedFiles); + + parsedArgs = DefaultParse(new[] { "/embed", "/debug:portable", "a.cs", "b.cs", "c.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(); + AssertEx.Equal(parsedArgs.SourceFiles, parsedArgs.EmbeddedFiles); + AssertEx.Equal( + new[] { "a.cs", "b.cs", "c.cs" }.Select(f => Path.Combine(_baseDirectory, f)), + parsedArgs.EmbeddedFiles.Select(f => f.Path)); + + parsedArgs = DefaultParse(new[] { "/embed:a.cs", "/embed:b.cs", "/debug:embedded", "a.cs", "b.cs", "c.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(); + AssertEx.Equal( + new[] { "a.cs", "b.cs" }.Select(f => Path.Combine(_baseDirectory, f)), + parsedArgs.EmbeddedFiles.Select(f => f.Path)); + + parsedArgs = DefaultParse(new[] { "/embed:a.cs;b.cs", "/debug:portable", "a.cs", "b.cs", "c.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(); + AssertEx.Equal( + new[] { "a.cs", "b.cs" }.Select(f => Path.Combine(_baseDirectory, f)), + parsedArgs.EmbeddedFiles.Select(f => f.Path)); + + parsedArgs = DefaultParse(new[] { "/embed:a.txt", "/embed", "/debug:portable", "a.cs", "b.cs", "c.cs" }, _baseDirectory); + parsedArgs.Errors.Verify();; + AssertEx.Equal( + new[] { "a.txt", "a.cs", "b.cs", "c.cs" }.Select(f => Path.Combine(_baseDirectory, f)), + parsedArgs.EmbeddedFiles.Select(f => f.Path)); + + parsedArgs = DefaultParse(new[] { "/embed", "a.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(Diagnostic(ErrorCode.ERR_CannotEmbedWithoutPdb)); + + parsedArgs = DefaultParse(new[] { "/embed:a.txt", "a.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(Diagnostic(ErrorCode.ERR_CannotEmbedWithoutPdb)); + + parsedArgs = DefaultParse(new[] { "/embed", "/debug-", "a.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(Diagnostic(ErrorCode.ERR_CannotEmbedWithoutPdb)); + + parsedArgs = DefaultParse(new[] { "/embed:a.txt", "/debug-", "a.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(Diagnostic(ErrorCode.ERR_CannotEmbedWithoutPdb)); + + // These should fail when native PDB support is added. + parsedArgs = DefaultParse(new[] { "/embed", "/debug:full", "a.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(Diagnostic(ErrorCode.ERR_CannotEmbedWithoutPdb)); + + parsedArgs = DefaultParse(new[] { "/embed", "/debug:full", "a.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(Diagnostic(ErrorCode.ERR_CannotEmbedWithoutPdb)); + + parsedArgs = DefaultParse(new[] { "/embed", "/debug:pdbonly", "a.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(Diagnostic(ErrorCode.ERR_CannotEmbedWithoutPdb)); + + parsedArgs = DefaultParse(new[] { "/embed", "/debug+", "a.cs" }, _baseDirectory); + parsedArgs.Errors.Verify(Diagnostic(ErrorCode.ERR_CannotEmbedWithoutPdb)); + } + + [Theory] + [InlineData("/debug:portable", "/embed", new[] {"embed.cs", "embed2.cs", "embed.xyz" })] + [InlineData("/debug:portable", "/embed:embed.cs", new[] {"embed.cs", "embed.xyz" })] + [InlineData("/debug:portable", "/embed:embed2.cs", new[] {"embed2.cs" })] + [InlineData("/debug:portable", "/embed:embed.xyz", new[] {"embed.xyz" })] + [InlineData("/debug:embedded", "/embed", new[] { "embed.cs", "embed2.cs", "embed.xyz" })] + [InlineData("/debug:embedded", "/embed:embed.cs", new[] { "embed.cs", "embed.xyz" })] + [InlineData("/debug:embedded", "/embed:embed2.cs", new[] { "embed2.cs" })] + [InlineData("/debug:embedded", "/embed:embed.xyz", new[] {"embed.xyz" })] + public void Embed_EndToEnd(string debugSwitch, string embedSwitch, string[] expectedEmbedded) + { + // embed.cs: large enough to compress, has #line directives + const string embed_cs = +@"/////////////////////////////////////////////////////////////////////////////// +class Program { + static void Main() { +#line 1 ""embed.xyz"" + System.Console.WriteLine(""Hello, World""); + +#line 3 + System.Console.WriteLine(""Goodbye, World""); + } +} +///////////////////////////////////////////////////////////////////////////////"; + + // embed2.cs: small enough to not compress, no sequence points + const string embed2_cs = +@"class C +{ +}"; + // target of #line + const string embed_xyz = +@"print Hello, World + +print Goodbye, World"; + + Assert.True(embed_cs.Length >= EmbeddedText.CompressionThreshold); + Assert.True(embed2_cs.Length < EmbeddedText.CompressionThreshold); + + var dir = Temp.CreateDirectory(); + var src = dir.CreateFile("embed.cs"); + var src2 = dir.CreateFile("embed2.cs"); + var txt = dir.CreateFile("embed.xyz"); + + src.WriteAllText(embed_cs); + src2.WriteAllText(embed2_cs); + txt.WriteAllText(embed_xyz); + + var expectedEmbeddedMap = new Dictionary(); + if (expectedEmbedded.Contains("embed.cs")) + { + expectedEmbeddedMap.Add(src.Path, embed_cs); + } + + if (expectedEmbedded.Contains("embed2.cs")) + { + expectedEmbeddedMap.Add(src2.Path, embed2_cs); + } + + if (expectedEmbedded.Contains("embed.xyz")) + { + expectedEmbeddedMap.Add(txt.Path, embed_xyz); + } + + var output = new StringWriter(CultureInfo.InvariantCulture); + var csc = new MockCSharpCompiler(null, dir.Path, new[] { "/nologo", debugSwitch, embedSwitch, "embed.cs", "embed2.cs" }); + int exitCode = csc.Run(output); + Assert.Equal("", output.ToString().Trim()); + Assert.Equal(0, exitCode); + + bool embedded = debugSwitch == "/debug:embedded"; + using (var peReader = new PEReader(File.OpenRead(Path.Combine(dir.Path, "embed.exe")))) + { + var entry = peReader.ReadDebugDirectory().SingleOrDefault(e => e.Type == DebugDirectoryEntryType.EmbeddedPortablePdb); + Assert.Equal(embedded, entry.DataSize > 0); + + using (var mdProvider = embedded ? + peReader.ReadEmbeddedPortablePdbDebugDirectoryData(entry) : + MetadataReaderProvider.FromPortablePdbStream(File.OpenRead(Path.Combine(dir.Path, "embed.pdb")))) + { + var mdReader = mdProvider.GetMetadataReader(); + + foreach (var handle in mdReader.Documents) + { + var doc = mdReader.GetDocument(handle); + var docPath = mdReader.GetString(doc.Name); + + SourceText embeddedSource = mdReader.GetEmbeddedSource(handle); + if (embeddedSource == null) + { + continue; + } + + Assert.True(embeddedSource.Encoding is UTF8Encoding && embeddedSource.Encoding.GetPreamble().Length == 0); + Assert.Equal(expectedEmbeddedMap[docPath], embeddedSource.ToString()); + Assert.True(expectedEmbeddedMap.Remove(docPath)); + } + } + } + + Assert.Empty(expectedEmbeddedMap); + CleanupAllGeneratedFiles(src.Path); + } + [Fact] public void Optimize() { diff --git a/src/Compilers/CSharp/Test/Emit/PDB/PDBTests.cs b/src/Compilers/CSharp/Test/Emit/PDB/PDBTests.cs index 7b49f5cb5ce5a..4ab62a5fd50c8 100644 --- a/src/Compilers/CSharp/Test/Emit/PDB/PDBTests.cs +++ b/src/Compilers/CSharp/Test/Emit/PDB/PDBTests.cs @@ -141,6 +141,7 @@ public void SymWriterErrors() options: null, debugEntryPoint: null, sourceLinkStream: null, + embeddedTexts: null, testData: new CompilationTestData() { SymWriterFactory = () => new MockSymUnmanagedWriter() }); result.Diagnostics.Verify( @@ -173,6 +174,7 @@ public void SymWriterErrors2() options: null, debugEntryPoint: null, sourceLinkStream: null, + embeddedTexts: null, testData: new CompilationTestData() { SymWriterFactory = () => new object() }); result.Diagnostics.Verify( @@ -205,6 +207,7 @@ public void SymWriterErrors3() options: null, debugEntryPoint: null, sourceLinkStream: null, + embeddedTexts: null, testData: new CompilationTestData() { SymWriterFactory = () => new MockSymUnmanagedWriter() }); result.Diagnostics.Verify( diff --git a/src/Compilers/CSharp/Test/Emit/PDB/PortablePdbTests.cs b/src/Compilers/CSharp/Test/Emit/PDB/PortablePdbTests.cs index 86908bd7fbd3c..dbcbc9fbca6a8 100644 --- a/src/Compilers/CSharp/Test/Emit/PDB/PortablePdbTests.cs +++ b/src/Compilers/CSharp/Test/Emit/PDB/PortablePdbTests.cs @@ -362,5 +362,93 @@ public static void Main() // error CS0041: Unexpected error writing debug information -- 'Error!' Diagnostic(ErrorCode.FTL_DebugEmitFailure).WithArguments("Error!").WithLocation(1, 1)); } + + [Fact] + public void EmbeddedSource() + { + string source = @" +using System; + +class C +{ + public static void Main() + { + Console.WriteLine(); + } +} +"; + var tree = Parse(source, "f:/build/foo.cs"); + var c = CreateCompilationWithMscorlib(tree, options: TestOptions.DebugDll); + + var pdbStream = new MemoryStream(); + c.EmitToArray( + EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), + pdbStream: pdbStream, + embeddedTexts: new[] { EmbeddedText.FromSource(tree.FilePath, tree.GetText()) }); + pdbStream.Position = 0; + + using (var provider = MetadataReaderProvider.FromPortablePdbStream(pdbStream)) + { + var pdbReader = provider.GetMetadataReader(); + + var embeddedSource = + (from documentHandle in pdbReader.Documents + let document = pdbReader.GetDocument(documentHandle) + select new + { + FilePath = pdbReader.GetString(document.Name), + Text = pdbReader.GetEmbeddedSource(documentHandle) + }).Single(); + + Assert.Equal(embeddedSource.FilePath, "f:/build/foo.cs"); + Assert.Equal(source, embeddedSource.Text.ToString()); + } + } + + [Fact] + public void EmbeddedSource_InEmbeddedPdb() + { + string source = @" +using System; + +class C +{ + public static void Main() + { + Console.WriteLine(); + } +} +"; + var tree = Parse(source, "f:/build/foo.cs"); + var c = CreateCompilationWithMscorlib(tree, options: TestOptions.DebugDll); + + var pdbStream = new MemoryStream(); + var peBlob = c.EmitToArray( + EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Embedded), + embeddedTexts: new[] { EmbeddedText.FromSource(tree.FilePath, tree.GetText()) }); + pdbStream.Position = 0; + + using (var peReader = new PEReader(peBlob)) + { + var embeddedEntry = peReader.ReadDebugDirectory().Single(e => e.Type == DebugDirectoryEntryType.EmbeddedPortablePdb); + + using (var embeddedMetadataProvider = peReader.ReadEmbeddedPortablePdbDebugDirectoryData(embeddedEntry)) + { + var pdbReader = embeddedMetadataProvider.GetMetadataReader(); + + var embeddedSource = + (from documentHandle in pdbReader.Documents + let document = pdbReader.GetDocument(documentHandle) + select new + { + FilePath = pdbReader.GetString(document.Name), + Text = pdbReader.GetEmbeddedSource(documentHandle) + }).Single(); + + Assert.Equal(embeddedSource.FilePath, "f:/build/foo.cs"); + Assert.Equal(source, embeddedSource.Text.ToString()); + } + } + } } } \ No newline at end of file diff --git a/src/Compilers/CSharp/Test/Symbol/Compilation/CompilationAPITests.cs b/src/Compilers/CSharp/Test/Symbol/Compilation/CompilationAPITests.cs index e06b2fd979c64..b905ba2cee218 100644 --- a/src/Compilers/CSharp/Test/Symbol/Compilation/CompilationAPITests.cs +++ b/src/Compilers/CSharp/Test/Symbol/Compilation/CompilationAPITests.cs @@ -274,34 +274,46 @@ public void Emit_BadArgs() { var comp = CSharpCompilation.Create("Compilation", options: TestOptions.ReleaseDll); - Assert.Throws(() => comp.Emit(peStream: null)); - Assert.Throws(() => comp.Emit(peStream: new TestStream(canRead: true, canWrite: false, canSeek: true))); - Assert.Throws(() => comp.Emit(peStream: new MemoryStream(), pdbStream: new TestStream(canRead: true, canWrite: false, canSeek: true))); - Assert.Throws(() => comp.Emit(peStream: new MemoryStream(), pdbStream: new MemoryStream(), options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Embedded))); + Assert.Throws("peStream", () => comp.Emit(peStream: null)); + Assert.Throws("peStream", () => comp.Emit(peStream: new TestStream(canRead: true, canWrite: false, canSeek: true))); + Assert.Throws("pdbStream", () => comp.Emit(peStream: new MemoryStream(), pdbStream: new TestStream(canRead: true, canWrite: false, canSeek: true))); + Assert.Throws("pdbStream", () => comp.Emit(peStream: new MemoryStream(), pdbStream: new MemoryStream(), options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Embedded))); - Assert.Throws(() => comp.Emit( + Assert.Throws("sourceLinkStream", () => comp.Emit( peStream: new MemoryStream(), pdbStream: new MemoryStream(), options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), sourceLinkStream: new TestStream(canRead: false, canWrite: true, canSeek: true))); - Assert.Throws(() => comp.Emit( + Assert.Throws("sourceLinkStream", () => comp.Emit( peStream: new MemoryStream(), pdbStream: new MemoryStream(), options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Pdb), sourceLinkStream: new MemoryStream())); - Assert.Throws(() => comp.Emit( + Assert.Throws("sourceLinkStream", () => comp.Emit( peStream: new MemoryStream(), pdbStream: null, options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), sourceLinkStream: new MemoryStream())); - Assert.Throws(() => comp.Emit( + Assert.Throws("embeddedTexts", () => comp.Emit( + peStream: new MemoryStream(), + pdbStream: new MemoryStream(), + options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Pdb), + embeddedTexts: new[] { EmbeddedText.FromStream("_", new MemoryStream()) })); + + Assert.Throws("embeddedTexts", () => comp.Emit( + peStream: new MemoryStream(), + pdbStream: null, + options: EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), + embeddedTexts: new[] { EmbeddedText.FromStream("_", new MemoryStream()) })); + + Assert.Throws("win32Resources", () => comp.Emit( peStream: new MemoryStream(), win32Resources: new TestStream(canRead: true, canWrite: false, canSeek: false))); - Assert.Throws(() => comp.Emit( + Assert.Throws("win32Resources", () => comp.Emit( peStream: new MemoryStream(), win32Resources: new TestStream(canRead: false, canWrite: false, canSeek: true))); diff --git a/src/Compilers/Core/CodeAnalysisTest/CodeAnalysisTest.csproj b/src/Compilers/Core/CodeAnalysisTest/CodeAnalysisTest.csproj index 5ed5ef4d4c8ea..f3afd1571fa46 100644 --- a/src/Compilers/Core/CodeAnalysisTest/CodeAnalysisTest.csproj +++ b/src/Compilers/Core/CodeAnalysisTest/CodeAnalysisTest.csproj @@ -21,6 +21,7 @@ FusionAssemblyIdentity.cs + diff --git a/src/Compilers/Core/CodeAnalysisTest/EmbeddedTextTests.cs b/src/Compilers/Core/CodeAnalysisTest/EmbeddedTextTests.cs new file mode 100644 index 0000000000000..9370aad90dd2e --- /dev/null +++ b/src/Compilers/Core/CodeAnalysisTest/EmbeddedTextTests.cs @@ -0,0 +1,265 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.CodeAnalysis.Text; +using Roslyn.Utilities; +using Xunit; +using System.Text; +using System.IO.Compression; +using Roslyn.Test.Utilities; +using System.Linq; +using System.Collections.Immutable; +using System.Reflection; + +namespace Microsoft.CodeAnalysis.UnitTests +{ + public class EmbeddedTextTests + { + [Fact] + public void FromBytes_ArgumentErrors() + { + Assert.Throws("filePath", () => EmbeddedText.FromBytes(null, default(ArraySegment))); + Assert.Throws("filePath", () => EmbeddedText.FromBytes("", default(ArraySegment))); + Assert.Throws("bytes", () => EmbeddedText.FromBytes("path", default(ArraySegment))); + Assert.Throws("checksumAlgorithm", () => EmbeddedText.FromBytes("path", new ArraySegment(new byte[0], 0, 0), SourceHashAlgorithm.None)); + } + + [Fact] + public void FromSource_ArgumentErrors() + { + Assert.Throws("filePath", () => EmbeddedText.FromSource(null, null)); + Assert.Throws("filePath", () => EmbeddedText.FromSource("", null)); + Assert.Throws("text", () => EmbeddedText.FromSource("path", null)); + + // no encoding + Assert.Throws("text", () => EmbeddedText.FromSource("path", SourceText.From("source"))); + + // embedding not allowed + Assert.Throws("text", () => EmbeddedText.FromSource("path", SourceText.From(new byte[0], 0, Encoding.UTF8, canBeEmbedded: false))); + Assert.Throws("text", () => EmbeddedText.FromSource("path", SourceText.From(new MemoryStream(new byte[0]), Encoding.UTF8, canBeEmbedded: false))); + } + + [Fact] + public void FromStream_ArgumentErrors() + { + Assert.Throws("filePath", () => EmbeddedText.FromStream(null, null)); + Assert.Throws("filePath", () => EmbeddedText.FromStream("", null)); + Assert.Throws("stream", () => EmbeddedText.FromStream("path", null)); + Assert.Throws("stream", () => EmbeddedText.FromStream("path", new CannotReadStream())); + Assert.Throws("stream", () => EmbeddedText.FromStream("path", new CannotSeekStream())); + Assert.Throws("checksumAlgorithm", () => EmbeddedText.FromStream("path", new MemoryStream(), SourceHashAlgorithm.None)); + } + + [Fact] + public void FromStream_IOErrors() + { + Assert.Throws(() => EmbeddedText.FromStream("path", new HugeStream())); + Assert.Throws(() => EmbeddedText.FromStream("path", new TruncatingStream(10))); + Assert.Throws(() => EmbeddedText.FromStream("path", new TruncatingStream(1000))); + + // Should be Assert.Throws, but impeded by https://github.com/dotnet/roslyn/issues/12926 + var ex = Assert.Throws(() => EmbeddedText.FromStream("path", new ReadFailsStream())); + Assert.IsType(ex.InnerException); + } + + private const string SmallSource = @"class P {}"; + private const string LargeSource = @" +////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////// +class Program +{ + static void Main() {} +} +////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////// +"; + + [Fact] + public void FromBytes_Empty() + { + var text = EmbeddedText.FromBytes("pathToEmpty", new ArraySegment(new byte[0], 0, 0), SourceHashAlgorithm.Sha1); + Assert.Equal("pathToEmpty", text.FilePath); + Assert.Equal(text.ChecksumAlgorithm, SourceHashAlgorithm.Sha1); + AssertEx.Equal(SourceText.CalculateChecksum(new byte[0], 0, 0, SourceHashAlgorithm.Sha1), text.Checksum); + AssertEx.Equal(new byte[] { 0, 0, 0, 0 }, text.Blob); + } + + [Fact] + public void FromStream_Empty() + { + var text = EmbeddedText.FromStream("pathToEmpty", new MemoryStream(new byte[0]), SourceHashAlgorithm.Sha1); + var checksum = SourceText.CalculateChecksum(new byte[0], 0, 0, SourceHashAlgorithm.Sha1); + + Assert.Equal("pathToEmpty", text.FilePath); + Assert.Equal(text.ChecksumAlgorithm, SourceHashAlgorithm.Sha1); + AssertEx.Equal(checksum, text.Checksum); + AssertEx.Equal(new byte[] { 0, 0, 0, 0 }, text.Blob); + } + + [Fact] + public void FromSource_Empty() + { + var source = SourceText.From("", new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), SourceHashAlgorithm.Sha1); + var text = EmbeddedText.FromSource("pathToEmpty", source); + var checksum = SourceText.CalculateChecksum(new byte[0], 0, 0, SourceHashAlgorithm.Sha1); + + Assert.Equal("pathToEmpty", text.FilePath); + Assert.Equal(SourceHashAlgorithm.Sha1, text.ChecksumAlgorithm); + AssertEx.Equal(checksum, text.Checksum); + AssertEx.Equal(new byte[] { 0, 0, 0, 0 }, text.Blob); + } + + [Fact] + public void FromBytes_Small() + { + var bytes = Encoding.UTF8.GetBytes(SmallSource); + var checksum = SourceText.CalculateChecksum(bytes, 0, bytes.Length, SourceHashAlgorithm.Sha1); + var text = EmbeddedText.FromBytes("pathToSmall", new ArraySegment(bytes, 0, bytes.Length)); + + Assert.Equal("pathToSmall", text.FilePath); + Assert.Equal(text.ChecksumAlgorithm, SourceHashAlgorithm.Sha1); + AssertEx.Equal(checksum, text.Checksum); + AssertEx.Equal(new byte[] { 0, 0, 0, 0 }, text.Blob.Take(4)); + AssertEx.Equal(bytes, text.Blob.Skip(4)); + } + + [Fact] + public void FromBytes_SmallSpan() + { + var bytes = Encoding.UTF8.GetBytes(SmallSource); + var padddedBytes = new byte[] { 0 }.Concat(bytes).Concat(new byte[] { 0 }).ToArray(); + var checksum = SourceText.CalculateChecksum(bytes, 0, bytes.Length, SourceHashAlgorithm.Sha1); + var text = EmbeddedText.FromBytes("pathToSmall", new ArraySegment(padddedBytes, 1, bytes.Length)); + + Assert.Equal("pathToSmall", text.FilePath); + AssertEx.Equal(checksum, text.Checksum); + Assert.Equal(SourceHashAlgorithm.Sha1, text.ChecksumAlgorithm); + AssertEx.Equal(new byte[] { 0, 0, 0, 0 }, text.Blob.Take(4)); + AssertEx.Equal(bytes, text.Blob.Skip(4)); + } + + [Fact] + public void FromSource_Small() + { + var source = SourceText.From(SmallSource, Encoding.UTF8, SourceHashAlgorithm.Sha1); + var text = EmbeddedText.FromSource("pathToSmall", source); + + Assert.Equal("pathToSmall", text.FilePath); + Assert.Equal(SourceHashAlgorithm.Sha1, text.ChecksumAlgorithm); + AssertEx.Equal(source.GetChecksum(), text.Checksum); + AssertEx.Equal(new byte[] { 0, 0, 0, 0 }, text.Blob.Take(4)); + AssertEx.Equal(Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes(SmallSource)), text.Blob.Skip(4)); + } + + [Fact] + public void FromBytes_Large() + { + var bytes = Encoding.Unicode.GetBytes(LargeSource); + var checksum = SourceText.CalculateChecksum(bytes, 0, bytes.Length, SourceHashAlgorithm.Sha256); + var text = EmbeddedText.FromBytes("pathToLarge", new ArraySegment(bytes, 0, bytes.Length), SourceHashAlgorithm.Sha256); + + Assert.Equal("pathToLarge", text.FilePath); + Assert.Equal(SourceHashAlgorithm.Sha256, text.ChecksumAlgorithm); + AssertEx.Equal(checksum, text.Checksum); + AssertEx.Equal(BitConverter.GetBytes(bytes.Length), text.Blob.Take(4)); + AssertEx.Equal(bytes, Decompress(text.Blob.Skip(4))); + } + + [Fact] + public void FromBytes_LargeSpan() + { + var bytes = Encoding.Unicode.GetBytes(LargeSource); + var paddedBytes = new byte[] { 0 }.Concat(bytes).Concat(new byte[] { 0 }).ToArray(); + var checksum = SourceText.CalculateChecksum(bytes, 0, bytes.Length, SourceHashAlgorithm.Sha256); + var text = EmbeddedText.FromBytes("pathToLarge", new ArraySegment(paddedBytes, 1, bytes.Length), SourceHashAlgorithm.Sha256); + + Assert.Equal("pathToLarge", text.FilePath); + AssertEx.Equal(checksum, text.Checksum); + Assert.Equal(SourceHashAlgorithm.Sha256, text.ChecksumAlgorithm); + AssertEx.Equal(BitConverter.GetBytes(bytes.Length), text.Blob.Take(4)); + AssertEx.Equal(bytes, Decompress(text.Blob.Skip(4))); + } + + [Fact] + public void FromSource_Large() + { + var source = SourceText.From(LargeSource, Encoding.Unicode, SourceHashAlgorithm.Sha256); + var text = EmbeddedText.FromSource("pathToLarge", source); + + Assert.Equal("pathToLarge", text.FilePath); + Assert.Equal(SourceHashAlgorithm.Sha256, text.ChecksumAlgorithm); + AssertEx.Equal(source.GetChecksum(), text.Checksum); + AssertEx.Equal(BitConverter.GetBytes(Encoding.Unicode.GetPreamble().Length + LargeSource.Length * sizeof(char)), text.Blob.Take(4)); + AssertEx.Equal(Encoding.Unicode.GetPreamble().Concat(Encoding.Unicode.GetBytes(LargeSource)), Decompress(text.Blob.Skip(4))); + } + + [Fact] + public void FromSource_Precomputed() + { + byte[] bytes = Encoding.ASCII.GetBytes(LargeSource); + bytes[0] = 0xFF; // invalid ASCII, should be reflected in checksum, blob. + + foreach (bool useStream in new[] { true, false }) + { + var source = useStream ? + SourceText.From(new MemoryStream(bytes), Encoding.ASCII, SourceHashAlgorithm.Sha1, canBeEmbedded: true) : + SourceText.From(bytes, bytes.Length, Encoding.ASCII, SourceHashAlgorithm.Sha1, canBeEmbedded: true); + + var text = EmbeddedText.FromSource("pathToPrecomputed", source); + Assert.Equal("pathToPrecomputed", text.FilePath); + Assert.Equal(SourceHashAlgorithm.Sha1, text.ChecksumAlgorithm); + AssertEx.Equal(SourceText.CalculateChecksum(bytes, 0, bytes.Length, SourceHashAlgorithm.Sha1), source.GetChecksum()); + AssertEx.Equal(source.GetChecksum(), text.Checksum); + AssertEx.Equal(BitConverter.GetBytes(bytes.Length), text.Blob.Take(4)); + AssertEx.Equal(bytes, Decompress(text.Blob.Skip(4))); + } + } + + private byte[] Decompress(IEnumerable bytes) + { + var destination = new MemoryStream(); + using (var source = new DeflateStream(new MemoryStream(bytes.ToArray()), CompressionMode.Decompress)) + { + source.CopyTo(destination); + } + + return destination.ToArray(); + } + + private sealed class CannotReadStream : MemoryStream + { + public override bool CanRead => false; + } + + private sealed class CannotSeekStream : MemoryStream + { + public override bool CanSeek => false; + } + + private sealed class HugeStream : MemoryStream + { + public override long Length => (long)int.MaxValue + 1; + } + + private sealed class TruncatingStream : MemoryStream + { + public TruncatingStream(long length) + { + Length = length; + } + + public override long Length { get; } + public override int Read(byte[] buffer, int offset, int count) => 0; + } + + private sealed class ReadFailsStream : MemoryStream + { + public override int Read(byte[] buffer, int offset, int count) + { + throw new IOException(); + } + } + } +} diff --git a/src/Compilers/Core/CodeAnalysisTest/Text/LargeTextTests.cs b/src/Compilers/Core/CodeAnalysisTest/Text/LargeTextTests.cs index 77e9c25f1c784..93f8afed4736d 100644 --- a/src/Compilers/Core/CodeAnalysisTest/Text/LargeTextTests.cs +++ b/src/Compilers/Core/CodeAnalysisTest/Text/LargeTextTests.cs @@ -26,7 +26,7 @@ private static SourceText CreateSourceText(string s, Encoding encoding = null) private static SourceText CreateSourceText(Stream stream, Encoding encoding = null) { - return LargeText.Decode(stream, encoding ?? Encoding.UTF8, SourceHashAlgorithm.Sha1, throwIfBinaryDetected: true); + return LargeText.Decode(stream, encoding ?? Encoding.UTF8, SourceHashAlgorithm.Sha1, throwIfBinaryDetected: true, canBeEmbedded: false); } private const string HelloWorld = "Hello, world!"; diff --git a/src/Compilers/Core/CodeAnalysisTest/Text/SourceTextTests.cs b/src/Compilers/Core/CodeAnalysisTest/Text/SourceTextTests.cs index 7a9596c14b6e0..f00e3239fa93d 100644 --- a/src/Compilers/Core/CodeAnalysisTest/Text/SourceTextTests.cs +++ b/src/Compilers/Core/CodeAnalysisTest/Text/SourceTextTests.cs @@ -124,12 +124,12 @@ public void ChecksumAndBOM() VerifyChecksum(SourceText.From(streamBOM, encodingBOM, checksumAlgorigthm), checksumBOM); // LargeText from stream no BOM. Checksum should ignore explicit encoding. - VerifyChecksum(LargeText.Decode(streamNoBOM, encodingNoBOM, checksumAlgorigthm, throwIfBinaryDetected: false), checksumNoBOM); - VerifyChecksum(LargeText.Decode(streamNoBOM, encodingBOM, checksumAlgorigthm, throwIfBinaryDetected: false), checksumNoBOM); + VerifyChecksum(LargeText.Decode(streamNoBOM, encodingNoBOM, checksumAlgorigthm, throwIfBinaryDetected: false, canBeEmbedded: false), checksumNoBOM); + VerifyChecksum(LargeText.Decode(streamNoBOM, encodingBOM, checksumAlgorigthm, throwIfBinaryDetected: false, canBeEmbedded: false), checksumNoBOM); // LargeText from stream with BOM. Checksum should include BOM. - VerifyChecksum(LargeText.Decode(streamBOM, encodingNoBOM, checksumAlgorigthm, throwIfBinaryDetected: false), checksumBOM); - VerifyChecksum(LargeText.Decode(streamBOM, encodingBOM, checksumAlgorigthm, throwIfBinaryDetected: false), checksumBOM); + VerifyChecksum(LargeText.Decode(streamBOM, encodingNoBOM, checksumAlgorigthm, throwIfBinaryDetected: false, canBeEmbedded: false), checksumBOM); + VerifyChecksum(LargeText.Decode(streamBOM, encodingBOM, checksumAlgorigthm, throwIfBinaryDetected: false, canBeEmbedded: false), checksumBOM); // LargeText from writer no BOM. Checksum includes BOM // from explicit encoding. This is inconsistent with the diff --git a/src/Compilers/Core/CodeAnalysisTest/Text/TextChangeTests.cs b/src/Compilers/Core/CodeAnalysisTest/Text/TextChangeTests.cs index 0c1dcf425419d..c64a099db7229 100644 --- a/src/Compilers/Core/CodeAnalysisTest/Text/TextChangeTests.cs +++ b/src/Compilers/Core/CodeAnalysisTest/Text/TextChangeTests.cs @@ -558,7 +558,7 @@ public void TestLargeTextWriterReusesLargeChunks() private SourceText CreateLargeText(params char[][] chunks) { - return new LargeText(ImmutableArray.Create(chunks), Encoding.UTF8, default(ImmutableArray), SourceHashAlgorithm.Sha256); + return new LargeText(ImmutableArray.Create(chunks), Encoding.UTF8, default(ImmutableArray), SourceHashAlgorithm.Sha256, default(ImmutableArray)); } private ImmutableArray GetChunks(SourceText text) diff --git a/src/Compilers/Core/MSBuildTask/Shared/ManagedCompiler.cs b/src/Compilers/Core/MSBuildTask/Shared/ManagedCompiler.cs index 7a0e9ecae01cd..d5436f7da834f 100644 --- a/src/Compilers/Core/MSBuildTask/Shared/ManagedCompiler.cs +++ b/src/Compilers/Core/MSBuildTask/Shared/ManagedCompiler.cs @@ -51,6 +51,12 @@ public ITaskItem[] AdditionalFiles get { return (ITaskItem[])_store[nameof(AdditionalFiles)]; } } + public ITaskItem[] EmbeddedFiles + { + set { _store[nameof(EmbeddedFiles)] = value; } + get { return (ITaskItem[])_store[nameof(EmbeddedFiles)]; } + } + public ITaskItem[] Analyzers { set { _store[nameof(Analyzers)] = value; } @@ -724,6 +730,7 @@ internal void AddResponseFileCommandsForSwitchesSinceInitialReleaseThatAreNeeded commandLine.AppendSwitchIfNotNull("/sourcelink:", SourceLink); AddFeatures(commandLine, Features); + AddEmbeddedFilesToCommandLine(commandLine); } /// @@ -765,16 +772,26 @@ internal static void AddAnalyzersToCommandLine(CommandLineBuilderExtension comma /// private void AddAdditionalFilesToCommandLine(CommandLineBuilderExtension commandLine) { - // If there were no additional files passed in, don't add any /additionalfile: switches - // on the command-line. - if (AdditionalFiles == null) + if (AdditionalFiles != null) { - return; + foreach (ITaskItem additionalFile in AdditionalFiles) + { + commandLine.AppendSwitchIfNotNull("/additionalfile:", additionalFile.ItemSpec); + } } + } - foreach (ITaskItem additionalFile in AdditionalFiles) + /// + /// Adds a "/embed:" switch to the command line for each pdb embedded file. + /// + private void AddEmbeddedFilesToCommandLine(CommandLineBuilderExtension commandLine) + { + if (EmbeddedFiles != null) { - commandLine.AppendSwitchIfNotNull("/additionalfile:", additionalFile.ItemSpec); + foreach (ITaskItem embeddedFile in EmbeddedFiles) + { + commandLine.AppendSwitchIfNotNull("/embed:", embeddedFile.ItemSpec); + } } } diff --git a/src/Compilers/Core/MSBuildTask/Shared/Microsoft.CSharp.Core.targets b/src/Compilers/Core/MSBuildTask/Shared/Microsoft.CSharp.Core.targets index 8293f097fc711..14d100b6532fb 100644 --- a/src/Compilers/Core/MSBuildTask/Shared/Microsoft.CSharp.Core.targets +++ b/src/Compilers/Core/MSBuildTask/Shared/Microsoft.CSharp.Core.targets @@ -14,7 +14,8 @@ $(Win32Manifest); @(CustomAdditionalCompileInputs); $(ResolvedCodeAnalysisRuleSet); - @(AdditionalFiles)" + @(AdditionalFiles); + @(EmbeddedFiles)" Outputs="@(DocFileItem); @(IntermediateAssembly); @(_DebugSymbolsIntermediatePath); @@ -84,6 +85,7 @@ DelaySign="$(DelaySign)" DisabledWarnings="$(NoWarn)" DocumentationFile="@(DocFileItem)" + EmbeddedFiles="@(EmbeddedFiles)" EmitDebugInformation="$(DebugSymbols)" EnvironmentVariables="$(CscEnvironment)" ErrorEndLocation="$(ErrorEndLocation)" diff --git a/src/Compilers/Core/Portable/AdditionalTextFile.cs b/src/Compilers/Core/Portable/AdditionalTextFile.cs index 68f9234f445bd..7579d39d91878 100644 --- a/src/Compilers/Core/Portable/AdditionalTextFile.cs +++ b/src/Compilers/Core/Portable/AdditionalTextFile.cs @@ -49,7 +49,7 @@ public AdditionalTextFile(CommandLineSourceFile sourceFile, CommonCompiler compi if (_text == null) { var diagnostics = new List(); - _text = _compiler.ReadFileContent(_sourceFile, diagnostics); + _text = _compiler.TryReadFileContent(_sourceFile, diagnostics); _diagnostics = diagnostics; } } diff --git a/src/Compilers/Core/Portable/CodeAnalysis.csproj b/src/Compilers/Core/Portable/CodeAnalysis.csproj index 60517675029f9..324c47776bd75 100644 --- a/src/Compilers/Core/Portable/CodeAnalysis.csproj +++ b/src/Compilers/Core/Portable/CodeAnalysis.csproj @@ -52,8 +52,12 @@ + + + + diff --git a/src/Compilers/Core/Portable/CodeAnalysisResources.Designer.cs b/src/Compilers/Core/Portable/CodeAnalysisResources.Designer.cs index 9c98489a3d3e9..7c0597a421149 100644 --- a/src/Compilers/Core/Portable/CodeAnalysisResources.Designer.cs +++ b/src/Compilers/Core/Portable/CodeAnalysisResources.Designer.cs @@ -387,6 +387,15 @@ internal static string DuplicateAnalyzerInstances { } } + /// + /// Looks up a localized string similar to Embedded texts are only supported when emitting Portable PDB.. + /// + internal static string EmbeddedTextsRequirePortablePdb { + get { + return ResourceManager.GetString("EmbeddedTextsRequirePortablePdb", resourceCulture); + } + } + /// /// Looks up a localized string similar to A key in the pathMap is empty.. /// @@ -1108,6 +1117,15 @@ internal static string SourceLinkRequiresPortablePdb { } } + /// + /// Looks up a localized string similar to SourceText cannot be embedded. Provide encoding or canBeEmbedded=true at construction.. + /// + internal static string SourceTextCannotBeEmbedded { + get { + return ResourceManager.GetString("SourceTextCannotBeEmbedded", resourceCulture); + } + } + /// /// Looks up a localized string similar to The span does not include the end of a line.. /// @@ -1135,6 +1153,15 @@ internal static string StartMustNotBeNegative { } } + /// + /// Looks up a localized string similar to Stream is too long.. + /// + internal static string StreamIsTooLong { + get { + return ResourceManager.GetString("StreamIsTooLong", resourceCulture); + } + } + /// /// Looks up a localized string similar to Stream must be readable.. /// diff --git a/src/Compilers/Core/Portable/CodeAnalysisResources.resx b/src/Compilers/Core/Portable/CodeAnalysisResources.resx index bb43692878be1..5551defb203f4 100644 --- a/src/Compilers/Core/Portable/CodeAnalysisResources.resx +++ b/src/Compilers/Core/Portable/CodeAnalysisResources.resx @@ -529,7 +529,7 @@ If names are used in a tuple type, then there must be the same number of names and elements. - The compilation references multiple assemblies whose versions only differ in auto-generated build and/or revision numbers. + The compilation references multiple assemblies whose versions only differ in auto-generated build and/or revision numbers. The underlying type for a tuple must be tuple-compatible. @@ -543,4 +543,13 @@ Unrecognized resource file format. - + + SourceText cannot be embedded. Provide encoding or canBeEmbedded=true at construction. + + + Stream is too long. + + + Embedded texts are only supported when emitting Portable PDB. + + \ No newline at end of file diff --git a/src/Compilers/Core/Portable/CommandLine/CommonCommandLineArguments.cs b/src/Compilers/Core/Portable/CommandLine/CommonCommandLineArguments.cs index 78dcdf64d8f74..c1da4d9998c4b 100644 --- a/src/Compilers/Core/Portable/CommandLine/CommonCommandLineArguments.cs +++ b/src/Compilers/Core/Portable/CommandLine/CommonCommandLineArguments.cs @@ -142,6 +142,11 @@ public abstract class CommandLineArguments /// public ImmutableArray AdditionalFiles { get; internal set; } + /// + /// A set of files to embed in the PDB. + /// + public ImmutableArray EmbeddedFiles { get; internal set; } + /// /// Report additional information related to analyzers, such as analyzer execution time. /// diff --git a/src/Compilers/Core/Portable/CommandLine/CommonCommandLineParser.cs b/src/Compilers/Core/Portable/CommandLine/CommonCommandLineParser.cs index c256631cd34c0..6a91f6b8ef756 100644 --- a/src/Compilers/Core/Portable/CommandLine/CommonCommandLineParser.cs +++ b/src/Compilers/Core/Portable/CommandLine/CommonCommandLineParser.cs @@ -972,7 +972,7 @@ internal IEnumerable ParseFileArgument(string arg, string } } - internal IEnumerable ParseAdditionalFileArgument(string value, string baseDirectory, IList errors) + internal IEnumerable ParseSeparatedFileArgument(string value, string baseDirectory, IList errors) { foreach (string path in ParseSeparatedPaths(value).Where((path) => !string.IsNullOrWhiteSpace(path))) { diff --git a/src/Compilers/Core/Portable/CommandLine/CommonCompiler.cs b/src/Compilers/Core/Portable/CommandLine/CommonCompiler.cs index dbb2ab3c64657..0467172e74b11 100644 --- a/src/Compilers/Core/Portable/CommandLine/CommonCompiler.cs +++ b/src/Compilers/Core/Portable/CommandLine/CommonCompiler.cs @@ -11,8 +11,10 @@ using System.Text; using System.Threading; using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Emit; using Microsoft.CodeAnalysis.Text; using Roslyn.Utilities; +using Microsoft.CodeAnalysis.Collections; namespace Microsoft.CodeAnalysis { @@ -30,6 +32,13 @@ internal abstract partial class CommonCompiler public CommandLineArguments Arguments { get; } public IAnalyzerAssemblyLoader AssemblyLoader { get; private set; } public abstract DiagnosticFormatter DiagnosticFormatter { get; } + + /// + /// The set of source file paths that are in the set of embedded paths. + /// This is used to prevent reading source files that are embedded twice. + /// + public IReadOnlySet EmbeddedSourcePaths { get; } + private readonly HashSet _reportedDiagnostics = new HashSet(); public abstract Compilation CreateCompilation(TextWriter consoleOutput, TouchedFileLogger touchedFilesLogger, ErrorLogger errorLoggerOpt); @@ -56,6 +65,7 @@ public CommonCompiler(CommandLineParser parser, string responseFile, string[] ar this.Arguments = parser.Parse(allArgs, baseDirectory, sdkDirectoryOpt, additionalReferenceDirectories); this.MessageProvider = parser.MessageProvider; this.AssemblyLoader = assemblyLoader; + this.EmbeddedSourcePaths = GetEmbedddedSourcePaths(Arguments); if (Arguments.ParseOptions.Features.ContainsKey("debug-determinism")) { @@ -130,10 +140,10 @@ internal List ResolveMetadataReferences( /// Source file information. /// Storage for diagnostics. /// File content or null on failure. - internal SourceText ReadFileContent(CommandLineSourceFile file, IList diagnostics) + internal SourceText TryReadFileContent(CommandLineSourceFile file, IList diagnostics) { string discarded; - return ReadFileContent(file, diagnostics, out discarded); + return TryReadFileContent(file, diagnostics, out discarded); } /// @@ -143,18 +153,15 @@ internal SourceText ReadFileContent(CommandLineSourceFile file, IListStorage for diagnostics. /// If given opens successfully, set to normalized absolute path of the file, null otherwise. /// File content or null on failure. - internal SourceText ReadFileContent(CommandLineSourceFile file, IList diagnostics, out string normalizedFilePath) + internal SourceText TryReadFileContent(CommandLineSourceFile file, IList diagnostics, out string normalizedFilePath) { var filePath = file.Path; try { - // PERF: Using a very small buffer size for the FileStream opens up an optimization within EncodedStringText where - // we read the entire FileStream into a byte array in one shot. For files that are actually smaller than the buffer - // size, FileStream.Read still allocates the internal buffer. - using (var data = PortableShim.FileStream.Create(filePath, PortableShim.FileMode.Open, PortableShim.FileAccess.Read, PortableShim.FileShare.ReadWrite, bufferSize: 1, options: PortableShim.FileOptions.None)) + using (var data = OpenFileForReadWithSmallBufferOptimization(filePath)) { normalizedFilePath = (string)PortableShim.FileStream.Name.GetValue(data); - return EncodedStringText.Create(data, Arguments.Encoding, Arguments.ChecksumAlgorithm); + return EncodedStringText.Create(data, Arguments.Encoding, Arguments.ChecksumAlgorithm, canBeEmbedded: EmbeddedSourcePaths.Contains(file.Path)); } } catch (Exception e) @@ -165,6 +172,127 @@ internal SourceText ReadFileContent(CommandLineSourceFile file, IList diagnostics) + { + try + { + using (var stream = OpenFileForReadWithSmallBufferOptimization(filePath)) + { + const int LargeObjectHeapLimit = 80 * 1024; + if (stream.Length < LargeObjectHeapLimit) + { + byte[] buffer = EncodedStringText.TryGetByteArrayFromStream(stream); + if (buffer != null) + { + return EmbeddedText.FromBytes(filePath, new ArraySegment(buffer, 0, (int)stream.Length), Arguments.ChecksumAlgorithm); + } + } + + return EmbeddedText.FromStream(filePath, stream, Arguments.ChecksumAlgorithm); + } + } + catch (Exception e) + { + diagnostics.Add(MessageProvider.CreateDiagnostic(ToFileReadDiagnostics(this.MessageProvider, e, filePath))); + return null; + } + } + + private ImmutableArray AcquireEmbeddedTexts(Compilation compilation, IList diagnostics) + { + if (Arguments.EmbeddedFiles.IsEmpty) + { + return ImmutableArray.Empty; + } + + var embeddedTreeMap = new Dictionary(Arguments.EmbeddedFiles.Length); + var embeddedFileOrderedSet = new OrderedSet(Arguments.EmbeddedFiles.Select(e => e.Path)); + + foreach (var tree in compilation.SyntaxTrees) + { + // Skip trees that will not have their text embedded. + if (!EmbeddedSourcePaths.Contains(tree.FilePath)) + { + continue; + } + + // Skip trees with duplicated paths. (VB allows this and "first tree wins" is same as PDB emit policy.) + if (embeddedTreeMap.ContainsKey(tree.FilePath)) + { + continue; + } + + // map embedded file path to corresponding source tree + embeddedTreeMap.Add(tree.FilePath, tree); + + // also embed the text of any #line directive targets of embedded tree + ResolveEmbeddedFilesFromExternalSourceDirectives(tree, compilation.Options.SourceReferenceResolver, embeddedFileOrderedSet, diagnostics); + } + + var embeddedTextBuilder = ImmutableArray.CreateBuilder(embeddedFileOrderedSet.Count); + foreach (var path in embeddedFileOrderedSet) + { + SyntaxTree tree; + EmbeddedText text; + + if (embeddedTreeMap.TryGetValue(path, out tree)) + { + text = EmbeddedText.FromSource(path, tree.GetText()); + Debug.Assert(text != null); + } + else + { + text = TryReadEmbeddedFileContent(path, diagnostics); + Debug.Assert(text != null || diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error)); + } + + // We can safely add nulls because result will be ignored if any error is produced. + // This allows the MoveToImmutable to work below in all cases. + embeddedTextBuilder.Add(text); + } + + return embeddedTextBuilder.MoveToImmutable(); + } + + + protected abstract void ResolveEmbeddedFilesFromExternalSourceDirectives( + SyntaxTree tree, + SourceReferenceResolver resolver, + OrderedSet embeddedFiles, + IList diagnostics); + + private static IReadOnlySet GetEmbedddedSourcePaths(CommandLineArguments arguments) + { + if (arguments.EmbeddedFiles.IsEmpty) + { + return SpecializedCollections.EmptyReadOnlySet(); + } + + // Note that we require an exact match between source and embedded file paths (case-sensitive + // and without normalization). If two files are the same but spelled differently, they will + // be handled as separate files, meaning the embedding pass will read the content a second + // time. This can also lead to more than one document entry in the PDB for the same document + // if the PDB document de-duping policy in emit (normalize + case-sensitive in C#, + // normalize + case-insensitive in VB) is not enough to converge them. + var set = new HashSet(arguments.EmbeddedFiles.Select(f => f.Path)); + set.IntersectWith(arguments.SourceFiles.Select(f => f.Path)); + return SpecializedCollections.StronglyTypedReadOnlySet(set); + } + internal static DiagnosticInfo ToFileReadDiagnostics(CommonMessageProvider messageProvider, Exception e, string filePath) { DiagnosticInfo diagnosticInfo; @@ -358,14 +486,21 @@ internal int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, Cancella return Failed; } - var diagnostics = new List(); - ImmutableArray analyzers = ResolveAnalyzersFromArguments(diagnostics, MessageProvider); - var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnostics, MessageProvider, touchedFilesLogger); - if (ReportErrors(diagnostics, consoleOutput, errorLogger)) + var diagnosticInfos = new List(); + ImmutableArray analyzers = ResolveAnalyzersFromArguments(diagnosticInfos, MessageProvider); + var additionalTextFiles = ResolveAdditionalFilesFromArguments(diagnosticInfos, MessageProvider, touchedFilesLogger); + if (ReportErrors(diagnosticInfos, consoleOutput, errorLogger)) { return Failed; } + var diagnostics = new List(); + ImmutableArray embeddedTexts = AcquireEmbeddedTexts(compilation, diagnostics); + if (ReportErrors(diagnostics, consoleOutput, errorLogger)) + { + return Failed; + } + bool reportAnalyzer = false; CancellationTokenSource analyzerCts = null; AnalyzerManager analyzerManager = null; @@ -439,6 +574,7 @@ internal int RunCore(TextWriter consoleOutput, ErrorLogger errorLogger, Cancella emitOptions, debugEntryPoint: null, sourceLinkStream: sourceLinkStreamOpt, + embeddedTexts: embeddedTexts, testData: null, cancellationToken: cancellationToken); diff --git a/src/Compilers/Core/Portable/Compilation/Compilation.cs b/src/Compilers/Core/Portable/Compilation/Compilation.cs index 4e6a18b238838..155e7dec2ad16 100644 --- a/src/Compilers/Core/Portable/Compilation/Compilation.cs +++ b/src/Compilers/Core/Portable/Compilation/Compilation.cs @@ -1511,6 +1511,7 @@ internal abstract CommonPEModuleBuilder CreateModuleBuilder( EmitOptions emitOptions, IMethodSymbol debugEntryPoint, Stream sourceLinkStream, + IEnumerable embeddedTexts, IEnumerable manifestResources, CompilationTestData testData, DiagnosticBag diagnostics, @@ -1527,7 +1528,7 @@ internal abstract bool CompileMethods( Predicate filterOpt, CancellationToken cancellationToken); - internal bool StartSourceChecksumCalculation(DebugDocumentsBuilder documentsBuilder, DiagnosticBag diagnostics) + internal bool StartSourceChecksumCalculation(DebugDocumentsBuilder documentsBuilder, IEnumerable embeddedTexts, DiagnosticBag diagnostics) { // Check that all syntax trees are debuggable: bool allTreesDebuggable = true; @@ -1545,6 +1546,32 @@ internal bool StartSourceChecksumCalculation(DebugDocumentsBuilder documentsBuil return false; } + // Add debug documents for all embedded text first. This ensures that embedding + // takes priority over the syntax tree pass, which will not embed. + if (!embeddedTexts.IsEmpty()) + { + var embeddedDocuments = ArrayBuilder.GetInstance(); + + foreach (var text in embeddedTexts) + { + Debug.Assert(!string.IsNullOrEmpty(text.FilePath)); + string normalizedPath = documentsBuilder.NormalizeDebugDocumentPath(text.FilePath, basePath: null); + var existingDoc = documentsBuilder.TryGetDebugDocumentForNormalizedPath(normalizedPath); + if (existingDoc == null) + { + var document = new Cci.DebugSourceDocument( + normalizedPath, + DebugSourceDocumentLanguageId, + () => text.GetDebugSourceInfo()); + + documentsBuilder.AddDebugDocument(document); + embeddedDocuments.Add(document); + } + } + + documentsBuilder.EmbeddedDocuments = embeddedDocuments.ToImmutableAndFree(); + } + // Add debug documents for all trees with distinct paths. foreach (var tree in SyntaxTrees) { @@ -1559,7 +1586,7 @@ internal bool StartSourceChecksumCalculation(DebugDocumentsBuilder documentsBuil documentsBuilder.AddDebugDocument(new Cci.DebugSourceDocument( normalizedPath, DebugSourceDocumentLanguageId, - () => tree.GetChecksumAndAlgorithm())); + () => tree.GetDebugSourceInfo())); } } } @@ -1646,6 +1673,7 @@ internal void EnsureAnonymousTypeTemplates(CancellationToken cancellationToken) debugEntryPoint: null, manifestResources: null, sourceLinkStream: null, + embeddedTexts: null, testData: null, diagnostics: discardedDiagnostics, cancellationToken: cancellationToken); @@ -1689,7 +1717,9 @@ public EmitResult Emit( win32Resources, manifestResources, options, - null, + default(IMethodSymbol), + default(Stream), + default(IEnumerable), cancellationToken); } @@ -1714,6 +1744,7 @@ public EmitResult Emit( options, debugEntryPoint, default(Stream), + default(IEnumerable), cancellationToken); } @@ -1746,6 +1777,10 @@ public EmitResult Emit( /// Stream containing information linking the compilation to a source control. /// Only supported when emitting Portable PDBs. /// + /// + /// Texts to embed in the PDB. + /// Only supported when emitting Portable PDBs. + /// /// To cancel the emit process. public EmitResult Emit( Stream peStream, @@ -1756,6 +1791,7 @@ public EmitResult Emit( EmitOptions options = null, IMethodSymbol debugEntryPoint = null, Stream sourceLinkStream = null, + IEnumerable embeddedTexts = null, CancellationToken cancellationToken = default(CancellationToken)) { if (peStream == null) @@ -1803,6 +1839,15 @@ public EmitResult Emit( throw new ArgumentException(CodeAnalysisResources.StreamMustSupportRead, nameof(sourceLinkStream)); } } + if (embeddedTexts != null && !embeddedTexts.IsEmpty()) + { + if (options == null || + options.DebugInformationFormat == DebugInformationFormat.Pdb || + options.DebugInformationFormat == DebugInformationFormat.PortablePdb && pdbStream == null) + { + throw new ArgumentException(CodeAnalysisResources.EmbeddedTextsRequirePortablePdb, nameof(embeddedTexts)); + } + } return Emit( peStream, @@ -1813,6 +1858,7 @@ public EmitResult Emit( options, debugEntryPoint, sourceLinkStream, + embeddedTexts, testData: null, cancellationToken: cancellationToken); } @@ -1830,6 +1876,7 @@ internal EmitResult Emit( EmitOptions options, IMethodSymbol debugEntryPoint, Stream sourceLinkStream, + IEnumerable embeddedTexts, CompilationTestData testData, CancellationToken cancellationToken) { @@ -1844,6 +1891,7 @@ internal EmitResult Emit( options, debugEntryPoint, sourceLinkStream, + embeddedTexts, testData, cancellationToken); @@ -1992,6 +2040,7 @@ internal CommonPEModuleBuilder CheckOptionsAndCreateModuleBuilder( EmitOptions options, IMethodSymbol debugEntryPoint, Stream sourceLinkStream, + IEnumerable embeddedTexts, CompilationTestData testData, CancellationToken cancellationToken) { @@ -2039,6 +2088,7 @@ internal CommonPEModuleBuilder CheckOptionsAndCreateModuleBuilder( options, debugEntryPoint, sourceLinkStream, + embeddedTexts, manifestResources, testData, diagnostics, diff --git a/src/Compilers/Core/Portable/Diagnostic/CommonMessageProvider.cs b/src/Compilers/Core/Portable/Diagnostic/CommonMessageProvider.cs index e013e996dfcbe..9ddf5976323e7 100644 --- a/src/Compilers/Core/Portable/Diagnostic/CommonMessageProvider.cs +++ b/src/Compilers/Core/Portable/Diagnostic/CommonMessageProvider.cs @@ -74,6 +74,11 @@ public Diagnostic CreateDiagnostic(int code, Location location) return CreateDiagnostic(code, location, SpecializedCollections.EmptyObjects); } + /// + /// Create a simple language specific diagnostic with no location for given info. + /// + public abstract Diagnostic CreateDiagnostic(DiagnosticInfo info); + /// /// Create a simple language specific diagnostic for given error code. /// diff --git a/src/Compilers/Core/Portable/EmbeddedText.cs b/src/Compilers/Core/Portable/EmbeddedText.cs new file mode 100644 index 0000000000000..b148d601eb281 --- /dev/null +++ b/src/Compilers/Core/Portable/EmbeddedText.cs @@ -0,0 +1,379 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.CodeAnalysis.Text; +using Roslyn.Utilities; +using System; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Reflection.Metadata; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.CodeAnalysis +{ + /// + /// Represents text to be embedded in a PDB. + /// + public sealed class EmbeddedText + { + /// + /// The maximum number of bytes in to write out uncompressed. + /// + /// This prevents wasting resources on compressing tiny files with little to negative gain + /// in PDB file size. + /// + /// Chosen as the point at which we start to see > 10% blob size reduction using all + /// current source files in corefx and roslyn as sample data. + /// + internal const int CompressionThreshold = 200; + + /// + /// The path to the file to embed. + /// + /// See remarks of + public string FilePath { get; } + + /// + /// Hash algorithm to use to calculate checksum of the text that's saved to PDB. + /// + public SourceHashAlgorithm ChecksumAlgorithm { get; } + + /// + /// The hash of the uncrompressed bytes + /// that's saved to the PDB. + /// + public ImmutableArray Checksum { get; } + + private EmbeddedText(string filePath, ImmutableArray checksum, SourceHashAlgorithm checksumAlgorithm, ImmutableArray blob) + { + Debug.Assert(filePath?.Length > 0); + Debug.Assert(Cci.DebugSourceDocument.IsSupportedAlgorithm(checksumAlgorithm)); + Debug.Assert(!blob.IsDefault && blob.Length >= sizeof(int)); + + FilePath = filePath; + Checksum = checksum; + ChecksumAlgorithm = checksumAlgorithm; + Blob = blob; + } + + /// + /// The content that will be written to the PDB. + /// + /// + /// Internal since this is an implementation detail. The only public + /// contract is that you can pass EmbeddedText instances to Emit. + /// It just so happened that doing this up-front was most practical + /// and efficient, but we don't want to be tied to it. + /// + /// For efficiency, the format of this blob is exactly as it is written + /// to the PDB,which prevents extra copies being made during emit. + /// + /// The first 4 bytes (little endian int32) indicate the format: + /// + /// 0: data that follows is uncompressed + /// Positive: data that follows is deflate compressed and value is original, uncompressed size + /// Negative: invalid at this time, but reserved to mark a different format in the future. + /// + internal ImmutableArray Blob { get; } + + /// + /// Constructs a for embedding the given . + /// + /// The file path (pre-normalization) to use in the PDB. + /// The source text to embed. + /// + /// is null. + /// is null. + /// + /// + /// empty. + /// cannot be embedded (see ). + /// + public static EmbeddedText FromSource(string filePath, SourceText text) + { + ValidateFilePath(filePath); + + if (text == null) + { + throw new ArgumentNullException(nameof(text)); + } + + if (!text.CanBeEmbedded) + { + throw new ArgumentException(CodeAnalysisResources.SourceTextCannotBeEmbedded, nameof(text)); + } + + if (!text.PrecomputedEmbeddedTextBlob.IsDefault) + { + return new EmbeddedText(filePath, text.GetChecksum(), text.ChecksumAlgorithm, text.PrecomputedEmbeddedTextBlob); + } + + return new EmbeddedText(filePath, text.GetChecksum(), text.ChecksumAlgorithm, CreateBlob(text)); + } + + /// + /// Constructs an from stream content. + /// + /// The file path (pre-normalization) to use in the PDB. + /// The stream. + /// Hash algorithm to use to calculate checksum of the text that's saved to PDB. + /// + /// is null. + /// is null. + /// + /// + /// is empty. + /// doesn't support reading or seeking. + /// is not supported. + /// + /// An I/O error occurs. + /// Reads from the beginning of the stream. Leaves the stream open. + public static EmbeddedText FromStream(string filePath, Stream stream, SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1) + { + ValidateFilePath(filePath); + + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } + + if (!stream.CanRead || !stream.CanSeek) + { + throw new ArgumentException(CodeAnalysisResources.StreamMustSupportReadAndSeek, nameof(stream)); + } + + SourceText.ValidateChecksumAlgorithm(checksumAlgorithm); + + return new EmbeddedText( + filePath, + SourceText.CalculateChecksum(stream, checksumAlgorithm), + checksumAlgorithm, + CreateBlob(stream)); + } + + /// + /// Constructs an from bytes. + /// + /// The file path (pre-normalization) to use in the PDB. + /// The bytes. + /// Hash algorithm to use to calculate checksum of the text that's saved to PDB. + /// + /// is default-initialized. + /// is null. + /// + /// + /// is empty. + /// is not supported. + /// + /// An I/O error occurs. + /// Reads from the beginning of the stream. Leaves the stream open. + public static EmbeddedText FromBytes(string filePath, ArraySegment bytes, SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1) + { + ValidateFilePath(filePath); + + if (bytes.Array == null) + { + throw new ArgumentNullException(nameof(bytes)); + } + + SourceText.ValidateChecksumAlgorithm(checksumAlgorithm); + + return new EmbeddedText( + filePath, + SourceText.CalculateChecksum(bytes.Array, bytes.Offset, bytes.Count, checksumAlgorithm), + checksumAlgorithm, + CreateBlob(bytes)); + } + + /// is null. + /// is empty. + private static void ValidateFilePath(string filePath) + { + if (filePath == null) + { + throw new ArgumentNullException(nameof(filePath)); + } + + if (filePath.Length == 0) + { + throw new ArgumentException(CodeAnalysisResources.ArgumentCannotBeEmpty, nameof(filePath)); + } + } + + /// + /// Creates the blob to be saved to the PDB. + /// + internal static ImmutableArray CreateBlob(Stream stream) + { + Debug.Assert(stream != null); + Debug.Assert(stream.CanRead); + Debug.Assert(stream.CanSeek); + + long longLength = stream.Length; + Debug.Assert(longLength >= 0); + + if (longLength > int.MaxValue) + { + throw new IOException(CodeAnalysisResources.StreamIsTooLong); + } + + stream.Seek(0, SeekOrigin.Begin); + int length = (int)longLength; + + if (length < CompressionThreshold) + { + using (var builder = Cci.PooledBlobBuilder.GetInstance()) + { + builder.WriteInt32(0); + int bytesWritten = builder.TryWriteBytes(stream, length); + + if (length != bytesWritten) + { + throw new EndOfStreamException(); + } + + return builder.ToImmutableArray(); + } + } + else + { + using (var builder = BlobBuildingStream.GetInstance()) + { + builder.WriteInt32(length); + + using (var deflater = new CountingDeflateStream(builder, CompressionLevel.Optimal, leaveOpen: true)) + { + stream.CopyTo(deflater); + + if (length != deflater.BytesWritten) + { + throw new EndOfStreamException(); + } + } + + return builder.ToImmutableArray(); + } + } + } + + internal static ImmutableArray CreateBlob(ArraySegment bytes) + { + Debug.Assert(bytes.Array != null); + + if (bytes.Count < CompressionThreshold) + { + using (var builder = Cci.PooledBlobBuilder.GetInstance()) + { + builder.WriteInt32(0); + builder.WriteBytes(bytes.Array, bytes.Offset, bytes.Count); + return builder.ToImmutableArray(); + } + } + else + { + using (var builder = BlobBuildingStream.GetInstance()) + { + builder.WriteInt32(bytes.Count); + + using (var deflater = new CountingDeflateStream(builder, CompressionLevel.Optimal, leaveOpen: true)) + { + deflater.Write(bytes.Array, bytes.Offset, bytes.Count); + } + + return builder.ToImmutableArray(); + } + } + } + + private static ImmutableArray CreateBlob(SourceText text) + { + Debug.Assert(text != null); + Debug.Assert(text.CanBeEmbedded); + Debug.Assert(text.Encoding != null); + Debug.Assert(text.PrecomputedEmbeddedTextBlob.IsDefault); + + int maxByteCount; + try + { + maxByteCount = text.Encoding.GetMaxByteCount(text.Length); + } + catch (ArgumentOutOfRangeException) + { + // Encoding does not provide a way to predict that max byte count would not + // fit in Int32 and we must therefore catch ArgumentOutOfRange to handle that + // case. + maxByteCount = int.MaxValue; + } + + using (var builder = BlobBuildingStream.GetInstance()) + { + if (maxByteCount < CompressionThreshold) + { + builder.WriteInt32(0); + + using (var writer = new StreamWriter(builder, text.Encoding, bufferSize: Math.Max(1, text.Length), leaveOpen: true)) + { + text.Write(writer); + } + } + else + { + Blob reserved = builder.ReserveBytes(4); + + using (var deflater = new CountingDeflateStream(builder, CompressionLevel.Optimal, leaveOpen: true)) + { + using (var writer = new StreamWriter(deflater, text.Encoding, bufferSize: 1024, leaveOpen: true)) + { + text.Write(writer); + } + + new BlobWriter(reserved).WriteInt32(deflater.BytesWritten); + } + } + + return builder.ToImmutableArray(); + } + } + + internal Cci.DebugSourceInfo GetDebugSourceInfo() + { + return new Cci.DebugSourceInfo(Checksum, ChecksumAlgorithm, Blob); + } + + private sealed class CountingDeflateStream : DeflateStream + { + public CountingDeflateStream(Stream stream, CompressionLevel compressionLevel, bool leaveOpen) + : base(stream, compressionLevel, leaveOpen) + { + } + + public int BytesWritten { get; private set; } + + public override void Write(byte[] array, int offset, int count) + { + base.Write(array, offset, count); + + // checked arithmetic is release-enabled quasi-assert. We start with at most + // int.MaxValue chars so compression or encoding would have to be abysmal for + // this to overflow. We'd probably be lucky to even get this far but if we do + // we should fail fast. + checked { BytesWritten += count; } + } + + public override void WriteByte(byte value) + { + base.WriteByte(value); + + // same rationale for checked arithmetic as above. + checked { BytesWritten++; }; + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + throw ExceptionUtilities.Unreachable; + } + } + } +} diff --git a/src/Compilers/Core/Portable/Emit/CommonPEModuleBuilder.cs b/src/Compilers/Core/Portable/Emit/CommonPEModuleBuilder.cs index ca6e8d9354176..2098e67606089 100644 --- a/src/Compilers/Core/Portable/Emit/CommonPEModuleBuilder.cs +++ b/src/Compilers/Core/Portable/Emit/CommonPEModuleBuilder.cs @@ -11,6 +11,7 @@ using Microsoft.CodeAnalysis.CodeGen; using Microsoft.CodeAnalysis.Emit.NoPia; using Roslyn.Utilities; +using Microsoft.CodeAnalysis.Text; namespace Microsoft.CodeAnalysis.Emit { @@ -79,6 +80,18 @@ internal abstract class PEModuleBuilder _embedddedTexts = SpecializedCollections.EmptyEnumerable(); + + public IEnumerable EmbeddedTexts + { + get { return _embedddedTexts; } + set + { + Debug.Assert(value != null); + _embedddedTexts = value; + } + } + public abstract TEmbeddedTypesManager EmbeddedTypesManagerOpt { get; } /// @@ -913,6 +926,8 @@ int Cci.IModule.HintNumberOfMethodDefinitions int Cci.IModule.DebugDocumentCount => DebugDocumentsBuilder.DebugDocumentCount; + IEnumerable Cci.IModule.EmbeddedDocuments => DebugDocumentsBuilder.EmbeddedDocuments; + #endregion #region INamedEntity diff --git a/src/Compilers/Core/Portable/Emit/DebugDocumentsBuilder.cs b/src/Compilers/Core/Portable/Emit/DebugDocumentsBuilder.cs index dda57e15ff6d6..1a2af79bcc1d6 100644 --- a/src/Compilers/Core/Portable/Emit/DebugDocumentsBuilder.cs +++ b/src/Compilers/Core/Portable/Emit/DebugDocumentsBuilder.cs @@ -3,6 +3,10 @@ using Roslyn.Utilities; using System; using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; namespace Microsoft.CodeAnalysis.Emit { @@ -17,6 +21,7 @@ internal sealed class DebugDocumentsBuilder private readonly ConcurrentDictionary _debugDocuments; private readonly ConcurrentCache, string> _normalizedPathsCache; private readonly SourceReferenceResolver _resolverOpt; + private ImmutableArray _embeddedDocuments; public DebugDocumentsBuilder(SourceReferenceResolver resolverOpt, bool isDocumentNameCaseSensitive) { @@ -28,6 +33,13 @@ public DebugDocumentsBuilder(SourceReferenceResolver resolverOpt, bool isDocumen StringComparer.OrdinalIgnoreCase); _normalizedPathsCache = new ConcurrentCache, string>(16); + _embeddedDocuments = ImmutableArray.Empty; + } + + internal ImmutableArray EmbeddedDocuments + { + get { return _embeddedDocuments; } + set { Debug.Assert(value != null); _embeddedDocuments = value; } } internal int DebugDocumentCount => _debugDocuments.Count; @@ -71,5 +83,7 @@ internal string NormalizeDebugDocumentPath(string path, string basePath) return normalizedPath; } + + } } diff --git a/src/Compilers/Core/Portable/Emit/DebugInformationFormat.cs b/src/Compilers/Core/Portable/Emit/DebugInformationFormat.cs index 88a7e0edcb557..14661b3473757 100644 --- a/src/Compilers/Core/Portable/Emit/DebugInformationFormat.cs +++ b/src/Compilers/Core/Portable/Emit/DebugInformationFormat.cs @@ -15,5 +15,10 @@ internal static bool IsValid(this DebugInformationFormat value) { return value >= DebugInformationFormat.Pdb && value <= DebugInformationFormat.Embedded; } + + internal static bool IsPortable(this DebugInformationFormat value) + { + return value == DebugInformationFormat.PortablePdb || value == DebugInformationFormat.Embedded; + } } } diff --git a/src/Compilers/Core/Portable/EncodedStringText.cs b/src/Compilers/Core/Portable/EncodedStringText.cs index 36dbb83059cb9..1847452e0aa4e 100644 --- a/src/Compilers/Core/Portable/EncodedStringText.cs +++ b/src/Compilers/Core/Portable/EncodedStringText.cs @@ -61,6 +61,7 @@ private static Encoding GetFallbackEncoding() /// If not specified auto-detect heuristics are used to determine the encoding. If these heuristics fail the decoding is assumed to be Encoding.Default. /// Note that if the stream starts with Byte Order Mark the value of is ignored. /// + /// Indicates if the file can be embedded in the PDB. /// Hash algorithm used to calculate document checksum. /// /// The stream content can't be decoded using the specified , or @@ -69,18 +70,21 @@ private static Encoding GetFallbackEncoding() /// An IO error occurred while reading from the stream. internal static SourceText Create(Stream stream, Encoding defaultEncoding = null, - SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1) + SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1, + bool canBeEmbedded = false) { return Create(stream, s_fallbackEncoding, defaultEncoding: defaultEncoding, - checksumAlgorithm: checksumAlgorithm); + checksumAlgorithm: checksumAlgorithm, + canBeEmbedded: canBeEmbedded); } // internal for testing internal static SourceText Create(Stream stream, Lazy getEncoding, Encoding defaultEncoding = null, - SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1) + SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1, + bool canBeEmbedded = false) { Debug.Assert(stream != null); Debug.Assert(stream.CanRead && stream.CanSeek); @@ -90,7 +94,7 @@ internal static SourceText Create(Stream stream, Lazy getEncoding, { try { - return Decode(stream, s_utf8Encoding, checksumAlgorithm, throwIfBinaryDetected: false); + return Decode(stream, s_utf8Encoding, checksumAlgorithm, throwIfBinaryDetected: false, canBeEmbedded: canBeEmbedded); } catch (DecoderFallbackException) { @@ -115,12 +119,19 @@ internal static SourceText Create(Stream stream, Lazy getEncoding, /// The expected encoding of the stream. The actual encoding used may be different if byte order marks are detected. /// The checksum algorithm to use. /// Throw if binary (non-text) data is detected. + /// Indicates if the text can be embedded in the PDB. /// The decoded from the stream. /// The decoder was unable to decode the stream with the given encoding. + /// Error reading from stream. /// /// internal for unit testing /// - internal static SourceText Decode(Stream data, Encoding encoding, SourceHashAlgorithm checksumAlgorithm, bool throwIfBinaryDetected = false) + internal static SourceText Decode( + Stream data, + Encoding encoding, + SourceHashAlgorithm checksumAlgorithm, + bool throwIfBinaryDetected = false, + bool canBeEmbedded = false) { Debug.Assert(data != null); Debug.Assert(encoding != null); @@ -128,16 +139,16 @@ internal static SourceText Decode(Stream data, Encoding encoding, SourceHashAlgo data.Seek(0, SeekOrigin.Begin); // For small streams, see if we can read the byte buffer directly. - if (encoding.GetMaxCharCount((int)data.Length) < LargeObjectHeapLimitInChars) + if (encoding.GetMaxCharCountOrThrowIfHuge(data) < LargeObjectHeapLimitInChars) { byte[] buffer = TryGetByteArrayFromStream(data); if (buffer != null) { - return SourceText.From(buffer, (int)data.Length, encoding, checksumAlgorithm, throwIfBinaryDetected); + return SourceText.From(buffer, (int)data.Length, encoding, checksumAlgorithm, throwIfBinaryDetected, canBeEmbedded); } } - return SourceText.From(data, encoding, checksumAlgorithm, throwIfBinaryDetected); + return SourceText.From(data, encoding, checksumAlgorithm, throwIfBinaryDetected, canBeEmbedded); } /// @@ -148,7 +159,7 @@ internal static SourceText Decode(Stream data, Encoding encoding, SourceHashAlgo /// The contents of as a byte array or null if the stream can't easily /// be read into a byte array. /// - private static byte[] TryGetByteArrayFromStream(Stream data) + internal static byte[] TryGetByteArrayFromStream(Stream data) { byte[] buffer; diff --git a/src/Compilers/Core/Portable/GlobalSuppressions.cs b/src/Compilers/Core/Portable/GlobalSuppressions.cs index 7845b1f38f580..81a04ac25c91d 100644 --- a/src/Compilers/Core/Portable/GlobalSuppressions.cs +++ b/src/Compilers/Core/Portable/GlobalSuppressions.cs @@ -7,3 +7,9 @@ [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "", Scope = "member", Target = "~M:Microsoft.CodeAnalysis.Compilation.CreateTupleTypeSymbol(System.Collections.Immutable.ImmutableArray{Microsoft.CodeAnalysis.ITypeSymbol},System.Collections.Immutable.ImmutableArray{System.String})~Microsoft.CodeAnalysis.INamedTypeSymbol")] [assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "", Scope = "member", Target = "~M:Microsoft.CodeAnalysis.Compilation.CreateTupleTypeSymbol(Microsoft.CodeAnalysis.INamedTypeSymbol,System.Collections.Immutable.ImmutableArray{System.String})~Microsoft.CodeAnalysis.INamedTypeSymbol")] +// These arise from modifications to existing API. The analysis seems too strict as the first +// required paramter type is different for all overloads and so there is no ambiguity. +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "", Scope = "member", Target = "~M:Microsoft.CodeAnalysis.Text.SourceText.From(System.IO.Stream,System.Text.Encoding,Microsoft.CodeAnalysis.Text.SourceHashAlgorithm,System.Boolean,System.Boolean)~Microsoft.CodeAnalysis.Text.SourceText")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "", Scope = "member", Target = "~M:Microsoft.CodeAnalysis.Text.SourceText.From(System.Byte[],System.Int32,System.Text.Encoding,Microsoft.CodeAnalysis.Text.SourceHashAlgorithm,System.Boolean,System.Boolean)~Microsoft.CodeAnalysis.Text.SourceText")] +[assembly: System.Diagnostics.CodeAnalysis.SuppressMessage("ApiDesign", "RS0027:Public API with optional parameter(s) should have the most parameters amongst its public overloads.", Justification = "", Scope = "member", Target = "~M:Microsoft.CodeAnalysis.Text.SourceText.From(System.String,System.Text.Encoding,Microsoft.CodeAnalysis.Text.SourceHashAlgorithm)~Microsoft.CodeAnalysis.Text.SourceText")] + diff --git a/src/Compilers/Core/Portable/InternalUtilities/BlobBuildingStream.cs b/src/Compilers/Core/Portable/InternalUtilities/BlobBuildingStream.cs new file mode 100644 index 0000000000000..4c1bc63eee461 --- /dev/null +++ b/src/Compilers/Core/Portable/InternalUtilities/BlobBuildingStream.cs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.Reflection.Metadata; + +namespace Roslyn.Utilities +{ + /// + /// A write-only memory stream backed by a . + /// + internal sealed class BlobBuildingStream : Stream + { + private static ObjectPool s_pool = new ObjectPool(() => new BlobBuildingStream()); + private BlobBuilder _builder; + + /// + /// The chunk size to be used by the underlying BlobBuilder. + /// + /// + /// The current single use case for this type is embedded sources in PDBs. + /// + /// 32 KB is: + /// + /// * Large enough to handle 99.6% all VB and C# files in Roslyn and CoreFX + /// without allocating additional chunks. + /// + /// * Small enough to avoid the large object heap. + /// + /// * Large enough to handle the files in the 0.4% case without allocating tons + /// of small chunks. Very large source files are often generated in build + /// (e.g. Syntax.xml.Generated.vb is 390KB compressed!) and those are actually + /// attractive candidates for embedding, so we don't want to discount the large + /// case too heavily.) + /// + /// * We pool the outer BlobBuildingStream but only retain the first allocated chunk. + /// + public const int ChunkSize = 32 * 1024; + + public override bool CanWrite => true; + public override bool CanRead => false; + public override bool CanSeek => false; + public override long Length => _builder.Count; + + public static BlobBuildingStream GetInstance() + { + return s_pool.Allocate(); + } + + private BlobBuildingStream() + { + // NOTE: We pool the wrapping BlobBuildingStream, but not individual chunks. + // The first chunk will be reused, but any further chunks will be freed when we're done building blob. + _builder = new BlobBuilder(ChunkSize); + } + + public override void Write(byte[] buffer, int offset, int count) + { + _builder.WriteBytes(buffer, offset, count); + } + + public override void WriteByte(byte value) + { + _builder.WriteByte(value); + } + + public void WriteInt32(int value) + { + _builder.WriteInt32(value); + } + + public Blob ReserveBytes(int byteCount) + { + return _builder.ReserveBytes(byteCount); + } + + public ImmutableArray ToImmutableArray() + { + return _builder.ToImmutableArray(); + } + + public void Free() + { + _builder.Clear(); // frees all but first chunk + s_pool.Free(this); // return first chunk to pool + } + + public override void Flush() + { + } + + protected override void Dispose(bool disposing) + { + Debug.Assert(disposing); + Free(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override long Position + { + get { throw new NotSupportedException(); } + set { throw new NotSupportedException(); } + } + + } +} diff --git a/src/Compilers/Core/Portable/InternalUtilities/EncodingExtensions.cs b/src/Compilers/Core/Portable/InternalUtilities/EncodingExtensions.cs new file mode 100644 index 0000000000000..b039b6806074e --- /dev/null +++ b/src/Compilers/Core/Portable/InternalUtilities/EncodingExtensions.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.CodeAnalysis; +using System; +using System.Diagnostics; +using System.IO; +using System.Text; + +namespace Roslyn.Utilities +{ + internal static class EncodingExtensions + { + /// + /// Get maximum char count needed to decode the entire stream. + /// + /// Stream is so big that max char count can't fit in . + internal static int GetMaxCharCountOrThrowIfHuge(this Encoding encoding, Stream stream) + { + Debug.Assert(stream.CanSeek); + long length = stream.Length; + + if (length <= int.MaxValue) + { + try + { + return encoding.GetMaxCharCount((int)length); + } + catch (ArgumentOutOfRangeException) + { + // Encoding does not provide a way to predict that max byte count would not + // fit in Int32 and we must therefore catch ArgumentOutOfRange to handle that + // case. + } + } + +#if WORKSPACE_DESKTOP + throw new IOException(WorkspacesResources.Stream_is_too_long); +#else + throw new IOException(CodeAnalysisResources.StreamIsTooLong); +#endif + } + } +} diff --git a/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.Empty.Set.cs b/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.Empty.Set.cs index de128f0b00147..a6f9880a13ae4 100644 --- a/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.Empty.Set.cs +++ b/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.Empty.Set.cs @@ -10,8 +10,11 @@ internal partial class SpecializedCollections private partial class Empty { internal class Set : Collection, ISet +#if COMPILERCORE + , IReadOnlySet +#endif { - public static readonly new ISet Instance = new Set(); + public static readonly new Set Instance = new Set(); protected Set() { diff --git a/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.ReadOnly.Set.cs b/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.ReadOnly.Set.cs index caf627aee9492..6e227449cccfd 100644 --- a/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.ReadOnly.Set.cs +++ b/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.ReadOnly.Set.cs @@ -10,6 +10,9 @@ internal partial class SpecializedCollections private partial class ReadOnly { internal class Set : Collection, ISet +#if COMPILERCORE + , IReadOnlySet +#endif where TUnderlying : ISet { public Set(TUnderlying underlying) diff --git a/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.cs b/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.cs index 631eeace99f6b..0f8a66111d66b 100644 --- a/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.cs +++ b/src/Compilers/Core/Portable/InternalUtilities/SpecializedCollections.cs @@ -44,6 +44,13 @@ public static ISet EmptySet() return Empty.Set.Instance; } +#if COMPILERCORE + public static IReadOnlySet EmptyReadOnlySet() + { + return Empty.Set.Instance; + } +#endif + public static IDictionary EmptyDictionary() { return Empty.Dictionary.Instance; @@ -93,22 +100,13 @@ public static ISet ReadOnlySet(ISet set) : new ReadOnly.Set, T>(set); } - public static ISet ReadOnlySet(IEnumerable values) +#if COMPILERCORE + public static IReadOnlySet StronglyTypedReadOnlySet(ISet set) { - var set = values as ISet; - if (set != null) - { - return ReadOnlySet(set); - } - - HashSet result = null; - foreach (var item in values) - { - result = result ?? new HashSet(); - result.Add(item); - } - - return ReadOnlySet(result); + return set == null || set.Count == 0 + ? EmptyReadOnlySet() + : new ReadOnly.Set, T>(set); } +#endif } } diff --git a/src/Compilers/Core/Portable/NativePdbWriter/PdbWriter.cs b/src/Compilers/Core/Portable/NativePdbWriter/PdbWriter.cs index c7712ee6b15e7..919fe58afadf7 100644 --- a/src/Compilers/Core/Portable/NativePdbWriter/PdbWriter.cs +++ b/src/Compilers/Core/Portable/NativePdbWriter/PdbWriter.cs @@ -118,7 +118,8 @@ internal enum PdbWriterOperation : byte DefineKickoffMethod, OpenMapTokensToSourceSpans, MapTokenToSourceSpan, - CloseMapTokensToSourceSpans + CloseMapTokensToSourceSpans, + SetSource } public bool LogOperation(PdbWriterOperation op) @@ -956,13 +957,13 @@ private ISymUnmanagedDocumentWriter GetDocumentWriter(DebugSourceDocument docume _documentMap.Add(document, writer); - var checksumAndAlgorithm = document.ChecksumAndAlgorithm; - if (!checksumAndAlgorithm.Item1.IsDefault) + DebugSourceInfo info = document.GetSourceInfo(); + if (!info.Checksum.IsDefault) { try { - var algorithmId = checksumAndAlgorithm.Item2; - var checksum = checksumAndAlgorithm.Item1.ToArray(); + var algorithmId = info.ChecksumAlgorithmId; + var checksum = info.Checksum.ToArray(); var checksumSize = (uint)checksum.Length; writer.SetCheckSum(algorithmId, checksumSize, checksum); if (_callLogger.LogOperation(OP.SetCheckSum)) @@ -977,6 +978,9 @@ private ISymUnmanagedDocumentWriter GetDocumentWriter(DebugSourceDocument docume throw new PdbWritingException(ex); } } + + // embedded text not currently supported for native PDB and we should have validated that + Debug.Assert(info.EmbeddedTextBlob.IsDefault); } return writer; diff --git a/src/Compilers/Core/Portable/PEWriter/DebugSourceDocument.cs b/src/Compilers/Core/Portable/PEWriter/DebugSourceDocument.cs index 3af3fd6159d5d..12fa3b85fdfd8 100644 --- a/src/Compilers/Core/Portable/PEWriter/DebugSourceDocument.cs +++ b/src/Compilers/Core/Portable/PEWriter/DebugSourceDocument.cs @@ -19,8 +19,7 @@ internal sealed class DebugSourceDocument private readonly string _location; private readonly Guid _language; private readonly bool _isComputedChecksum; - - private readonly Task, Guid>> _checksumAndAlgorithm; + private readonly Task _sourceInfo; public DebugSourceDocument(string location, Guid language) { @@ -33,10 +32,10 @@ public DebugSourceDocument(string location, Guid language) /// /// Use to create a document when checksum is computed based on actual source stream. /// - public DebugSourceDocument(string location, Guid language, Func, Guid>> checksumAndAlgorithm) + public DebugSourceDocument(string location, Guid language, Func sourceInfo) : this(location, language) { - _checksumAndAlgorithm = Task.Run(checksumAndAlgorithm); + _sourceInfo = Task.Run(sourceInfo); _isComputedChecksum = true; } @@ -46,17 +45,29 @@ public DebugSourceDocument(string location, Guid language, Func checksum, Guid algorithm) : this(location, language) { - _checksumAndAlgorithm = Task.FromResult(ValueTuple.Create(checksum, algorithm)); + _sourceInfo = Task.FromResult(new DebugSourceInfo(checksum, algorithm)); } internal static bool IsSupportedAlgorithm(SourceHashAlgorithm algorithm) { - Guid guid; - return TryGetAlgorithmGuid(algorithm, out guid); + // Dev12 debugger supports MD5, SHA1. + // Dev14 debugger supports MD5, SHA1, SHA256. + // MD5 is obsolete. + + switch (algorithm) + { + case SourceHashAlgorithm.Sha1: + case SourceHashAlgorithm.Sha256: + return true; + default: + return false; + } } - internal static bool TryGetAlgorithmGuid(SourceHashAlgorithm algorithm, out Guid guid) + internal static Guid GetAlgorithmGuid(SourceHashAlgorithm algorithm) { + Debug.Assert(IsSupportedAlgorithm(algorithm)); + // Dev12 debugger supports MD5, SHA1. // Dev14 debugger supports MD5, SHA1, SHA256. // MD5 is obsolete. @@ -66,16 +77,13 @@ internal static bool TryGetAlgorithmGuid(SourceHashAlgorithm algorithm, out Guid switch (algorithm) { case SourceHashAlgorithm.Sha1: - guid = new Guid((int)0xff1816ec, (short)0xaa5e, 0x4d10, 0x87, 0xf7, 0x6f, 0x49, 0x63, 0x83, 0x34, 0x60); - return true; - + return new Guid((int)0xff1816ec, (short)0xaa5e, 0x4d10, 0x87, 0xf7, 0x6f, 0x49, 0x63, 0x83, 0x34, 0x60); + case SourceHashAlgorithm.Sha256: - guid = new Guid((int)0x8829d00f, 0x11b8, 0x4213, 0x87, 0x8b, 0x77, 0x0e, 0x85, 0x97, 0xac, 0x16); - return true; + return new Guid((int)0x8829d00f, 0x11b8, 0x4213, 0x87, 0x8b, 0x77, 0x0e, 0x85, 0x97, 0xac, 0x16); default: - guid = default(Guid); - return false; + throw ExceptionUtilities.UnexpectedValue(algorithm); } } } @@ -100,12 +108,9 @@ public string Location get { return _location; } } - public ValueTuple, Guid> ChecksumAndAlgorithm + public DebugSourceInfo GetSourceInfo() { - get - { - return _checksumAndAlgorithm?.Result ?? default(ValueTuple, Guid>); - } + return _sourceInfo?.Result ?? default(DebugSourceInfo); } /// diff --git a/src/Compilers/Core/Portable/PEWriter/DebugSourceInfo.cs b/src/Compilers/Core/Portable/PEWriter/DebugSourceInfo.cs new file mode 100644 index 0000000000000..413a1cfb393ba --- /dev/null +++ b/src/Compilers/Core/Portable/PEWriter/DebugSourceInfo.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.CodeAnalysis.Text; +using System; +using System.Collections.Immutable; + +namespace Microsoft.Cci +{ + /// + /// Represents the portion of a that are derived + /// from the source document content, and which can be computed asynchronously. + /// + internal struct DebugSourceInfo + { + /// + /// The ID of the hash algorithm used. + /// + public readonly Guid ChecksumAlgorithmId; + + /// + /// The hash of the document content. + /// + public readonly ImmutableArray Checksum; + + /// + /// The source text to embed in the PDB. (If any, otherwise default.) + /// + public readonly ImmutableArray EmbeddedTextBlob; + + public DebugSourceInfo( + ImmutableArray checksum, + SourceHashAlgorithm checksumAlgorithm, + ImmutableArray embeddedTextBlob = default(ImmutableArray)) + : this(checksum, DebugSourceDocument.GetAlgorithmGuid(checksumAlgorithm), embeddedTextBlob) + { + } + + public DebugSourceInfo( + ImmutableArray checksum, + Guid checksumAlgorithmId, + ImmutableArray embeddedTextBlob = default(ImmutableArray)) + { + ChecksumAlgorithmId = checksumAlgorithmId; + Checksum = checksum; + EmbeddedTextBlob = embeddedTextBlob; + } + } +} \ No newline at end of file diff --git a/src/Compilers/Core/Portable/PEWriter/MetadataWriter.DynamicAnalysis.cs b/src/Compilers/Core/Portable/PEWriter/MetadataWriter.DynamicAnalysis.cs index da3f0ab6a403c..ce1d439a685b2 100644 --- a/src/Compilers/Core/Portable/PEWriter/MetadataWriter.DynamicAnalysis.cs +++ b/src/Compilers/Core/Portable/PEWriter/MetadataWriter.DynamicAnalysis.cs @@ -211,12 +211,12 @@ private int GetOrAddDocument(DebugSourceDocument document, Dictionary + /// Add document entries for any embedded text document that does not yet have an entry. + /// + /// + /// This is done after serializing method debug info to ensure that we embed all requested + /// text even if there are no correspodning sequence points. + /// + public void AddRemainingEmbeddedDocuments(IEnumerable documents) + { + foreach (var document in documents) + { + Debug.Assert(document.GetSourceInfo().EmbeddedTextBlob != null); + GetOrAddDocument(document, _documentIndex); + } + } + #endregion #region Edit and Continue diff --git a/src/Compilers/Core/Portable/PEWriter/PeWriter.cs b/src/Compilers/Core/Portable/PEWriter/PeWriter.cs index 1ec1e8c865f54..c2bc248d44d95 100644 --- a/src/Compilers/Core/Portable/PEWriter/PeWriter.cs +++ b/src/Compilers/Core/Portable/PEWriter/PeWriter.cs @@ -89,6 +89,9 @@ public static bool WritePeToStream( nativePdbWriterOpt.AssertAllDefinitionsHaveTokens(mdWriter.Module.GetSymbolToLocationMap()); #endif } + + // embedded text not currently supported for native PDB and we should have validated that + Debug.Assert(!mdWriter.Module.EmbeddedDocuments.Any()); } Stream peStream = getPeStream(); @@ -133,6 +136,8 @@ public static bool WritePeToStream( BlobBuilder portablePdbToEmbed = null; if (mdWriter.EmitStandaloneDebugMetadata) { + mdWriter.AddRemainingEmbeddedDocuments(mdWriter.Module.EmbeddedDocuments); + var portablePdbBlob = new BlobBuilder(); var portablePdbBuilder = mdWriter.GetPortablePdbBuilder(metadataRootBuilder.Sizes, debugEntryPointHandle, deterministicIdProvider); pdbContentId = portablePdbBuilder.Serialize(portablePdbBlob); diff --git a/src/Compilers/Core/Portable/PEWriter/PooledBlobBuilder.cs b/src/Compilers/Core/Portable/PEWriter/PooledBlobBuilder.cs index e82dc7ccc1533..48f67fc046377 100644 --- a/src/Compilers/Core/Portable/PEWriter/PooledBlobBuilder.cs +++ b/src/Compilers/Core/Portable/PEWriter/PooledBlobBuilder.cs @@ -1,11 +1,12 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using Roslyn.Utilities; +using System; using System.Reflection.Metadata; namespace Microsoft.Cci { - internal sealed class PooledBlobBuilder : BlobBuilder + internal sealed class PooledBlobBuilder : BlobBuilder, IDisposable { private const int PoolSize = 128; private const int ChunkSize = 1024; @@ -42,5 +43,10 @@ protected override void FreeChunk() { base.Free(); } + + void IDisposable.Dispose() + { + Free(); + } } } diff --git a/src/Compilers/Core/Portable/PEWriter/Units.cs b/src/Compilers/Core/Portable/PEWriter/Units.cs index b94a3f966473e..d62e1a98178e2 100644 --- a/src/Compilers/Core/Portable/PEWriter/Units.cs +++ b/src/Compilers/Core/Portable/PEWriter/Units.cs @@ -215,6 +215,11 @@ internal interface IModule : IUnit, IModuleReference int DebugDocumentCount { get; } Stream SourceLinkStream { get; } + + /// + /// Documents that will have their text embedded in the PDB. + /// + IEnumerable EmbeddedDocuments { get; } } internal struct DefinitionWithLocation diff --git a/src/Compilers/Core/Portable/PublicAPI.Unshipped.txt b/src/Compilers/Core/Portable/PublicAPI.Unshipped.txt index d1fa4294ecc40..ac99159640c0a 100644 --- a/src/Compilers/Core/Portable/PublicAPI.Unshipped.txt +++ b/src/Compilers/Core/Portable/PublicAPI.Unshipped.txt @@ -1,9 +1,12 @@ +*REMOVED*Microsoft.CodeAnalysis.Compilation.Emit(System.IO.Stream peStream, System.IO.Stream pdbStream = null, System.IO.Stream xmlDocumentationStream = null, System.IO.Stream win32Resources = null, System.Collections.Generic.IEnumerable manifestResources = null, Microsoft.CodeAnalysis.Emit.EmitOptions options = null, Microsoft.CodeAnalysis.IMethodSymbol debugEntryPoint = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> Microsoft.CodeAnalysis.Emit.EmitResult +*REMOVED*static Microsoft.CodeAnalysis.Text.SourceText.From(System.IO.Stream stream, System.Text.Encoding encoding = null, Microsoft.CodeAnalysis.Text.SourceHashAlgorithm checksumAlgorithm = Microsoft.CodeAnalysis.Text.SourceHashAlgorithm.Sha1, bool throwIfBinaryDetected = false) -> Microsoft.CodeAnalysis.Text.SourceText +*REMOVED*static Microsoft.CodeAnalysis.Text.SourceText.From(byte[] buffer, int length, System.Text.Encoding encoding = null, Microsoft.CodeAnalysis.Text.SourceHashAlgorithm checksumAlgorithm = Microsoft.CodeAnalysis.Text.SourceHashAlgorithm.Sha1, bool throwIfBinaryDetected = false) -> Microsoft.CodeAnalysis.Text.SourceText +Microsoft.CodeAnalysis.CommandLineArguments.EmbeddedFiles.get -> System.Collections.Immutable.ImmutableArray Microsoft.CodeAnalysis.CommandLineArguments.SourceLink.get -> string Microsoft.CodeAnalysis.Compilation.CreateAnonymousTypeSymbol(System.Collections.Immutable.ImmutableArray memberTypes, System.Collections.Immutable.ImmutableArray memberNames) -> Microsoft.CodeAnalysis.INamedTypeSymbol Microsoft.CodeAnalysis.Compilation.CreateTupleTypeSymbol(Microsoft.CodeAnalysis.INamedTypeSymbol underlyingType, System.Collections.Immutable.ImmutableArray elementNames = default(System.Collections.Immutable.ImmutableArray)) -> Microsoft.CodeAnalysis.INamedTypeSymbol Microsoft.CodeAnalysis.Compilation.CreateTupleTypeSymbol(System.Collections.Immutable.ImmutableArray elementTypes, System.Collections.Immutable.ImmutableArray elementNames = default(System.Collections.Immutable.ImmutableArray)) -> Microsoft.CodeAnalysis.INamedTypeSymbol -*REMOVED*Microsoft.CodeAnalysis.Compilation.Emit(System.IO.Stream peStream, System.IO.Stream pdbStream = null, System.IO.Stream xmlDocumentationStream = null, System.IO.Stream win32Resources = null, System.Collections.Generic.IEnumerable manifestResources = null, Microsoft.CodeAnalysis.Emit.EmitOptions options = null, Microsoft.CodeAnalysis.IMethodSymbol debugEntryPoint = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> Microsoft.CodeAnalysis.Emit.EmitResult -Microsoft.CodeAnalysis.Compilation.Emit(System.IO.Stream peStream, System.IO.Stream pdbStream = null, System.IO.Stream xmlDocumentationStream = null, System.IO.Stream win32Resources = null, System.Collections.Generic.IEnumerable manifestResources = null, Microsoft.CodeAnalysis.Emit.EmitOptions options = null, Microsoft.CodeAnalysis.IMethodSymbol debugEntryPoint = null, System.IO.Stream sourceLinkStream = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> Microsoft.CodeAnalysis.Emit.EmitResult +Microsoft.CodeAnalysis.Compilation.Emit(System.IO.Stream peStream, System.IO.Stream pdbStream = null, System.IO.Stream xmlDocumentationStream = null, System.IO.Stream win32Resources = null, System.Collections.Generic.IEnumerable manifestResources = null, Microsoft.CodeAnalysis.Emit.EmitOptions options = null, Microsoft.CodeAnalysis.IMethodSymbol debugEntryPoint = null, System.IO.Stream sourceLinkStream = null, System.Collections.Generic.IEnumerable embeddedTexts = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> Microsoft.CodeAnalysis.Emit.EmitResult Microsoft.CodeAnalysis.Compilation.Emit(System.IO.Stream peStream, System.IO.Stream pdbStream, System.IO.Stream xmlDocumentationStream, System.IO.Stream win32Resources, System.Collections.Generic.IEnumerable manifestResources, Microsoft.CodeAnalysis.Emit.EmitOptions options, Microsoft.CodeAnalysis.IMethodSymbol debugEntryPoint, System.Threading.CancellationToken cancellationToken) -> Microsoft.CodeAnalysis.Emit.EmitResult Microsoft.CodeAnalysis.CompilationOptions.WithConcurrentBuild(bool concurrent) -> Microsoft.CodeAnalysis.CompilationOptions Microsoft.CodeAnalysis.Diagnostics.AnalysisContext.RegisterOperationAction(System.Action action, params Microsoft.CodeAnalysis.OperationKind[] operationKinds) -> void @@ -52,6 +55,10 @@ Microsoft.CodeAnalysis.Diagnostics.Telemetry.AnalyzerTelemetryInfo.SemanticModel Microsoft.CodeAnalysis.Diagnostics.Telemetry.AnalyzerTelemetryInfo.SymbolActionsCount.set -> void Microsoft.CodeAnalysis.Diagnostics.Telemetry.AnalyzerTelemetryInfo.SyntaxNodeActionsCount.set -> void Microsoft.CodeAnalysis.Diagnostics.Telemetry.AnalyzerTelemetryInfo.SyntaxTreeActionsCount.set -> void +Microsoft.CodeAnalysis.EmbeddedText +Microsoft.CodeAnalysis.EmbeddedText.Checksum.get -> System.Collections.Immutable.ImmutableArray +Microsoft.CodeAnalysis.EmbeddedText.ChecksumAlgorithm.get -> Microsoft.CodeAnalysis.Text.SourceHashAlgorithm +Microsoft.CodeAnalysis.EmbeddedText.FilePath.get -> string Microsoft.CodeAnalysis.Emit.EmitOptions.EmitOptions(bool metadataOnly = false, Microsoft.CodeAnalysis.Emit.DebugInformationFormat debugInformationFormat = (Microsoft.CodeAnalysis.Emit.DebugInformationFormat)0, string pdbFilePath = null, string outputNameOverride = null, int fileAlignment = 0, ulong baseAddress = 0, bool highEntropyVirtualAddressSpace = false, Microsoft.CodeAnalysis.SubsystemVersion subsystemVersion = default(Microsoft.CodeAnalysis.SubsystemVersion), string runtimeMetadataVersion = null, bool tolerateErrors = false, bool includePrivateMembers = false, string instrument = "") -> void Microsoft.CodeAnalysis.Emit.EmitOptions.EmitOptions(bool metadataOnly, Microsoft.CodeAnalysis.Emit.DebugInformationFormat debugInformationFormat, string pdbFilePath, string outputNameOverride, int fileAlignment, ulong baseAddress, bool highEntropyVirtualAddressSpace, Microsoft.CodeAnalysis.SubsystemVersion subsystemVersion, string runtimeMetadataVersion, bool tolerateErrors, bool includePrivateMembers) -> void Microsoft.CodeAnalysis.Emit.EmitOptions.Instrument.get -> string @@ -686,6 +693,7 @@ Microsoft.CodeAnalysis.Semantics.UnaryOperationKind.UnsignedPrefixIncrement = 77 Microsoft.CodeAnalysis.SymbolDisplayFormat.RemoveGenericsOptions(Microsoft.CodeAnalysis.SymbolDisplayGenericsOptions options) -> Microsoft.CodeAnalysis.SymbolDisplayFormat Microsoft.CodeAnalysis.SymbolDisplayFormat.RemoveLocalOptions(Microsoft.CodeAnalysis.SymbolDisplayLocalOptions options) -> Microsoft.CodeAnalysis.SymbolDisplayFormat Microsoft.CodeAnalysis.SymbolDisplayFormat.RemoveMiscellaneousOptions(Microsoft.CodeAnalysis.SymbolDisplayMiscellaneousOptions options) -> Microsoft.CodeAnalysis.SymbolDisplayFormat +Microsoft.CodeAnalysis.Text.SourceText.CanBeEmbedded.get -> bool Microsoft.CodeAnalysis.Text.SourceText.GetChecksum() -> System.Collections.Immutable.ImmutableArray abstract Microsoft.CodeAnalysis.Diagnostics.OperationBlockStartAnalysisContext.RegisterOperationAction(System.Action action, System.Collections.Immutable.ImmutableArray operationKinds) -> void abstract Microsoft.CodeAnalysis.Diagnostics.OperationBlockStartAnalysisContext.RegisterOperationBlockEndAction(System.Action action) -> void @@ -765,6 +773,9 @@ override Microsoft.CodeAnalysis.Semantics.OperationWalker.VisitVariableDeclarati override Microsoft.CodeAnalysis.Semantics.OperationWalker.VisitWhileUntilLoopStatement(Microsoft.CodeAnalysis.Semantics.IWhileUntilLoopStatement operation) -> void override Microsoft.CodeAnalysis.Semantics.OperationWalker.VisitWithStatement(Microsoft.CodeAnalysis.Semantics.IWithStatement operation) -> void override Microsoft.CodeAnalysis.Semantics.OperationWalker.VisitYieldBreakStatement(Microsoft.CodeAnalysis.Semantics.IReturnStatement operation) -> void +static Microsoft.CodeAnalysis.EmbeddedText.FromBytes(string filePath, System.ArraySegment bytes, Microsoft.CodeAnalysis.Text.SourceHashAlgorithm checksumAlgorithm = Microsoft.CodeAnalysis.Text.SourceHashAlgorithm.Sha1) -> Microsoft.CodeAnalysis.EmbeddedText +static Microsoft.CodeAnalysis.EmbeddedText.FromSource(string filePath, Microsoft.CodeAnalysis.Text.SourceText text) -> Microsoft.CodeAnalysis.EmbeddedText +static Microsoft.CodeAnalysis.EmbeddedText.FromStream(string filePath, System.IO.Stream stream, Microsoft.CodeAnalysis.Text.SourceHashAlgorithm checksumAlgorithm = Microsoft.CodeAnalysis.Text.SourceHashAlgorithm.Sha1) -> Microsoft.CodeAnalysis.EmbeddedText static Microsoft.CodeAnalysis.Semantics.OperationExtensions.Descendants(this Microsoft.CodeAnalysis.IOperation operation) -> System.Collections.Generic.IEnumerable static Microsoft.CodeAnalysis.Semantics.OperationExtensions.DescendantsAndSelf(this Microsoft.CodeAnalysis.IOperation operation) -> System.Collections.Generic.IEnumerable static Microsoft.CodeAnalysis.Semantics.OperationExtensions.GetRootOperation(this Microsoft.CodeAnalysis.ISymbol symbol, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> Microsoft.CodeAnalysis.IOperation @@ -782,6 +793,10 @@ static Microsoft.CodeAnalysis.Semantics.UnaryAndBinaryOperationExtensions.GetUna static Microsoft.CodeAnalysis.Semantics.UnaryAndBinaryOperationExtensions.GetUnaryOperandKind(this Microsoft.CodeAnalysis.Semantics.IUnaryOperatorExpression unary) -> Microsoft.CodeAnalysis.Semantics.UnaryOperandKind static Microsoft.CodeAnalysis.SeparatedSyntaxList.implicit operator Microsoft.CodeAnalysis.SeparatedSyntaxList(Microsoft.CodeAnalysis.SeparatedSyntaxList nodes) -> Microsoft.CodeAnalysis.SeparatedSyntaxList static Microsoft.CodeAnalysis.SeparatedSyntaxList.implicit operator Microsoft.CodeAnalysis.SeparatedSyntaxList(Microsoft.CodeAnalysis.SeparatedSyntaxList nodes) -> Microsoft.CodeAnalysis.SeparatedSyntaxList +static Microsoft.CodeAnalysis.Text.SourceText.From(System.IO.Stream stream, System.Text.Encoding encoding = null, Microsoft.CodeAnalysis.Text.SourceHashAlgorithm checksumAlgorithm = Microsoft.CodeAnalysis.Text.SourceHashAlgorithm.Sha1, bool throwIfBinaryDetected = false, bool canBeEmbedded = false) -> Microsoft.CodeAnalysis.Text.SourceText +static Microsoft.CodeAnalysis.Text.SourceText.From(System.IO.Stream stream, System.Text.Encoding encoding, Microsoft.CodeAnalysis.Text.SourceHashAlgorithm checksumAlgorithm, bool throwIfBinaryDetected) -> Microsoft.CodeAnalysis.Text.SourceText +static Microsoft.CodeAnalysis.Text.SourceText.From(byte[] buffer, int length, System.Text.Encoding encoding = null, Microsoft.CodeAnalysis.Text.SourceHashAlgorithm checksumAlgorithm = Microsoft.CodeAnalysis.Text.SourceHashAlgorithm.Sha1, bool throwIfBinaryDetected = false, bool canBeEmbedded = false) -> Microsoft.CodeAnalysis.Text.SourceText +static Microsoft.CodeAnalysis.Text.SourceText.From(byte[] buffer, int length, System.Text.Encoding encoding, Microsoft.CodeAnalysis.Text.SourceHashAlgorithm checksumAlgorithm, bool throwIfBinaryDetected) -> Microsoft.CodeAnalysis.Text.SourceText virtual Microsoft.CodeAnalysis.Diagnostics.AnalysisContext.RegisterOperationAction(System.Action action, System.Collections.Immutable.ImmutableArray operationKinds) -> void virtual Microsoft.CodeAnalysis.Diagnostics.AnalysisContext.RegisterOperationBlockAction(System.Action action) -> void virtual Microsoft.CodeAnalysis.Diagnostics.AnalysisContext.RegisterOperationBlockStartAction(System.Action action) -> void diff --git a/src/Compilers/Core/Portable/Syntax/SyntaxTree.cs b/src/Compilers/Core/Portable/Syntax/SyntaxTree.cs index 9038279c2aa15..2f94ed99ca27b 100644 --- a/src/Compilers/Core/Portable/Syntax/SyntaxTree.cs +++ b/src/Compilers/Core/Portable/Syntax/SyntaxTree.cs @@ -313,7 +313,10 @@ internal int GetDisplayLineNumber(TextSpan span) /// this tree. public abstract IList GetChanges(SyntaxTree oldTree); - internal ValueTuple, Guid> GetChecksumAndAlgorithm() + /// + /// Gets the checksum + algorithm id to use in the PDB. + /// + internal Cci.DebugSourceInfo GetDebugSourceInfo() { if (_lazyChecksum.IsDefault) { @@ -323,13 +326,11 @@ internal ValueTuple, Guid> GetChecksumAndAlgorithm() } Debug.Assert(!_lazyChecksum.IsDefault); - Guid guid; - if (!Cci.DebugSourceDocument.TryGetAlgorithmGuid(_lazyHashAlgorithm, out guid)) - { - throw ExceptionUtilities.Unreachable; - } + Debug.Assert(_lazyHashAlgorithm != default(SourceHashAlgorithm)); - return ValueTuple.Create(_lazyChecksum, guid); + // NOTE: If this tree is to be embedded, it's debug source info should have + // been obtained via EmbeddedText.GetDebugSourceInfo() and not here. + return new Cci.DebugSourceInfo(_lazyChecksum, _lazyHashAlgorithm); } /// diff --git a/src/Compilers/Core/Portable/Text/LargeText.cs b/src/Compilers/Core/Portable/Text/LargeText.cs index 31e920c42217d..11941bc67f3fc 100644 --- a/src/Compilers/Core/Portable/Text/LargeText.cs +++ b/src/Compilers/Core/Portable/Text/LargeText.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading; using Roslyn.Utilities; +using System.Diagnostics; namespace Microsoft.CodeAnalysis.Text { @@ -25,8 +26,8 @@ internal sealed class LargeText : SourceText private readonly int _length; private readonly Encoding _encoding; - internal LargeText(ImmutableArray chunks, Encoding encoding, ImmutableArray checksum, SourceHashAlgorithm checksumAlgorithm) - : base(checksum, checksumAlgorithm) + internal LargeText(ImmutableArray chunks, Encoding encoding, ImmutableArray checksum, SourceHashAlgorithm checksumAlgorithm, ImmutableArray embeddedTextBlob) + : base(checksum, checksumAlgorithm, embeddedTextBlob) { _chunks = chunks; _encoding = encoding; @@ -41,17 +42,19 @@ internal LargeText(ImmutableArray chunks, Encoding encoding, ImmutableAr _length = offset; } - internal static SourceText Decode(Stream stream, Encoding encoding, SourceHashAlgorithm checksumAlgorithm, bool throwIfBinaryDetected) + internal static SourceText Decode(Stream stream, Encoding encoding, SourceHashAlgorithm checksumAlgorithm, bool throwIfBinaryDetected, bool canBeEmbedded) { stream.Seek(0, SeekOrigin.Begin); - int length = (int)stream.Length; - if (length == 0) + long longLength = stream.Length; + if (longLength == 0) { return SourceText.From(string.Empty, encoding, checksumAlgorithm); } - var maxCharRemainingGuess = encoding.GetMaxCharCount(length); + var maxCharRemainingGuess = encoding.GetMaxCharCountOrThrowIfHuge(stream); + Debug.Assert(longLength > 0 && longLength <= int.MaxValue); // GetMaxCharCountOrThrowIfHuge should have thrown. + int length = (int)longLength; using (var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true, bufferSize: Math.Min(length, 4096), leaveOpen: true)) { @@ -91,8 +94,11 @@ internal static SourceText Decode(Stream stream, Encoding encoding, SourceHashAl chunks.Add(chunk); } + // We must compute the checksum and embedded text blob now while we still have the original bytes in hand. + // We cannot re-encode to obtain checksum and blob as the encoding is not guaranteed to round-trip. var checksum = CalculateChecksum(stream, checksumAlgorithm); - return new LargeText(chunks.ToImmutableAndFree(), reader.CurrentEncoding, checksum, checksumAlgorithm); + var embeddedTextBlob = canBeEmbedded ? EmbeddedText.CreateBlob(stream) : default(ImmutableArray); + return new LargeText(chunks.ToImmutableAndFree(), reader.CurrentEncoding, checksum, checksumAlgorithm, embeddedTextBlob); } } diff --git a/src/Compilers/Core/Portable/Text/LargeTextWriter.cs b/src/Compilers/Core/Portable/Text/LargeTextWriter.cs index 2882cb5b4da2e..ee0ec545821e8 100644 --- a/src/Compilers/Core/Portable/Text/LargeTextWriter.cs +++ b/src/Compilers/Core/Portable/Text/LargeTextWriter.cs @@ -27,7 +27,7 @@ public LargeTextWriter(Encoding encoding, SourceHashAlgorithm checksumAlgorithm, public override SourceText ToSourceText() { this.Flush(); - return new LargeText(_chunks.ToImmutableAndFree(), _encoding, default(ImmutableArray), _checksumAlgorithm); + return new LargeText(_chunks.ToImmutableAndFree(), _encoding, default(ImmutableArray), _checksumAlgorithm, default(ImmutableArray)); } public override Encoding Encoding diff --git a/src/Compilers/Core/Portable/Text/SourceText.cs b/src/Compilers/Core/Portable/Text/SourceText.cs index ff99b92a6d3bc..1ddbdfe67b926 100644 --- a/src/Compilers/Core/Portable/Text/SourceText.cs +++ b/src/Compilers/Core/Portable/Text/SourceText.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.ComponentModel; using System.Diagnostics; using System.IO; using System.Linq; @@ -27,7 +28,8 @@ public abstract class SourceText private SourceTextContainer _lazyContainer; private TextLineCollection _lazyLineInfo; private ImmutableArray _lazyChecksum; - + private ImmutableArray _precomputedEmbeddedTextBlob; + private static readonly Encoding s_utf8EncodingWithNoBOM = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: false); protected SourceText(ImmutableArray checksum = default(ImmutableArray), SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1, SourceTextContainer container = null) @@ -44,6 +46,24 @@ public abstract class SourceText _lazyContainer = container; } + internal SourceText(ImmutableArray checksum, SourceHashAlgorithm checksumAlgorithm, ImmutableArray embeddedTextBlob) + : this(checksum, checksumAlgorithm, container: null) + { + // We should never have precomputed the embedded text blob without precomputing the checksum. + Debug.Assert(embeddedTextBlob.IsDefault || !checksum.IsDefault); + + if (!checksum.IsDefault && embeddedTextBlob.IsDefault) + { + // We can't compute the embedded text blob lazily if we're given a precomputed checksum. + // This happens when source bytes/stream were given, but canBeEmbedded=true was not passed. + _precomputedEmbeddedTextBlob = ImmutableArray.Empty; + } + else + { + _precomputedEmbeddedTextBlob = embeddedTextBlob; + } + } + internal static void ValidateChecksumAlgorithm(SourceHashAlgorithm checksumAlgorithm) { if (!Cci.DebugSourceDocument.IsSupportedAlgorithm(checksumAlgorithm)) @@ -77,6 +97,11 @@ public static SourceText From(string text, Encoding encoding = null, SourceHashA return new StringText(text, encoding, checksumAlgorithm: checksumAlgorithm); } + // 1.0 BACKCOMPAT OVERLOAD - DO NOT TOUCH + [EditorBrowsable(EditorBrowsableState.Never)] + public static SourceText From(Stream stream, Encoding encoding, SourceHashAlgorithm checksumAlgorithm, bool throwIfBinaryDetected) + => From(stream, encoding, checksumAlgorithm, throwIfBinaryDetected, canBeEmbedded: false); + /// /// Constructs a from stream content. /// @@ -90,6 +115,7 @@ public static SourceText From(string text, Encoding encoding = null, SourceHashA /// /// If the decoded text contains at least two consecutive NUL /// characters, then an is thrown. + /// True if the text can be passed to and be embedded in a PDB. /// is null. /// /// doesn't support reading or seeking. @@ -99,7 +125,12 @@ public static SourceText From(string text, Encoding encoding = null, SourceHashA /// Two consecutive NUL characters were detected in the decoded text and was true. /// An I/O error occurs. /// Reads from the beginning of the stream. Leaves the stream open. - public static SourceText From(Stream stream, Encoding encoding = null, SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1, bool throwIfBinaryDetected = false) + public static SourceText From( + Stream stream, + Encoding encoding = null, + SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1, + bool throwIfBinaryDetected = false, + bool canBeEmbedded = false) { if (stream == null) { @@ -116,9 +147,9 @@ public static SourceText From(Stream stream, Encoding encoding = null, SourceHas encoding = encoding ?? s_utf8EncodingWithNoBOM; // If the resulting string would end up on the large object heap, then use LargeEncodedText. - if (encoding.GetMaxCharCount((int)stream.Length) >= LargeObjectHeapLimitInChars) + if (encoding.GetMaxCharCountOrThrowIfHuge(stream) >= LargeObjectHeapLimitInChars) { - return LargeText.Decode(stream, encoding, checksumAlgorithm, throwIfBinaryDetected); + return LargeText.Decode(stream, encoding, checksumAlgorithm, throwIfBinaryDetected, canBeEmbedded); } string text = Decode(stream, encoding, out encoding); @@ -127,10 +158,18 @@ public static SourceText From(Stream stream, Encoding encoding = null, SourceHas throw new InvalidDataException(); } + // We must compute the checksum and embedded text blob now while we still have the original bytes in hand. + // We cannot re-encode to obtain checksum and blob as the encoding is not guaranteed to round-trip. var checksum = CalculateChecksum(stream, checksumAlgorithm); - return new StringText(text, encoding, checksum, checksumAlgorithm); + var embeddedTextBlob = canBeEmbedded ? EmbeddedText.CreateBlob(stream) : default(ImmutableArray); + return new StringText(text, encoding, checksum, checksumAlgorithm, embeddedTextBlob); } + // 1.0 BACKCOMPAT OVERLOAD - DO NOT TOUCH + [EditorBrowsable(EditorBrowsableState.Never)] + public static SourceText From(byte[] buffer, int length, Encoding encoding, SourceHashAlgorithm checksumAlgorithm, bool throwIfBinaryDetected) + => From(buffer, length, encoding, checksumAlgorithm, throwIfBinaryDetected, canBeEmbedded: false); + /// /// Constructs a from a byte array. /// @@ -146,12 +185,19 @@ public static SourceText From(Stream stream, Encoding encoding = null, SourceHas /// If the decoded text contains at least two consecutive NUL /// characters, then an is thrown. /// The decoded text. + /// True if the text can be passed to and be embedded in a PDB. /// The is null. /// The is negative or longer than the . /// is not supported. /// If the given encoding is set to use a throwing decoder as a fallback /// Two consecutive NUL characters were detected in the decoded text and was true. - public static SourceText From(byte[] buffer, int length, Encoding encoding = null, SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1, bool throwIfBinaryDetected = false) + public static SourceText From( + byte[] buffer, + int length, + Encoding encoding = null, + SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1, + bool throwIfBinaryDetected = false, + bool canBeEmbedded = false) { if (buffer == null) { @@ -171,9 +217,11 @@ public static SourceText From(byte[] buffer, int length, Encoding encoding = nul throw new InvalidDataException(); } - // Since we have the bytes in hand, it's easy to compute the checksum. + // We must compute the checksum and embedded text blob now while we still have the original bytes in hand. + // We cannot re-encode to obtain checksum and blob as the encoding is not guaranteed to round-trip. var checksum = CalculateChecksum(buffer, 0, length, checksumAlgorithm); - return new StringText(text, encoding, checksum, checksumAlgorithm); + var embeddedTextBlob = canBeEmbedded ? EmbeddedText.CreateBlob(new ArraySegment(buffer, 0, length)) : default(ImmutableArray); + return new StringText(text, encoding, checksum, checksumAlgorithm, embeddedTextBlob); } /// @@ -299,6 +347,42 @@ internal virtual SourceText StorageKey get { return this; } } + /// + /// Indicates whether this source text can be embedded in the PDB. + /// + /// + /// If this text was constructed via or + /// , then the canBeEmbedded arg must have + /// been true. + /// + /// Otherwise, must be non-null. + /// + public bool CanBeEmbedded + { + get + { + if (_precomputedEmbeddedTextBlob.IsDefault) + { + // If we didn't precompute the embedded text blob from bytes/stream, + // we can only support embedding if we have an encoding with which + // to encode the text in the PDB. + return Encoding != null; + } + + // We use a sentinel empty blob to indicate that embedding has been disallowed. + return !_precomputedEmbeddedTextBlob.IsEmpty; + } + } + + /// + /// If the text was created from a stream or byte[] and canBeEmbedded argument was true, + /// this provides the embedded text blob that was precomputed using the original stream + /// or byte[]. The precomputation was required in that case so that the bytes written to + /// the PDB match the original bytes exactly (and match the checksum of the original + /// bytes). + /// + internal ImmutableArray PrecomputedEmbeddedTextBlob => _precomputedEmbeddedTextBlob; + /// /// Returns a character at given position. /// @@ -428,7 +512,7 @@ public ImmutableArray GetChecksum() return _lazyChecksum; } - private static ImmutableArray CalculateChecksum(byte[] buffer, int offset, int count, SourceHashAlgorithm algorithmId) + internal static ImmutableArray CalculateChecksum(byte[] buffer, int offset, int count, SourceHashAlgorithm algorithmId) { using (var algorithm = CryptographicHashProvider.TryGetAlgorithm(algorithmId)) { diff --git a/src/Compilers/Core/Portable/Text/StringText.cs b/src/Compilers/Core/Portable/Text/StringText.cs index 767b144425f9a..f064925320a82 100644 --- a/src/Compilers/Core/Portable/Text/StringText.cs +++ b/src/Compilers/Core/Portable/Text/StringText.cs @@ -17,8 +17,13 @@ internal sealed class StringText : SourceText private readonly string _source; private readonly Encoding _encodingOpt; - internal StringText(string source, Encoding encodingOpt, ImmutableArray checksum = default(ImmutableArray), SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1) - : base(checksum, checksumAlgorithm) + internal StringText( + string source, + Encoding encodingOpt, + ImmutableArray checksum = default(ImmutableArray), + SourceHashAlgorithm checksumAlgorithm = SourceHashAlgorithm.Sha1, + ImmutableArray embeddedTextBlob = default(ImmutableArray)) + : base(checksum, checksumAlgorithm, embeddedTextBlob) { Debug.Assert(source != null); diff --git a/src/Compilers/VisualBasic/Portable/CommandLine/VisualBasicCommandLineParser.vb b/src/Compilers/VisualBasic/Portable/CommandLine/VisualBasicCommandLineParser.vb index 457484dbd1276..a98c428276f43 100644 --- a/src/Compilers/VisualBasic/Portable/CommandLine/VisualBasicCommandLineParser.vb +++ b/src/Compilers/VisualBasic/Portable/CommandLine/VisualBasicCommandLineParser.vb @@ -1143,7 +1143,7 @@ lVbRuntimePlus: Continue For End If - additionalFiles.AddRange(ParseAdditionalFileArgument(value, baseDirectory, diagnostics)) + additionalFiles.AddRange(ParseSeparatedFileArgument(value, baseDirectory, diagnostics)) Continue For End Select End If @@ -1365,7 +1365,8 @@ lVbRuntimePlus: .SourceLink = sourceLink, .DefaultCoreLibraryReference = defaultCoreLibraryReference, .PreferredUILang = preferredUILang, - .ReportAnalyzer = reportAnalyzer + .ReportAnalyzer = reportAnalyzer, + .EmbeddedFiles = ImmutableArray(Of CommandLineSourceFile).Empty } End Function diff --git a/src/Compilers/VisualBasic/Portable/CommandLine/VisualBasicCompiler.vb b/src/Compilers/VisualBasic/Portable/CommandLine/VisualBasicCompiler.vb index ce8c9c0bdcdb1..718a80f33e82a 100644 --- a/src/Compilers/VisualBasic/Portable/CommandLine/VisualBasicCompiler.vb +++ b/src/Compilers/VisualBasic/Portable/CommandLine/VisualBasicCompiler.vb @@ -3,6 +3,7 @@ Imports System.Collections.Immutable Imports System.IO Imports System.Threading.Tasks +Imports Microsoft.CodeAnalysis.Collections Imports Microsoft.CodeAnalysis.Diagnostics Namespace Microsoft.CodeAnalysis.VisualBasic @@ -54,7 +55,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic errorLogger As ErrorLogger) As SyntaxTree Dim fileReadDiagnostics As New List(Of DiagnosticInfo)() - Dim content = ReadFileContent(file, fileReadDiagnostics) + Dim content = TryReadFileContent(file, fileReadDiagnostics) If content Is Nothing Then ReportErrors(fileReadDiagnostics, consoleOutput, errorLogger) @@ -206,6 +207,12 @@ Namespace Microsoft.CodeAnalysis.VisualBasic messageProvider As CommonMessageProvider) As ImmutableArray(Of DiagnosticAnalyzer) Return Arguments.ResolveAnalyzersFromArguments(LanguageNames.VisualBasic, diagnostics, messageProvider, AssemblyLoader) End Function + + Protected Overrides Sub ResolveEmbeddedFilesFromExternalSourceDirectives(tree As SyntaxTree, resolver As SourceReferenceResolver, embeddedFiles As OrderedSet(Of String), diagnostics As IList(Of Diagnostic)) + ' Embedded files not yet supported by VB and we should not get this far. + Debug.Assert(False) + Throw ExceptionUtilities.Unreachable + End Sub End Class End Namespace diff --git a/src/Compilers/VisualBasic/Portable/Compilation/VisualBasicCompilation.vb b/src/Compilers/VisualBasic/Portable/Compilation/VisualBasicCompilation.vb index e0965afcede90..4215b91863849 100644 --- a/src/Compilers/VisualBasic/Portable/Compilation/VisualBasicCompilation.vb +++ b/src/Compilers/VisualBasic/Portable/Compilation/VisualBasicCompilation.vb @@ -2111,15 +2111,21 @@ Namespace Microsoft.CodeAnalysis.VisualBasic emitOptions As EmitOptions, debugEntryPoint As IMethodSymbol, sourceLinkStream As Stream, + embeddedTexts As IEnumerable(Of EmbeddedText), manifestResources As IEnumerable(Of ResourceDescription), testData As CompilationTestData, diagnostics As DiagnosticBag, cancellationToken As CancellationToken) As CommonPEModuleBuilder + If embeddedTexts?.Any() Then + Throw New ArgumentException(VBResources.EmbeddedTextsNotSupported, NameOf(embeddedTexts)) + End If + Return CreateModuleBuilder( emitOptions, debugEntryPoint, sourceLinkStream, + embeddedTexts, manifestResources, testData, diagnostics, @@ -2131,6 +2137,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic emitOptions As EmitOptions, debugEntryPoint As IMethodSymbol, sourceLinkStream As Stream, + embeddedTexts As IEnumerable(Of EmbeddedText), manifestResources As IEnumerable(Of ResourceDescription), testData As CompilationTestData, diagnostics As DiagnosticBag, @@ -2219,7 +2226,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic SynthesizedMetadataCompiler.ProcessSynthesizedMembers(Me, moduleBeingBuilt, cancellationToken) Else ' start generating PDB checksums if we need to emit PDBs - If emittingPdb AndAlso Not StartSourceChecksumCalculation(moduleBeingBuilt.DebugDocumentsBuilder, diagnostics) Then + If emittingPdb AndAlso Not StartSourceChecksumCalculation(moduleBeingBuilt.DebugDocumentsBuilder, moduleBeingBuilt.EmbeddedTexts, diagnostics) Then Return False End If @@ -2356,9 +2363,11 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Continue For End If - If CheckSumMatches(checkSumText, existingDoc.ChecksumAndAlgorithm.Item1) Then + Dim sourceInfo = existingDoc.GetSourceInfo() + + If CheckSumMatches(checkSumText, sourceInfo.Checksum) Then Dim guid As Guid = Guid.Parse(checksumDirective.Guid.ValueText) - If guid = existingDoc.ChecksumAndAlgorithm.Item2 Then + If guid = sourceInfo.ChecksumAlgorithmId Then ' all parts match, nothing to do Continue For End If diff --git a/src/Compilers/VisualBasic/Portable/Errors/MessageProvider.vb b/src/Compilers/VisualBasic/Portable/Errors/MessageProvider.vb index 5fef82f91cb31..efd500ca48bb8 100644 --- a/src/Compilers/VisualBasic/Portable/Errors/MessageProvider.vb +++ b/src/Compilers/VisualBasic/Portable/Errors/MessageProvider.vb @@ -95,6 +95,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Return New VBDiagnostic(ErrorFactory.ErrorInfo(CType(code, ERRID), args), location) End Function + Public Overrides Function CreateDiagnostic(info As DiagnosticInfo) As Diagnostic + Return New VBDiagnostic(info, Location.None) + End Function + Public Overrides Function GetErrorDisplayString(symbol As ISymbol) As String ' show extra info for assembly if possible such as version, public key token etc. If symbol.Kind = SymbolKind.Assembly OrElse symbol.Kind = SymbolKind.Namespace Then diff --git a/src/Compilers/VisualBasic/Portable/VBResources.Designer.vb b/src/Compilers/VisualBasic/Portable/VBResources.Designer.vb index 3311e34b2ddc5..80d1e76345b94 100644 --- a/src/Compilers/VisualBasic/Portable/VBResources.Designer.vb +++ b/src/Compilers/VisualBasic/Portable/VBResources.Designer.vb @@ -160,6 +160,15 @@ Namespace Microsoft.CodeAnalysis.VisualBasic End Get End Property + ''' + ''' Looks up a localized string similar to PDB embedded texts are not supported in VB.. + ''' + Friend ReadOnly Property EmbeddedTextsNotSupported() As String + Get + Return ResourceManager.GetString("EmbeddedTextsNotSupported", resourceCulture) + End Get + End Property + ''' ''' Looks up a localized string similar to Cannot find the interop type that matches the embedded type '{0}'. Are you missing an assembly reference?. ''' diff --git a/src/Compilers/VisualBasic/Portable/VBResources.resx b/src/Compilers/VisualBasic/Portable/VBResources.resx index ffb97f12f1122..00bbc09f0cd4f 100644 --- a/src/Compilers/VisualBasic/Portable/VBResources.resx +++ b/src/Compilers/VisualBasic/Portable/VBResources.resx @@ -5387,4 +5387,7 @@ An expression tree may not contain a call to a method or property that returns by reference. - + + PDB embedded texts are not supported in VB. + + \ No newline at end of file diff --git a/src/Compilers/VisualBasic/Test/CommandLine/CommandLineTests.vb b/src/Compilers/VisualBasic/Test/CommandLine/CommandLineTests.vb index 2ef9b3656479c..1494b2fa3f024 100644 --- a/src/Compilers/VisualBasic/Test/CommandLine/CommandLineTests.vb +++ b/src/Compilers/VisualBasic/Test/CommandLine/CommandLineTests.vb @@ -2524,6 +2524,20 @@ End Class") CleanupAllGeneratedFiles(src.Path) End Sub + + Public Sub Embed_NotYetSupportedByVB() + Dim parsedArgs = DefaultParse({"a.vb"}, _baseDirectory) + parsedArgs.Errors.Verify() + Assert.Empty(parsedArgs.EmbeddedFiles) + + parsedArgs = DefaultParse({"/embed", "/debug:portable", "a.vb"}, _baseDirectory) + parsedArgs.Errors.Verify(Diagnostic(ERRID.WRN_BadSwitch).WithArguments("/embed")) + Assert.Empty(parsedArgs.EmbeddedFiles) + + parsedArgs = DefaultParse({"/embed:a.vb", "/debug:portable", "a.vb"}, _baseDirectory) + parsedArgs.Errors.Verify(Diagnostic(ERRID.WRN_BadSwitch).WithArguments("/embed:a.vb")) + Assert.Empty(parsedArgs.EmbeddedFiles) + End Sub diff --git a/src/Compilers/VisualBasic/Test/Semantic/Compilation/CompilationAPITests.vb b/src/Compilers/VisualBasic/Test/Semantic/Compilation/CompilationAPITests.vb index ebeda6d905919..bac174683c9d5 100644 --- a/src/Compilers/VisualBasic/Test/Semantic/Compilation/CompilationAPITests.vb +++ b/src/Compilers/VisualBasic/Test/Semantic/Compilation/CompilationAPITests.vb @@ -242,34 +242,60 @@ End Namespace Public Sub Emit_BadArgs() Dim comp = VisualBasicCompilation.Create("Compilation", options:=TestOptions.ReleaseDll) - Assert.Throws(Of ArgumentNullException)(Sub() comp.Emit(peStream:=Nothing)) - Assert.Throws(Of ArgumentException)(Sub() comp.Emit(peStream:=New TestStream(canRead:=True, canWrite:=False, canSeek:=True))) - Assert.Throws(Of ArgumentException)(Sub() comp.Emit(peStream:=New MemoryStream(), pdbStream:=New TestStream(canRead:=True, canWrite:=False, canSeek:=True))) - Assert.Throws(Of ArgumentException)(Sub() comp.Emit(peStream:=New MemoryStream(), pdbStream:=New MemoryStream(), options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Embedded))) + Assert.Throws(Of ArgumentNullException)("peStream", Sub() comp.Emit(peStream:=Nothing)) + Assert.Throws(Of ArgumentException)("peStream", Sub() comp.Emit(peStream:=New TestStream(canRead:=True, canWrite:=False, canSeek:=True))) + Assert.Throws(Of ArgumentException)("pdbStream", Sub() comp.Emit(peStream:=New MemoryStream(), pdbStream:=New TestStream(canRead:=True, canWrite:=False, canSeek:=True))) + Assert.Throws(Of ArgumentException)("pdbStream", Sub() comp.Emit(peStream:=New MemoryStream(), pdbStream:=New MemoryStream(), options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Embedded))) - Assert.Throws(Of ArgumentException)(Sub() comp.Emit( + Assert.Throws(Of ArgumentException)("sourceLinkStream", Sub() comp.Emit( peStream:=New MemoryStream(), pdbStream:=New MemoryStream(), options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), sourceLinkStream:=New TestStream(canRead:=False, canWrite:=True, canSeek:=True))) - Assert.Throws(Of ArgumentException)(Sub() comp.Emit( + Assert.Throws(Of ArgumentException)("sourceLinkStream", Sub() comp.Emit( peStream:=New MemoryStream(), pdbStream:=New MemoryStream(), options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Pdb), sourceLinkStream:=New MemoryStream())) - Assert.Throws(Of ArgumentException)(Sub() comp.Emit( + Assert.Throws(Of ArgumentException)("sourceLinkStream", Sub() comp.Emit( peStream:=New MemoryStream(), pdbStream:=Nothing, options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), sourceLinkStream:=New MemoryStream())) - Assert.Throws(Of ArgumentException)(Sub() comp.Emit( + Assert.Throws(Of ArgumentException)("embeddedTexts", Sub() comp.Emit( + peStream:=New MemoryStream(), + pdbStream:=New MemoryStream(), + options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Pdb), + embeddedTexts:={EmbeddedText.FromStream("_", New MemoryStream())})) + + Assert.Throws(Of ArgumentException)("embeddedTexts", Sub() comp.Emit( + peStream:=New MemoryStream(), + pdbStream:=Nothing, + options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), + embeddedTexts:={EmbeddedText.FromStream("_", New MemoryStream())})) + + ' EmbeddedText not supported at all in VB for now + Assert.Throws(Of ArgumentException)("embeddedTexts", Sub() comp.Emit( + peStream:=New MemoryStream(), + pdbStream:=New MemoryStream(), + options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.PortablePdb), + embeddedTexts:={EmbeddedText.FromStream("_", New MemoryStream())})) + + ' EmbeddedText not supported at all in VB for now + Assert.Throws(Of ArgumentException)("embeddedTexts", Sub() comp.Emit( + peStream:=New MemoryStream(), + pdbStream:=Nothing, + options:=EmitOptions.Default.WithDebugInformationFormat(DebugInformationFormat.Embedded), + embeddedTexts:={EmbeddedText.FromStream("_", New MemoryStream())})) + + Assert.Throws(Of ArgumentException)("win32Resources", Sub() comp.Emit( peStream:=New MemoryStream(), win32Resources:=New TestStream(canRead:=True, canWrite:=False, canSeek:=False))) - Assert.Throws(Of ArgumentException)(Sub() comp.Emit( + Assert.Throws(Of ArgumentException)("win32Resources", Sub() comp.Emit( peStream:=New MemoryStream(), win32Resources:=New TestStream(canRead:=False, canWrite:=False, canSeek:=True))) diff --git a/src/Dependencies/CodeAnalysis.Metadata/PortableCustomDebugInfoKinds.cs b/src/Dependencies/CodeAnalysis.Metadata/PortableCustomDebugInfoKinds.cs index 629a1490cd188..ef45f2085fd76 100644 --- a/src/Dependencies/CodeAnalysis.Metadata/PortableCustomDebugInfoKinds.cs +++ b/src/Dependencies/CodeAnalysis.Metadata/PortableCustomDebugInfoKinds.cs @@ -13,5 +13,6 @@ internal static class PortableCustomDebugInfoKinds public static readonly Guid EncLocalSlotMap = new Guid("755F52A8-91C5-45BE-B4B8-209571E552BD"); public static readonly Guid EncLambdaAndClosureMap = new Guid("A643004C-0240-496F-A783-30D64F4979DE"); public static readonly Guid SourceLink = new Guid("CC110556-A091-4D38-9FEC-25AB9A351A6A"); + public static readonly Guid EmbeddedSource = new Guid("0E8A571B-6926-466E-B4AD-8AB04611F5FE"); } } diff --git a/src/NuGet/Microsoft.CodeAnalysis.Common.nuspec b/src/NuGet/Microsoft.CodeAnalysis.Common.nuspec index e783935500fa3..f4ad03f6ddccc 100644 --- a/src/NuGet/Microsoft.CodeAnalysis.Common.nuspec +++ b/src/NuGet/Microsoft.CodeAnalysis.Common.nuspec @@ -32,6 +32,7 @@ + diff --git a/src/Scripting/Core/Hosting/CommandLine/CommandLineRunner.cs b/src/Scripting/Core/Hosting/CommandLine/CommandLineRunner.cs index ee96cc0cae09a..5e4989ca5afc8 100644 --- a/src/Scripting/Core/Hosting/CommandLine/CommandLineRunner.cs +++ b/src/Scripting/Core/Hosting/CommandLine/CommandLineRunner.cs @@ -98,7 +98,7 @@ private int RunInteractiveCore(ErrorLogger errorLogger) } else { - code = _compiler.ReadFileContent(sourceFiles[0], diagnosticsInfos); + code = _compiler.TryReadFileContent(sourceFiles[0], diagnosticsInfos); } } diff --git a/src/Test/Utilities/Shared/Compilation/CompilationExtensions.cs b/src/Test/Utilities/Shared/Compilation/CompilationExtensions.cs index 5bd8cc4cd253f..6e5468e9e131a 100644 --- a/src/Test/Utilities/Shared/Compilation/CompilationExtensions.cs +++ b/src/Test/Utilities/Shared/Compilation/CompilationExtensions.cs @@ -26,7 +26,8 @@ internal static ImmutableArray EmitToArray( DiagnosticDescription[] expectedWarnings = null, Stream pdbStream = null, IMethodSymbol debugEntryPoint = null, - Stream sourceLinkStream = null) + Stream sourceLinkStream = null, + IEnumerable embeddedTexts = null) { var peStream = new MemoryStream(); @@ -49,6 +50,7 @@ internal static ImmutableArray EmitToArray( options: options, debugEntryPoint: debugEntryPoint, sourceLinkStream: sourceLinkStream, + embeddedTexts: embeddedTexts, testData: testData, cancellationToken: default(CancellationToken)); diff --git a/src/Test/Utilities/Shared/Compilation/IRuntimeEnvironment.cs b/src/Test/Utilities/Shared/Compilation/IRuntimeEnvironment.cs index 6aab781ce5c61..11103f395defd 100644 --- a/src/Test/Utilities/Shared/Compilation/IRuntimeEnvironment.cs +++ b/src/Test/Utilities/Shared/Compilation/IRuntimeEnvironment.cs @@ -217,6 +217,7 @@ EmitOptions emitOptions options: emitOptions ?? EmitOptions.Default, debugEntryPoint: null, sourceLinkStream: null, + embeddedTexts: null, testData: testData, cancellationToken: default(CancellationToken)); } diff --git a/src/Test/Utilities/Shared/Metadata/MetadataReaderUtils.cs b/src/Test/Utilities/Shared/Metadata/MetadataReaderUtils.cs index 00797669d9566..67f448f37800a 100644 --- a/src/Test/Utilities/Shared/Metadata/MetadataReaderUtils.cs +++ b/src/Test/Utilities/Shared/Metadata/MetadataReaderUtils.cs @@ -11,6 +11,9 @@ using Microsoft.CodeAnalysis; using Roslyn.Utilities; using Microsoft.CodeAnalysis.Debugging; +using Microsoft.CodeAnalysis.Text; +using System.IO; +using System.IO.Compression; namespace Roslyn.Test.Utilities { @@ -204,5 +207,43 @@ public static ImmutableArray GetSourceLinkBlob(this MetadataReader reader) where reader.GetGuid(cdi.Kind) == PortableCustomDebugInfoKinds.SourceLink select reader.GetBlobContent(cdi.Value)).Single(); } + + public static SourceText GetEmbeddedSource(this MetadataReader reader, DocumentHandle document) + { + byte[] bytes = (from handle in reader.GetCustomDebugInformation(document) + let cdi = reader.GetCustomDebugInformation(handle) + where reader.GetGuid(cdi.Kind) == PortableCustomDebugInfoKinds.EmbeddedSource + select reader.GetBlobBytes(cdi.Value)).SingleOrDefault(); + + if (bytes == null) + { + return null; + } + + int uncompressedSize = BitConverter.ToInt32(bytes, 0); + var stream = new MemoryStream(bytes, sizeof(int), bytes.Length - sizeof(int)); + + if (uncompressedSize != 0) + { + var decompressed = new MemoryStream(uncompressedSize); + + using (var deflater = new DeflateStream(stream, CompressionMode.Decompress)) + { + deflater.CopyTo(decompressed); + } + + if (decompressed.Length != uncompressedSize) + { + throw new InvalidDataException(); + } + + stream = decompressed; + } + + using (stream) + { + return EncodedStringText.Create(stream); + } + } } } diff --git a/src/Test/Utilities/Shared/Mocks/TestMessageProvider.cs b/src/Test/Utilities/Shared/Mocks/TestMessageProvider.cs index 9e1cff85b3aab..efe3060a57907 100644 --- a/src/Test/Utilities/Shared/Mocks/TestMessageProvider.cs +++ b/src/Test/Utilities/Shared/Mocks/TestMessageProvider.cs @@ -19,6 +19,11 @@ public override Diagnostic CreateDiagnostic(int code, Location location, params throw new NotImplementedException(); } + public override Diagnostic CreateDiagnostic(DiagnosticInfo info) + { + throw new NotImplementedException(); + } + public override string GetMessagePrefix(string id, DiagnosticSeverity severity, bool isWarningAsError, CultureInfo culture) { throw new NotImplementedException(); diff --git a/src/Tools/CommonNetCoreReferences/project.json b/src/Tools/CommonNetCoreReferences/project.json index 4c1b608fd07e8..8f061a56bc520 100644 --- a/src/Tools/CommonNetCoreReferences/project.json +++ b/src/Tools/CommonNetCoreReferences/project.json @@ -16,6 +16,7 @@ "System.Diagnostics.Tools": "4.0.1", "System.Dynamic.Runtime": "4.0.11", "System.Globalization": "4.0.11", + "System.IO.Compression": "4.1.0", "System.IO.FileSystem": "4.0.1", "System.IO.FileSystem.Primitives": "4.0.1", "System.IO.FileSystem.Watcher": "4.0.0", diff --git a/src/Workspaces/Core/Desktop/Workspaces.Desktop.csproj b/src/Workspaces/Core/Desktop/Workspaces.Desktop.csproj index d751ab31a529d..705b6761606eb 100644 --- a/src/Workspaces/Core/Desktop/Workspaces.Desktop.csproj +++ b/src/Workspaces/Core/Desktop/Workspaces.Desktop.csproj @@ -38,6 +38,9 @@ InternalUtilities\StreamExtensions.cs + + InternalUtilities\EncodingExtensions.cs + InternalUtilities\CoreClrShim.cs diff --git a/src/Workspaces/Core/Portable/WorkspacesResources.Designer.cs b/src/Workspaces/Core/Portable/WorkspacesResources.Designer.cs index ee9a0034163e2..27642ba0ed79a 100644 --- a/src/Workspaces/Core/Portable/WorkspacesResources.Designer.cs +++ b/src/Workspaces/Core/Portable/WorkspacesResources.Designer.cs @@ -945,6 +945,15 @@ internal static string Specified_path_must_be_absolute { } } + /// + /// Looks up a localized string similar to Stream is too long.. + /// + internal static string Stream_is_too_long { + get { + return ResourceManager.GetString("Stream_is_too_long", resourceCulture); + } + } + /// /// Looks up a localized string similar to Supplied diagnostic cannot be null.. /// diff --git a/src/Workspaces/Core/Portable/WorkspacesResources.resx b/src/Workspaces/Core/Portable/WorkspacesResources.resx index a40cb0cdb6588..4b6eedf77ae4c 100644 --- a/src/Workspaces/Core/Portable/WorkspacesResources.resx +++ b/src/Workspaces/Core/Portable/WorkspacesResources.resx @@ -486,4 +486,7 @@ Show Stack Trace + + Stream is too long. + \ No newline at end of file