From 36e487dab65684bb71cc5040cb517db66bb4aba3 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Mon, 13 Aug 2018 21:40:22 -0500 Subject: [PATCH 1/9] Add diagnostic and code fix verification helpers --- .../ConvertToConditional.Test.csproj | 4 +- .../CodeActionProviderTestFixture.cs | 4 + .../UnitTestFramework/CodeFixVerifier`2.cs | 61 + .../CustomDiagnosticVerifier`1.cs | 23 + .../UnitTestFramework/DiagnosticResult.cs | 176 +++ .../UnitTestFramework/DiagnosticVerifier`1.cs | 79 ++ .../UnitTestFramework/GenericAnalyzerTest.cs | 1254 +++++++++++++++++ .../UnitTestFramework/MetadataReferences.cs | 55 + .../Roslyn.UnitTestFramework.csproj | 23 +- .../TestDiagnosticProvider.cs | 42 + .../TestXmlReferenceResolver.cs | 41 + 11 files changed, 1755 insertions(+), 7 deletions(-) create mode 100644 samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs create mode 100644 samples/Shared/UnitTestFramework/CustomDiagnosticVerifier`1.cs create mode 100644 samples/Shared/UnitTestFramework/DiagnosticResult.cs create mode 100644 samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs create mode 100644 samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs create mode 100644 samples/Shared/UnitTestFramework/MetadataReferences.cs create mode 100644 samples/Shared/UnitTestFramework/TestDiagnosticProvider.cs create mode 100644 samples/Shared/UnitTestFramework/TestXmlReferenceResolver.cs diff --git a/samples/CSharp/ConvertToConditional/ConvertToConditional.Test/ConvertToConditional.Test.csproj b/samples/CSharp/ConvertToConditional/ConvertToConditional.Test/ConvertToConditional.Test.csproj index 8ca7fa081..147ffe3b9 100644 --- a/samples/CSharp/ConvertToConditional/ConvertToConditional.Test/ConvertToConditional.Test.csproj +++ b/samples/CSharp/ConvertToConditional/ConvertToConditional.Test/ConvertToConditional.Test.csproj @@ -1,4 +1,4 @@ - + netcoreapp2.0 @@ -6,7 +6,7 @@ - + diff --git a/samples/Shared/UnitTestFramework/CodeActionProviderTestFixture.cs b/samples/Shared/UnitTestFramework/CodeActionProviderTestFixture.cs index e2a95a0a5..71e1b7414 100644 --- a/samples/Shared/UnitTestFramework/CodeActionProviderTestFixture.cs +++ b/samples/Shared/UnitTestFramework/CodeActionProviderTestFixture.cs @@ -23,9 +23,13 @@ protected Document CreateDocument(string code) // find these assemblies in the running process string[] simpleNames = { "mscorlib", "System.Core", "System" }; +#if !NETSTANDARD1_5 IEnumerable references = AppDomain.CurrentDomain.GetAssemblies() .Where(a => simpleNames.Contains(a.GetName().Name, StringComparer.OrdinalIgnoreCase)) .Select(a => MetadataReference.CreateFromFile(a.Location)); +#else + IEnumerable references = Enumerable.Empty(); +#endif return new AdhocWorkspace().CurrentSolution .AddProject(projectId, "TestProject", "TestProject", LanguageName) diff --git a/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs b/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs new file mode 100644 index 000000000..79c433fc1 --- /dev/null +++ b/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Roslyn.UnitTestFramework +{ + public static class CodeFixVerifier + where TAnalyzer : DiagnosticAnalyzer, new() + where TCodeFix : CodeFixProvider, new() + { + public static DiagnosticResult[] EmptyDiagnosticResults + => DiagnosticVerifier.EmptyDiagnosticResults; + + public static DiagnosticResult Diagnostic(string diagnosticId = null) + => DiagnosticVerifier.Diagnostic(diagnosticId); + + public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) + => DiagnosticVerifier.Diagnostic(descriptor); + + public static DiagnosticResult CompilerError(string errorIdentifier) + => DiagnosticVerifier.CompilerError(errorIdentifier); + + public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult expected, CancellationToken cancellationToken) + => DiagnosticVerifier.VerifyCSharpDiagnosticAsync(source, expected, cancellationToken); + + public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult[] expected, CancellationToken cancellationToken) + => DiagnosticVerifier.VerifyCSharpDiagnosticAsync(source, expected, cancellationToken); + + public static Task VerifyCSharpFixAsync(string source, DiagnosticResult expected, string fixedSource, CancellationToken cancellationToken) + => VerifyCSharpFixAsync(source, new[] { expected }, fixedSource, cancellationToken); + + public static Task VerifyCSharpFixAsync(string source, DiagnosticResult[] expected, string fixedSource, CancellationToken cancellationToken) + { + CSharpTest test = new CSharpTest + { + TestCode = source, + FixedCode = fixedSource, + }; + + test.ExpectedDiagnostics.AddRange(expected); + return test.RunAsync(cancellationToken); + } + + public class CSharpTest : DiagnosticVerifier.CSharpTest + { + protected override IEnumerable GetCodeFixProviders() + => new[] { new TCodeFix() }; + } + + public class VisualBasicTest : DiagnosticVerifier.VisualBasicTest + { + protected override IEnumerable GetCodeFixProviders() + => new[] { new TCodeFix() }; + } + } +} diff --git a/samples/Shared/UnitTestFramework/CustomDiagnosticVerifier`1.cs b/samples/Shared/UnitTestFramework/CustomDiagnosticVerifier`1.cs new file mode 100644 index 000000000..942b4ab11 --- /dev/null +++ b/samples/Shared/UnitTestFramework/CustomDiagnosticVerifier`1.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Roslyn.UnitTestFramework +{ + public static class CustomDiagnosticVerifier + where TAnalyzer : DiagnosticAnalyzer, new() + { + public static DiagnosticResult[] EmptyDiagnosticResults + => DiagnosticVerifier.EmptyDiagnosticResults; + + public static DiagnosticResult Diagnostic(string diagnosticId = null) + => DiagnosticVerifier.Diagnostic(diagnosticId); + + public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) + => DiagnosticVerifier.Diagnostic(descriptor); + + public static DiagnosticResult CompilerError(string errorIdentifier) + => DiagnosticVerifier.CompilerError(errorIdentifier); + } +} diff --git a/samples/Shared/UnitTestFramework/DiagnosticResult.cs b/samples/Shared/UnitTestFramework/DiagnosticResult.cs new file mode 100644 index 000000000..571f4064d --- /dev/null +++ b/samples/Shared/UnitTestFramework/DiagnosticResult.cs @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Immutable; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.Text; + +namespace Roslyn.UnitTestFramework +{ + /// + /// Structure that stores information about a appearing in a source. + /// + public struct DiagnosticResult + { + private const string DefaultPath = "Test0.cs"; + + private static readonly object[] EmptyArguments = new object[0]; + + private ImmutableArray _spans; + private string _message; + + public DiagnosticResult(string id, DiagnosticSeverity severity) + : this() + { + Id = id; + Severity = severity; + } + + public DiagnosticResult(DiagnosticDescriptor descriptor) + : this() + { + Id = descriptor.Id; + Severity = descriptor.DefaultSeverity; + MessageFormat = descriptor.MessageFormat; + } + + public ImmutableArray Spans + { + get + { + return _spans.IsDefault ? ImmutableArray.Empty : _spans; + } + } + + public DiagnosticSeverity Severity + { + get; + private set; + } + + public string Id + { + get; + private set; + } + + public string Message + { + get + { + if (_message != null) + { + return _message; + } + + if (MessageFormat != null) + { + return string.Format(MessageFormat.ToString(), MessageArguments ?? EmptyArguments); + } + + return null; + } + } + + public LocalizableString MessageFormat + { + get; + private set; + } + + public object[] MessageArguments + { + get; + private set; + } + + public bool HasLocation + { + get + { + return (_spans != null) && (_spans.Length > 0); + } + } + + public DiagnosticResult WithSeverity(DiagnosticSeverity severity) + { + DiagnosticResult result = this; + result.Severity = severity; + return result; + } + + public DiagnosticResult WithArguments(params object[] arguments) + { + DiagnosticResult result = this; + result.MessageArguments = arguments; + return result; + } + + public DiagnosticResult WithMessage(string message) + { + DiagnosticResult result = this; + result._message = message; + return result; + } + + public DiagnosticResult WithMessageFormat(LocalizableString messageFormat) + { + DiagnosticResult result = this; + result.MessageFormat = messageFormat; + return result; + } + + public DiagnosticResult WithLocation(int line, int column) + { + return WithLocation(DefaultPath, line, column); + } + + public DiagnosticResult WithLocation(string path, int line, int column) + { + LinePosition linePosition = new LinePosition(line, column); + + return AppendSpan(new FileLinePositionSpan(path, linePosition, linePosition)); + } + + public DiagnosticResult WithSpan(int startLine, int startColumn, int endLine, int endColumn) + { + return WithSpan(DefaultPath, startLine, startColumn, endLine, endColumn); + } + + public DiagnosticResult WithSpan(string path, int startLine, int startColumn, int endLine, int endColumn) + { + return AppendSpan(new FileLinePositionSpan(path, new LinePosition(startLine, startColumn), new LinePosition(endLine, endColumn))); + } + + public DiagnosticResult WithLineOffset(int offset) + { + DiagnosticResult result = this; + ImmutableArray.Builder spansBuilder = result._spans.ToBuilder(); + for (int i = 0; i < result._spans.Length; i++) + { + LinePosition newStartLinePosition = new LinePosition(result._spans[i].StartLinePosition.Line + offset, result._spans[i].StartLinePosition.Character); + LinePosition newEndLinePosition = new LinePosition(result._spans[i].EndLinePosition.Line + offset, result._spans[i].EndLinePosition.Character); + + spansBuilder[i] = new FileLinePositionSpan(result._spans[i].Path, newStartLinePosition, newEndLinePosition); + } + + result._spans = spansBuilder.MoveToImmutable(); + return result; + } + + private DiagnosticResult AppendSpan(FileLinePositionSpan span) + { + ImmutableArray newSpans = _spans.Add(span); + + // clone the object, so that the fluent syntax will work on immutable objects. + return new DiagnosticResult + { + Id = Id, + _message = _message, + MessageFormat = MessageFormat, + MessageArguments = MessageArguments, + Severity = Severity, + _spans = newSpans, + }; + } + } +} diff --git a/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs b/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs new file mode 100644 index 000000000..e4a37aa2e --- /dev/null +++ b/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Roslyn.UnitTestFramework +{ + public static class DiagnosticVerifier + where TAnalyzer : DiagnosticAnalyzer, new() + { + public static DiagnosticResult[] EmptyDiagnosticResults { get; } = { }; + + public static DiagnosticResult Diagnostic(string diagnosticId = null) + { + TAnalyzer analyzer = new TAnalyzer(); + ImmutableArray supportedDiagnostics = analyzer.SupportedDiagnostics; + if (diagnosticId is null) + { + return Diagnostic(supportedDiagnostics.Single()); + } + else + { + return Diagnostic(supportedDiagnostics.Single(i => i.Id == diagnosticId)); + } + } + + public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) + { + return new DiagnosticResult(descriptor); + } + + public static DiagnosticResult CompilerError(string errorIdentifier) + { + return new DiagnosticResult(errorIdentifier, DiagnosticSeverity.Error); + } + + public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult expected, CancellationToken cancellationToken) + => VerifyCSharpDiagnosticAsync(source, new[] { expected }, cancellationToken); + + public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult[] expected, CancellationToken cancellationToken) + { + CSharpTest test = new CSharpTest + { + TestCode = source, + }; + + test.ExpectedDiagnostics.AddRange(expected); + return test.RunAsync(cancellationToken); + } + + public class CSharpTest : GenericAnalyzerTest + { + public override string Language => LanguageNames.CSharp; + + protected override IEnumerable GetDiagnosticAnalyzers() + => new[] { new TAnalyzer() }; + + protected override IEnumerable GetCodeFixProviders() + => Enumerable.Empty(); + } + + public class VisualBasicTest : GenericAnalyzerTest + { + public override string Language => LanguageNames.VisualBasic; + + protected override IEnumerable GetDiagnosticAnalyzers() + => new[] { new TAnalyzer() }; + + protected override IEnumerable GetCodeFixProviders() + => Enumerable.Empty(); + } + } +} diff --git a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs new file mode 100644 index 000000000..4c8a406fd --- /dev/null +++ b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs @@ -0,0 +1,1254 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeActions; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Formatting; +using Microsoft.CodeAnalysis.Options; +using Microsoft.CodeAnalysis.Simplification; +using Microsoft.CodeAnalysis.Text; +using Microsoft.CodeAnalysis.VisualBasic; +using Xunit; + +#if !NETSTANDARD1_5 +using Microsoft.CodeAnalysis.Host.Mef; +using Microsoft.VisualStudio.Composition; +#endif + +namespace Roslyn.UnitTestFramework +{ + public abstract class GenericAnalyzerTest + { + private const int DefaultNumberOfIncrementalIterations = -1000; + + private static readonly string DefaultFilePathPrefix = "Test"; + private static readonly string CSharpDefaultFileExt = "cs"; + private static readonly string VisualBasicDefaultExt = "vb"; + private static readonly string CSharpDefaultFilePath = DefaultFilePathPrefix + 0 + "." + CSharpDefaultFileExt; + private static readonly string VisualBasicDefaultFilePath = DefaultFilePathPrefix + 0 + "." + VisualBasicDefaultExt; + private static readonly string TestProjectName = "TestProject"; + +#if !NETSTANDARD1_5 + private static readonly Lazy ExportProviderFactory; + + static GenericAnalyzerTest() + { + ExportProviderFactory = new Lazy( + () => + { + AttributedPartDiscovery discovery = new AttributedPartDiscovery(Resolver.DefaultInstance, isNonPublicSupported: true); + DiscoveredParts parts = Task.Run(() => discovery.CreatePartsAsync(MefHostServices.DefaultAssemblies)).GetAwaiter().GetResult(); + ComposableCatalog catalog = ComposableCatalog.Create(Resolver.DefaultInstance).AddParts(parts); + + CompositionConfiguration configuration = CompositionConfiguration.Create(catalog); + RuntimeComposition runtimeComposition = RuntimeComposition.CreateRuntimeComposition(configuration); + return runtimeComposition.CreateExportProviderFactory(); + }, + LazyThreadSafetyMode.ExecutionAndPublication); + } +#endif + + /// + /// Gets the language name used for the test. + /// + /// + /// The language name used for the test. + /// + public abstract string Language + { + get; + } + + public string TestCode + { + set + { + Assert.Empty(TestSources); + if (value != null) + { + TestSources.Add(value); + } + } + } + + public List TestSources { get; } = new List(); + + public Dictionary XmlReferences { get; } = new Dictionary(); + + public List ExpectedDiagnostics { get; } = new List(); + + public List RemainingDiagnostics { get; } = new List(); + + public List BatchRemainingDiagnostics { get; } = new List(); + + public List DisabledDiagnostics { get; } = new List(); + + public int? CodeFixIndex { get; set; } + + public string FixedCode + { + set + { + Assert.Empty(FixedSources); + if (value != null) + { + FixedSources.Add(value); + } + } + } + + public List FixedSources { get; } = new List(); + + public string BatchFixedCode + { + set + { + Assert.Empty(BatchFixedSources); + if (value != null) + { + BatchFixedSources.Add(value); + } + } + } + + public List BatchFixedSources { get; } = new List(); + + public int? NumberOfIncrementalIterations { get; set; } + + public int? NumberOfFixAllIterations { get; set; } + + public bool AllowNewCompilerDiagnostics { get; set; } = false; + + public List> OptionsTransforms { get; } = new List>(); + + public List> SolutionTransforms { get; } = new List>(); + + public async Task RunAsync(CancellationToken cancellationToken) + { + Assert.NotEmpty(TestSources); + + DiagnosticResult[] expected = ExpectedDiagnostics.ToArray(); + await VerifyDiagnosticsAsync(TestSources.ToArray(), expected, filenames: null, cancellationToken).ConfigureAwait(false); + if (HasFixableDiagnostics()) + { + DiagnosticResult[] remainingDiagnostics = FixedSources.SequenceEqual(TestSources) ? expected : RemainingDiagnostics.ToArray(); + await VerifyDiagnosticsAsync(FixedSources.ToArray(), remainingDiagnostics, filenames: null, cancellationToken).ConfigureAwait(false); + if (BatchFixedSources.Any()) + { + DiagnosticResult[] batchRemainingDiagnostics = BatchFixedSources.SequenceEqual(TestSources) ? expected : BatchRemainingDiagnostics.ToArray(); + await VerifyDiagnosticsAsync(BatchFixedSources.ToArray(), batchRemainingDiagnostics, filenames: null, cancellationToken).ConfigureAwait(false); + } + + await VerifyFixAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Gets the analyzers being tested. + /// + /// + /// New instances of all the analyzers being tested. + /// + protected abstract IEnumerable GetDiagnosticAnalyzers(); + + /// + /// Returns the code fixes being tested - to be implemented in non-abstract class. + /// + /// The to be used for C# code. + protected abstract IEnumerable GetCodeFixProviders(); + + private bool HasFixableDiagnostics() + { + CodeFixProvider[] fixers = GetCodeFixProviders().ToArray(); + if (HasFixableDiagnosticsCore()) + { + if (FixedSources.Count > 0) + { + return true; + } + + Assert.Empty(RemainingDiagnostics); + Assert.Empty(BatchRemainingDiagnostics); + return false; + } + + Assert.True(FixedSources.Count == 0 + || (FixedSources.Count == 1 && string.IsNullOrEmpty(FixedSources[0])) + || FixedSources.SequenceEqual(TestSources)); + Assert.True(BatchFixedSources.Count == 0 + || (BatchFixedSources.Count == 1 && string.IsNullOrEmpty(BatchFixedSources[0])) + || BatchFixedSources.SequenceEqual(TestSources)); + Assert.Empty(RemainingDiagnostics); + Assert.Empty(BatchRemainingDiagnostics); + return false; + + // Local function + bool HasFixableDiagnosticsCore() + { + return ExpectedDiagnostics.Any(diagnostic => fixers.Any(fixer => fixer.FixableDiagnosticIds.Contains(diagnostic.Id))); + } + } + + private static bool IsSubjectToExclusion(DiagnosticResult result) + { + if (result.Id.StartsWith("CS", StringComparison.Ordinal)) + { + return false; + } + + if (result.Id.StartsWith("AD", StringComparison.Ordinal)) + { + return false; + } + + if (result.Spans.Length == 0) + { + return false; + } + + return true; + } + + /// + /// General method that gets a collection of actual s found in the source after the + /// analyzer is run, then verifies each of them. + /// + /// An array of strings to create source documents from to run the analyzers on. + /// A collection of s that should appear after the analyzer + /// is run on the sources. + /// The filenames or null if the default filename should be used. + /// The that the task will observe. + /// A representing the asynchronous operation. + private async Task VerifyDiagnosticsAsync(string[] sources, DiagnosticResult[] expected, string[] filenames, CancellationToken cancellationToken) + { + ImmutableArray analyzers = GetDiagnosticAnalyzers().ToImmutableArray(); + VerifyDiagnosticResults(await GetSortedDiagnosticsAsync(sources, analyzers, filenames, cancellationToken).ConfigureAwait(false), analyzers, expected); + + // If filenames is null we want to test for exclusions too + if (filenames == null) + { + // Also check if the analyzer honors exclusions + if (expected.Any(IsSubjectToExclusion)) + { + // Diagnostics reported by the compiler and analyzer diagnostics which don't have a location will + // still be reported. We also insert a new line at the beginning so we have to move all diagnostic + // locations which have a specific position down by one line. + DiagnosticResult[] expectedResults = expected + .Where(x => !IsSubjectToExclusion(x)) + .Select(x => x.WithLineOffset(1)) + .ToArray(); + + VerifyDiagnosticResults(await GetSortedDiagnosticsAsync(sources.Select(x => " // \r\n" + x).ToArray(), analyzers, null, cancellationToken).ConfigureAwait(false), analyzers, expectedResults); + } + } + } + + /// + /// Given an analyzer and a collection of documents to apply it to, run the analyzer and gather an array of + /// diagnostics found. The returned diagnostics are then ordered by location in the source documents. + /// + /// The analyzer to run on the documents. + /// The s that the analyzer will be run on. + /// The that the task will observe. + /// A collection of s that surfaced in the source code, sorted by + /// . + protected static async Task> GetSortedDiagnosticsFromDocumentsAsync(ImmutableArray analyzers, Document[] documents, CancellationToken cancellationToken) + { + HashSet projects = new HashSet(); + foreach (Document document in documents) + { + projects.Add(document.Project); + } + + ImmutableArray.Builder diagnostics = ImmutableArray.CreateBuilder(); + foreach (Project project in projects) + { + Compilation compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false); + CompilationWithAnalyzers compilationWithAnalyzers = compilation.WithAnalyzers(analyzers, project.AnalyzerOptions, cancellationToken); + ImmutableArray compilerDiagnostics = compilation.GetDiagnostics(cancellationToken); + IEnumerable compilerErrors = compilerDiagnostics.Where(i => i.Severity == DiagnosticSeverity.Error); + ImmutableArray diags = await compilationWithAnalyzers.GetAnalyzerDiagnosticsAsync().ConfigureAwait(false); + ImmutableArray allDiagnostics = await compilationWithAnalyzers.GetAllDiagnosticsAsync().ConfigureAwait(false); + IEnumerable failureDiagnostics = allDiagnostics.Where(diagnostic => diagnostic.Id == "AD0001"); + foreach (Diagnostic diag in diags.Concat(compilerErrors).Concat(failureDiagnostics)) + { + if (diag.Location == Location.None || diag.Location.IsInMetadata) + { + diagnostics.Add(diag); + } + else + { + for (int i = 0; i < documents.Length; i++) + { + Document document = documents[i]; + SyntaxTree tree = await document.GetSyntaxTreeAsync(cancellationToken).ConfigureAwait(false); + if (tree == diag.Location.SourceTree) + { + diagnostics.Add(diag); + } + } + } + } + } + + Diagnostic[] results = SortDistinctDiagnostics(diagnostics); + return results.ToImmutableArray(); + } + + /// + /// Sort s by location in source document. + /// + /// A collection of s to be sorted. + /// A collection containing the input , sorted by + /// and . + private static Diagnostic[] SortDistinctDiagnostics(IEnumerable diagnostics) + { + return diagnostics.OrderBy(d => d.Location.SourceSpan.Start).ThenBy(d => d.Id).ToArray(); + } + + /// + /// Given classes in the form of strings, their language, and an to apply to + /// it, return the s found in the string after converting it to a + /// . + /// + /// Classes in the form of strings. + /// The analyzers to be run on the sources. + /// The filenames or if the default filename should be used. + /// The that the task will observe. + /// A collection of s that surfaced in the source code, sorted by + /// . + private Task> GetSortedDiagnosticsAsync(string[] sources, ImmutableArray analyzers, string[] filenames, CancellationToken cancellationToken) + { + return GetSortedDiagnosticsFromDocumentsAsync(analyzers, GetDocuments(sources, filenames), cancellationToken); + } + + /// + /// Given an array of strings as sources and a language, turn them into a and return the + /// documents and spans of it. + /// + /// Classes in the form of strings. + /// The filenames or if the default filename should be used. + /// A collection of s representing the sources. + private Document[] GetDocuments(string[] sources, string[] filenames) + { + if (Language != LanguageNames.CSharp && Language != LanguageNames.VisualBasic) + { + throw new ArgumentException("Unsupported Language"); + } + + Project project = CreateProject(sources, Language, filenames); + Document[] documents = project.Documents.ToArray(); + + if (sources.Length != documents.Length) + { + throw new InvalidOperationException("Amount of sources did not match amount of Documents created"); + } + + return documents; + } + + /// + /// Create a project using the input strings as sources. + /// + /// + /// This method first creates a by calling , and then + /// applies compilation options to the project by calling . + /// + /// Classes in the form of strings. + /// The language the source classes are in. Values may be taken from the + /// class. + /// The filenames or if the default filename should be used. + /// A created out of the s created from the source + /// strings. + protected Project CreateProject(string[] sources, string language = LanguageNames.CSharp, string[] filenames = null) + { + Project project = CreateProjectImpl(sources, language, filenames); + return ApplyCompilationOptions(project); + } + + /// + /// Create a project using the input strings as sources. + /// + /// Classes in the form of strings. + /// The language the source classes are in. Values may be taken from the + /// class. + /// The filenames or if the default filename should be used. + /// A created out of the s created from the source + /// strings. + protected virtual Project CreateProjectImpl(string[] sources, string language, string[] filenames) + { + string fileNamePrefix = DefaultFilePathPrefix; + string fileExt = language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt; + + ProjectId projectId = ProjectId.CreateNewId(debugName: TestProjectName); + Solution solution = CreateSolution(projectId, language); + + int count = 0; + for (int i = 0; i < sources.Length; i++) + { + string source = sources[i]; + string newFileName = filenames?[i] ?? fileNamePrefix + count + "." + fileExt; + DocumentId documentId = DocumentId.CreateNewId(projectId, debugName: newFileName); + solution = solution.AddDocument(documentId, newFileName, SourceText.From(source)); + count++; + } + + return solution.GetProject(projectId); + } + + /// + /// Applies compilation options to a project. + /// + /// + /// The default implementation configures the project by enabling all supported diagnostics of analyzers + /// included in as well as AD0001. After configuring these + /// diagnostics, any diagnostic IDs indicated in are explicitly suppressed + /// using . + /// + /// The project. + /// The modified project. + protected virtual Project ApplyCompilationOptions(Project project) + { + IEnumerable analyzers = GetDiagnosticAnalyzers(); + + Dictionary supportedDiagnosticsSpecificOptions = new Dictionary(); + foreach (DiagnosticAnalyzer analyzer in analyzers) + { + foreach (DiagnosticDescriptor diagnostic in analyzer.SupportedDiagnostics) + { + // make sure the analyzers we are testing are enabled + supportedDiagnosticsSpecificOptions[diagnostic.Id] = ReportDiagnostic.Default; + } + } + + // Report exceptions during the analysis process as errors + supportedDiagnosticsSpecificOptions.Add("AD0001", ReportDiagnostic.Error); + + foreach (string id in DisabledDiagnostics) + { + supportedDiagnosticsSpecificOptions[id] = ReportDiagnostic.Suppress; + } + + // update the project compilation options + ImmutableDictionary modifiedSpecificDiagnosticOptions = supportedDiagnosticsSpecificOptions.ToImmutableDictionary().SetItems(project.CompilationOptions.SpecificDiagnosticOptions); + CompilationOptions modifiedCompilationOptions = project.CompilationOptions.WithSpecificDiagnosticOptions(modifiedSpecificDiagnosticOptions); + + Solution solution = project.Solution.WithProjectCompilationOptions(project.Id, modifiedCompilationOptions); + return solution.GetProject(project.Id); + } + + public static AdhocWorkspace CreateWorkspace() + { +#if NETSTANDARD1_5 + return new AdhocWorkspace(); +#else + ExportProvider exportProvider = ExportProviderFactory.Value.CreateExportProvider(); + MefV1HostServices host = MefV1HostServices.Create(exportProvider.AsExportProvider()); + return new AdhocWorkspace(host); +#endif + } + + /// + /// Creates a solution that will be used as parent for the sources that need to be checked. + /// + /// The project identifier to use. + /// The language for which the solution is being created. + /// The created solution. + protected virtual Solution CreateSolution(ProjectId projectId, string language) + { + CompilationOptions compilationOptions = language == LanguageNames.CSharp + ? (CompilationOptions)new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary, allowUnsafe: true) + : new VisualBasicCompilationOptions(OutputKind.DynamicallyLinkedLibrary); + + TestXmlReferenceResolver xmlReferenceResolver = new TestXmlReferenceResolver(); + foreach (KeyValuePair xmlReference in XmlReferences) + { + xmlReferenceResolver.XmlReferences.Add(xmlReference.Key, xmlReference.Value); + } + + compilationOptions = compilationOptions.WithXmlReferenceResolver(xmlReferenceResolver); + + Solution solution = CreateWorkspace() + .CurrentSolution + .AddProject(projectId, TestProjectName, TestProjectName, language) + .WithProjectCompilationOptions(projectId, compilationOptions) + .AddMetadataReference(projectId, MetadataReferences.CorlibReference) + .AddMetadataReference(projectId, MetadataReferences.SystemReference) + .AddMetadataReference(projectId, MetadataReferences.SystemCoreReference) + .AddMetadataReference(projectId, MetadataReferences.CSharpSymbolsReference) + .AddMetadataReference(projectId, MetadataReferences.CodeAnalysisReference); + + if (MetadataReferences.SystemRuntimeReference != null) + { + solution = solution.AddMetadataReference(projectId, MetadataReferences.SystemRuntimeReference); + } + + if (MetadataReferences.SystemValueTupleReference != null) + { + solution = solution.AddMetadataReference(projectId, MetadataReferences.SystemValueTupleReference); + } + + foreach (Func transform in OptionsTransforms) + { + solution.Workspace.Options = transform(solution.Workspace.Options); + } + + ParseOptions parseOptions = solution.GetProject(projectId).ParseOptions; + solution = solution.WithProjectParseOptions(projectId, parseOptions.WithDocumentationMode(DocumentationMode.Diagnose)); + + foreach (Func transform in SolutionTransforms) + { + solution = transform(solution, projectId); + } + + return solution; + } + + /// + /// Checks each of the actual s found and compares them with the corresponding + /// in the array of expected results. s are considered + /// equal only if the , , + /// , and of the + /// match the actual . + /// + /// The s found by the compiler after running the analyzer + /// on the source code. + /// The analyzers that have been run on the sources. + /// A collection of s describing the expected + /// diagnostics for the sources. + private static void VerifyDiagnosticResults(IEnumerable actualResults, ImmutableArray analyzers, DiagnosticResult[] expectedResults) + { + int expectedCount = expectedResults.Length; + int actualCount = actualResults.Count(); + + if (expectedCount != actualCount) + { + string diagnosticsOutput = actualResults.Any() ? FormatDiagnostics(analyzers, actualResults.ToArray()) : " NONE."; + + Assert.True( + false, + string.Format("Mismatch between number of diagnostics returned, expected \"{0}\" actual \"{1}\"\r\n\r\nDiagnostics:\r\n{2}\r\n", expectedCount, actualCount, diagnosticsOutput)); + } + + for (int i = 0; i < expectedResults.Length; i++) + { + Diagnostic actual = actualResults.ElementAt(i); + DiagnosticResult expected = expectedResults[i]; + + if (!expected.HasLocation) + { + if (actual.Location != Location.None) + { + string message = + string.Format( + "Expected:\nA project diagnostic with No location\nActual:\n{0}", + FormatDiagnostics(analyzers, actual)); + Assert.True(false, message); + } + } + else + { + VerifyDiagnosticLocation(analyzers, actual, actual.Location, expected.Spans.First()); + Location[] additionalLocations = actual.AdditionalLocations.ToArray(); + + if (additionalLocations.Length != expected.Spans.Length - 1) + { + Assert.True( + false, + string.Format( + "Expected {0} additional locations but got {1} for Diagnostic:\r\n {2}\r\n", + expected.Spans.Length - 1, + additionalLocations.Length, + FormatDiagnostics(analyzers, actual))); + } + + for (int j = 0; j < additionalLocations.Length; ++j) + { + VerifyDiagnosticLocation(analyzers, actual, additionalLocations[j], expected.Spans[j + 1]); + } + } + + if (actual.Id != expected.Id) + { + string message = + string.Format( + "Expected diagnostic id to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n {2}\r\n", + expected.Id, + actual.Id, + FormatDiagnostics(analyzers, actual)); + Assert.True(false, message); + } + + if (actual.Severity != expected.Severity) + { + string message = + string.Format( + "Expected diagnostic severity to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n {2}\r\n", + expected.Severity, + actual.Severity, + FormatDiagnostics(analyzers, actual)); + Assert.True(false, message); + } + + if (actual.GetMessage() != expected.Message) + { + string message = + string.Format( + "Expected diagnostic message to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n {2}\r\n", + expected.Message, + actual.GetMessage(), + FormatDiagnostics(analyzers, actual)); + Assert.True(false, message); + } + } + } + + /// + /// Helper method to that checks the location of a + /// and compares it with the location described by a + /// . + /// + /// The analyzer that have been run on the sources. + /// The diagnostic that was found in the code. + /// The location of the diagnostic found in the code. + /// The describing the expected location of the + /// diagnostic. + private static void VerifyDiagnosticLocation(ImmutableArray analyzers, Diagnostic diagnostic, Location actual, FileLinePositionSpan expected) + { + FileLinePositionSpan actualSpan = actual.GetLineSpan(); + + string message = + string.Format( + "Expected diagnostic to be in file \"{0}\" was actually in file \"{1}\"\r\n\r\nDiagnostic:\r\n {2}\r\n", + expected.Path, + actualSpan.Path, + FormatDiagnostics(analyzers, diagnostic)); + Assert.True( + actualSpan.Path == expected.Path || (actualSpan.Path != null && actualSpan.Path.Contains("Test0.") && expected.Path.Contains("Test.")), + message); + + LinePosition actualStartLinePosition = actualSpan.StartLinePosition; + LinePosition actualEndLinePosition = actualSpan.EndLinePosition; + + VerifyLinePosition(analyzers, diagnostic, actualSpan.StartLinePosition, expected.StartLinePosition, "start"); + if (expected.StartLinePosition < expected.EndLinePosition) + { + VerifyLinePosition(analyzers, diagnostic, actualSpan.EndLinePosition, expected.EndLinePosition, "end"); + } + } + + private static void VerifyLinePosition(ImmutableArray analyzers, Diagnostic diagnostic, LinePosition actualLinePosition, LinePosition expectedLinePosition, string positionText) + { + // Only check the line position if it matters + if (expectedLinePosition.Line > 0) + { + Assert.True( + (actualLinePosition.Line + 1) == expectedLinePosition.Line, + string.Format( + "Expected diagnostic to {0} on line \"{1}\" was actually on line \"{2}\"\r\n\r\nDiagnostic:\r\n {3}\r\n", + positionText, + expectedLinePosition.Line, + actualLinePosition.Line + 1, + FormatDiagnostics(analyzers, diagnostic))); + } + + // Only check the column position if it matters + if (expectedLinePosition.Character > 0) + { + Assert.True( + (actualLinePosition.Character + 1) == expectedLinePosition.Character, + string.Format( + "Expected diagnostic to {0} at column \"{1}\" was actually at column \"{2}\"\r\n\r\nDiagnostic:\r\n {3}\r\n", + positionText, + expectedLinePosition.Character, + actualLinePosition.Character + 1, + FormatDiagnostics(analyzers, diagnostic))); + } + } + + /// + /// Helper method to format a into an easily readable string. + /// + /// The analyzers that this verifier tests. + /// A collection of s to be formatted. + /// The formatted as a string. + private static string FormatDiagnostics(ImmutableArray analyzers, params Diagnostic[] diagnostics) + { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < diagnostics.Length; ++i) + { + string diagnosticsId = diagnostics[i].Id; + + builder.Append("// ").AppendLine(diagnostics[i].ToString()); + + DiagnosticAnalyzer applicableAnalyzer = analyzers.FirstOrDefault(a => a.SupportedDiagnostics.Any(dd => dd.Id == diagnosticsId)); + if (applicableAnalyzer != null) + { + Type analyzerType = applicableAnalyzer.GetType(); + + Location location = diagnostics[i].Location; + if (location == Location.None) + { + builder.AppendFormat("GetGlobalResult({0}.{1})", analyzerType.Name, diagnosticsId); + } + else + { + Assert.True( + location.IsInSource, + string.Format("Test base does not currently handle diagnostics in metadata locations. Diagnostic in metadata:\r\n{0}", diagnostics[i])); + + string resultMethodName = diagnostics[i].Location.SourceTree.FilePath.EndsWith(".cs") ? "GetCSharpResultAt" : "GetBasicResultAt"; + LinePosition linePosition = diagnostics[i].Location.GetLineSpan().StartLinePosition; + + builder.AppendFormat( + "{0}({1}, {2}, {3}.{4})", + resultMethodName, + linePosition.Line + 1, + linePosition.Character + 1, + analyzerType.Name, + diagnosticsId); + } + + if (i != diagnostics.Length - 1) + { + builder.Append(','); + } + + builder.AppendLine(); + } + } + + return builder.ToString(); + } + + /// + /// Called to test a C# code fix when applied on the input source as a string. + /// + /// An array of file names in the project before the code fix was applied. + /// An array of file names in the project after the code fix was applied. + /// The that the task will observe. + /// A representing the asynchronous operation. + protected async Task VerifyFixAsync(string[] oldFileNames = null, string[] newFileNames = null, CancellationToken cancellationToken = default) + { + string[] oldSources = TestSources.ToArray(); + string[] newSources = FixedSources.ToArray(); + string[] batchNewSources = BatchFixedSources.Any() ? BatchFixedSources.ToArray() : newSources; + + int numberOfIncrementalIterations; + int numberOfFixAllIterations; + if (NumberOfIncrementalIterations != null) + { + numberOfIncrementalIterations = NumberOfIncrementalIterations.Value; + } + else + { + if (!HasAnyChange(oldSources, newSources, oldFileNames, newFileNames)) + { + numberOfIncrementalIterations = 0; + } + else + { + numberOfIncrementalIterations = DefaultNumberOfIncrementalIterations; + } + } + + if (NumberOfFixAllIterations != null) + { + numberOfFixAllIterations = NumberOfFixAllIterations.Value; + } + else + { + if (!HasAnyChange(oldSources, batchNewSources, oldFileNames, newFileNames)) + { + numberOfFixAllIterations = 0; + } + else + { + numberOfFixAllIterations = 1; + } + } + + ConfiguredTaskAwaitable t1 = VerifyFixpublicAsync(Language, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, newSources, oldFileNames, newFileNames, numberOfIncrementalIterations, FixEachAnalyzerDiagnosticAsync, cancellationToken).ConfigureAwait(false); + + ImmutableArray fixAllProvider = GetCodeFixProviders().Select(codeFixProvider => codeFixProvider.GetFixAllProvider()).Where(codeFixProvider => codeFixProvider != null).ToImmutableArray(); + + if (fixAllProvider.IsEmpty) + { + await t1; + } + else + { + if (Debugger.IsAttached) + { + await t1; + } + + ConfiguredTaskAwaitable t2 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, oldFileNames, newFileNames, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInDocumentAsync, cancellationToken).ConfigureAwait(false); + if (Debugger.IsAttached) + { + await t2; + } + + ConfiguredTaskAwaitable t3 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, oldFileNames, newFileNames, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInProjectAsync, cancellationToken).ConfigureAwait(false); + if (Debugger.IsAttached) + { + await t3; + } + + ConfiguredTaskAwaitable t4 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, oldFileNames, newFileNames, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInSolutionAsync, cancellationToken).ConfigureAwait(false); + if (Debugger.IsAttached) + { + await t4; + } + + if (!Debugger.IsAttached) + { + // Allow the operations to run in parallel + await t1; + await t2; + await t3; + await t4; + } + } + } + + private async Task VerifyFixpublicAsync( + string language, + ImmutableArray analyzers, + ImmutableArray codeFixProviders, + string[] oldSources, + string[] newSources, + string[] oldFileNames, + string[] newFileNames, + int numberOfIterations, + Func, ImmutableArray, int?, Project, int, CancellationToken, Task> getFixedProject, + CancellationToken cancellationToken) + { + if (oldFileNames != null) + { + // Make sure the test case is consistent regarding the number of sources and file names before the code fix + Assert.Equal($"{oldSources.Length} old file names", $"{oldFileNames.Length} old file names"); + } + + if (newFileNames != null) + { + // Make sure the test case is consistent regarding the number of sources and file names after the code fix + Assert.Equal($"{newSources.Length} new file names", $"{newFileNames.Length} new file names"); + } + + Project project = CreateProject(oldSources, language, oldFileNames); + ImmutableArray compilerDiagnostics = await GetCompilerDiagnosticsAsync(project, cancellationToken).ConfigureAwait(false); + + project = await getFixedProject(analyzers, codeFixProviders, CodeFixIndex, project, numberOfIterations, cancellationToken).ConfigureAwait(false); + + IEnumerable newCompilerDiagnostics = GetNewDiagnostics(compilerDiagnostics, await GetCompilerDiagnosticsAsync(project, cancellationToken).ConfigureAwait(false)); + + // Check if applying the code fix introduced any new compiler diagnostics + if (!AllowNewCompilerDiagnostics && newCompilerDiagnostics.Any()) + { + // Format and get the compiler diagnostics again so that the locations make sense in the output + project = await ReformatProjectDocumentsAsync(project, cancellationToken).ConfigureAwait(false); + newCompilerDiagnostics = GetNewDiagnostics(compilerDiagnostics, await GetCompilerDiagnosticsAsync(project, cancellationToken).ConfigureAwait(false)); + + StringBuilder message = new StringBuilder(); + message.Append("Fix introduced new compiler diagnostics:\r\n"); + newCompilerDiagnostics.Aggregate(message, (sb, d) => sb.Append(d.ToString()).Append("\r\n")); + foreach (Document document in project.Documents) + { + message.Append("\r\n").Append(document.Name).Append(":\r\n"); + message.Append((await document.GetSyntaxRootAsync(cancellationToken).ConfigureAwait(false)).ToFullString()); + message.Append("\r\n"); + } + + Assert.True(false, message.ToString()); + } + + // After applying all of the code fixes, compare the resulting string to the inputted one + Document[] updatedDocuments = project.Documents.ToArray(); + + Assert.Equal($"{newSources.Length} documents", $"{updatedDocuments.Length} documents"); + + for (int i = 0; i < updatedDocuments.Length; i++) + { + string actual = await GetStringFromDocumentAsync(updatedDocuments[i], cancellationToken).ConfigureAwait(false); + Assert.Equal(newSources[i], actual); + + if (newFileNames != null) + { + Assert.Equal(newFileNames[i], updatedDocuments[i].Name); + } + } + } + + private static bool HasAnyChange(string[] oldSources, string[] newSources, string[] oldFileNames, string[] newFileNames) + { + if (!oldSources.SequenceEqual(newSources)) + { + return true; + } + + if (oldFileNames != null && newFileNames != null && !oldFileNames.SequenceEqual(newFileNames)) + { + return true; + } + + return false; + } + + private static async Task FixEachAnalyzerDiagnosticAsync(ImmutableArray analyzers, ImmutableArray codeFixProviders, int? codeFixIndex, Project project, int numberOfIterations, CancellationToken cancellationToken) + { + CodeFixProvider codeFixProvider = codeFixProviders.Single(); + + int expectedNumberOfIterations = numberOfIterations; + if (numberOfIterations < 0) + { + numberOfIterations = -numberOfIterations; + } + + ImmutableArray previousDiagnostics = ImmutableArray.Create(); + + bool done; + do + { + ImmutableArray analyzerDiagnostics = await GetSortedDiagnosticsFromDocumentsAsync(analyzers, project.Documents.ToArray(), cancellationToken).ConfigureAwait(false); + if (analyzerDiagnostics.Length == 0) + { + break; + } + + if (!AreDiagnosticsDifferent(analyzerDiagnostics, previousDiagnostics)) + { + break; + } + + if (--numberOfIterations < -1) + { + Assert.True(false, "The upper limit for the number of code fix iterations was exceeded"); + } + + previousDiagnostics = analyzerDiagnostics; + + done = true; + bool anyActions = false; + foreach (Diagnostic diagnostic in analyzerDiagnostics) + { + if (!codeFixProvider.FixableDiagnosticIds.Contains(diagnostic.Id)) + { + // do not pass unsupported diagnostics to a code fix provider + continue; + } + + List actions = new List(); + CodeFixContext context = new CodeFixContext(project.GetDocument(diagnostic.Location.SourceTree), diagnostic, (a, d) => actions.Add(a), cancellationToken); + await codeFixProvider.RegisterCodeFixesAsync(context).ConfigureAwait(false); + + if (actions.Count > 0) + { + anyActions = true; + + Project fixedProject = await ApplyFixAsync(project, actions.ElementAt(codeFixIndex.GetValueOrDefault(0)), cancellationToken).ConfigureAwait(false); + if (fixedProject != project) + { + done = false; + + project = await RecreateProjectDocumentsAsync(fixedProject, cancellationToken).ConfigureAwait(false); + break; + } + } + } + + if (!anyActions) + { + Assert.True(done); + + // Avoid counting iterations that do not provide any code actions + numberOfIterations++; + } + } + while (!done); + + if (expectedNumberOfIterations >= 0) + { + Assert.Equal($"{expectedNumberOfIterations} iterations", $"{expectedNumberOfIterations - numberOfIterations} iterations"); + } + + return project; + } + + private static Task FixAllAnalyzerDiagnosticsInDocumentAsync(ImmutableArray analyzers, ImmutableArray codeFixProviders, int? codeFixIndex, Project project, int numberOfIterations, CancellationToken cancellationToken) + { + return FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope.Document, analyzers, codeFixProviders, codeFixIndex, project, numberOfIterations, cancellationToken); + } + + private static Task FixAllAnalyzerDiagnosticsInProjectAsync(ImmutableArray analyzers, ImmutableArray codeFixProviders, int? codeFixIndex, Project project, int numberOfIterations, CancellationToken cancellationToken) + { + return FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope.Project, analyzers, codeFixProviders, codeFixIndex, project, numberOfIterations, cancellationToken); + } + + private static Task FixAllAnalyzerDiagnosticsInSolutionAsync(ImmutableArray analyzers, ImmutableArray codeFixProviders, int? codeFixIndex, Project project, int numberOfIterations, CancellationToken cancellationToken) + { + return FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope.Solution, analyzers, codeFixProviders, codeFixIndex, project, numberOfIterations, cancellationToken); + } + + private static async Task FixAllAnalyerDiagnosticsInScopeAsync(FixAllScope scope, ImmutableArray analyzers, ImmutableArray codeFixProviders, int? codeFixIndex, Project project, int numberOfIterations, CancellationToken cancellationToken) + { + CodeFixProvider codeFixProvider = codeFixProviders.Single(); + + int expectedNumberOfIterations = numberOfIterations; + if (numberOfIterations < 0) + { + numberOfIterations = -numberOfIterations; + } + + ImmutableArray previousDiagnostics = ImmutableArray.Create(); + + FixAllProvider fixAllProvider = codeFixProvider.GetFixAllProvider(); + + if (fixAllProvider == null) + { + return null; + } + + bool done; + do + { + ImmutableArray analyzerDiagnostics = await GetSortedDiagnosticsFromDocumentsAsync(analyzers, project.Documents.ToArray(), cancellationToken).ConfigureAwait(false); + if (analyzerDiagnostics.Length == 0) + { + break; + } + + if (!AreDiagnosticsDifferent(analyzerDiagnostics, previousDiagnostics)) + { + break; + } + + if (--numberOfIterations < -1) + { + Assert.True(false, "The upper limit for the number of fix all iterations was exceeded"); + } + + Diagnostic firstDiagnostic = null; + string equivalenceKey = null; + foreach (Diagnostic diagnostic in analyzerDiagnostics) + { + if (!codeFixProvider.FixableDiagnosticIds.Contains(diagnostic.Id)) + { + // do not pass unsupported diagnostics to a code fix provider + continue; + } + + List actions = new List(); + CodeFixContext context = new CodeFixContext(project.GetDocument(diagnostic.Location.SourceTree), diagnostic, (a, d) => actions.Add(a), cancellationToken); + await codeFixProvider.RegisterCodeFixesAsync(context).ConfigureAwait(false); + if (actions.Count > (codeFixIndex ?? 0)) + { + firstDiagnostic = diagnostic; + equivalenceKey = actions[codeFixIndex ?? 0].EquivalenceKey; + break; + } + } + + if (firstDiagnostic == null) + { + numberOfIterations++; + break; + } + + previousDiagnostics = analyzerDiagnostics; + + done = true; + + FixAllContext.DiagnosticProvider fixAllDiagnosticProvider = TestDiagnosticProvider.Create(analyzerDiagnostics); + + IEnumerable analyzerDiagnosticIds = analyzers.SelectMany(x => x.SupportedDiagnostics).Select(x => x.Id); + IEnumerable compilerDiagnosticIds = codeFixProvider.FixableDiagnosticIds.Where(x => x.StartsWith("CS", StringComparison.Ordinal)); + IEnumerable disabledDiagnosticIds = project.CompilationOptions.SpecificDiagnosticOptions.Where(x => x.Value == ReportDiagnostic.Suppress).Select(x => x.Key); + IEnumerable relevantIds = analyzerDiagnosticIds.Concat(compilerDiagnosticIds).Except(disabledDiagnosticIds).Distinct(); + FixAllContext fixAllContext = new FixAllContext(project.GetDocument(firstDiagnostic.Location.SourceTree), codeFixProvider, scope, equivalenceKey, relevantIds, fixAllDiagnosticProvider, cancellationToken); + + CodeAction action = await fixAllProvider.GetFixAsync(fixAllContext).ConfigureAwait(false); + if (action == null) + { + return project; + } + + Project fixedProject = await ApplyFixAsync(project, action, cancellationToken).ConfigureAwait(false); + if (fixedProject != project) + { + done = false; + + project = await RecreateProjectDocumentsAsync(fixedProject, cancellationToken).ConfigureAwait(false); + } + } + while (!done); + + if (expectedNumberOfIterations >= 0) + { + Assert.Equal($"{expectedNumberOfIterations} iterations", $"{expectedNumberOfIterations - numberOfIterations} iterations"); + } + + return project; + } + + /// + /// Get the existing compiler diagnostics on the input document. + /// + /// The to run the compiler diagnostic analyzers on. + /// The that the task will observe. + /// The compiler diagnostics that were found in the code. + private static async Task> GetCompilerDiagnosticsAsync(Project project, CancellationToken cancellationToken) + { + ImmutableArray allDiagnostics = ImmutableArray.Create(); + + foreach (Document document in project.Documents) + { + SemanticModel semanticModel = await document.GetSemanticModelAsync(cancellationToken).ConfigureAwait(false); + allDiagnostics = allDiagnostics.AddRange(semanticModel.GetDiagnostics(cancellationToken: cancellationToken)); + } + + return allDiagnostics; + } + + /// + /// Given a document, turn it into a string based on the syntax root. + /// + /// The to be converted to a string. + /// The that the task will observe. + /// A string containing the syntax of the after formatting. + private static async Task GetStringFromDocumentAsync(Document document, CancellationToken cancellationToken) + { + Document simplifiedDoc = await Simplifier.ReduceAsync(document, Simplifier.Annotation, cancellationToken: cancellationToken).ConfigureAwait(false); + Document formatted = await Formatter.FormatAsync(simplifiedDoc, Formatter.Annotation, cancellationToken: cancellationToken).ConfigureAwait(false); + SourceText sourceText = await formatted.GetTextAsync(cancellationToken).ConfigureAwait(false); + return sourceText.ToString(); + } + + /// + /// Implements a workaround for issue #936, force re-parsing to get the same sort of syntax tree as the original document. + /// + /// The project to update. + /// The . + /// The updated . + private static async Task RecreateProjectDocumentsAsync(Project project, CancellationToken cancellationToken) + { + foreach (DocumentId documentId in project.DocumentIds) + { + Document document = project.GetDocument(documentId); + document = await RecreateDocumentAsync(document, cancellationToken).ConfigureAwait(false); + project = document.Project; + } + + return project; + } + + private static async Task RecreateDocumentAsync(Document document, CancellationToken cancellationToken) + { + SourceText newText = await document.GetTextAsync(cancellationToken).ConfigureAwait(false); + newText = newText.WithChanges(new TextChange(new TextSpan(0, 0), " ")); + newText = newText.WithChanges(new TextChange(new TextSpan(0, 1), string.Empty)); + return document.WithText(newText); + } + + /// + /// Formats the whitespace in all documents of the specified . + /// + /// The project to update. + /// The . + /// The updated . + private static async Task ReformatProjectDocumentsAsync(Project project, CancellationToken cancellationToken) + { + foreach (DocumentId documentId in project.DocumentIds) + { + Document document = project.GetDocument(documentId); + document = await Formatter.FormatAsync(document, Formatter.Annotation, cancellationToken: cancellationToken).ConfigureAwait(false); + project = document.Project; + } + + return project; + } + + /// + /// Compare two collections of s, and return a list of any new diagnostics that appear + /// only in the second collection. + /// + /// Considers to be the same if they have the same s. + /// In the case of multiple diagnostics with the same in a row, this method may not + /// necessarily return the new one. + /// + /// + /// The s that existed in the code before the code fix was + /// applied. + /// The s that exist in the code after the code fix was + /// applied. + /// A list of s that only surfaced in the code after the code fix was + /// applied. + private static IEnumerable GetNewDiagnostics(IEnumerable diagnostics, IEnumerable newDiagnostics) + { + Diagnostic[] oldArray = diagnostics.OrderBy(d => d.Location.SourceSpan.Start).ToArray(); + Diagnostic[] newArray = newDiagnostics.OrderBy(d => d.Location.SourceSpan.Start).ToArray(); + + int oldIndex = 0; + int newIndex = 0; + + while (newIndex < newArray.Length) + { + if (oldIndex < oldArray.Length && oldArray[oldIndex].Id == newArray[newIndex].Id) + { + ++oldIndex; + ++newIndex; + } + else + { + yield return newArray[newIndex++]; + } + } + } + + /// + /// Apply the inputted to the inputted document. + /// Meant to be used to apply code fixes. + /// + /// The to apply the fix on. + /// A that will be applied to the + /// . + /// The that the task will observe. + /// A with the changes from the . + private static async Task ApplyFixAsync(Project project, CodeAction codeAction, CancellationToken cancellationToken) + { + ImmutableArray operations = await codeAction.GetOperationsAsync(cancellationToken).ConfigureAwait(false); + Solution solution = operations.OfType().Single().ChangedSolution; + return solution.GetProject(project.Id); + } + + private static bool AreDiagnosticsDifferent(ImmutableArray analyzerDiagnostics, ImmutableArray previousDiagnostics) + { + if (analyzerDiagnostics.Length != previousDiagnostics.Length) + { + return true; + } + + for (int i = 0; i < analyzerDiagnostics.Length; i++) + { + if ((analyzerDiagnostics[i].Id != previousDiagnostics[i].Id) + || (analyzerDiagnostics[i].Location.SourceSpan != previousDiagnostics[i].Location.SourceSpan)) + { + return true; + } + } + + return false; + } + } +} diff --git a/samples/Shared/UnitTestFramework/MetadataReferences.cs b/samples/Shared/UnitTestFramework/MetadataReferences.cs new file mode 100644 index 000000000..a4a6e77da --- /dev/null +++ b/samples/Shared/UnitTestFramework/MetadataReferences.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Immutable; +using System.Linq; +using System.Reflection; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; + +#if !NETSTANDARD1_5 +using System; +#endif + +namespace Roslyn.UnitTestFramework +{ + /// + /// Metadata references used to create test projects. + /// + public static class MetadataReferences + { + public static readonly MetadataReference CorlibReference = MetadataReference.CreateFromFile(typeof(object).GetTypeInfo().Assembly.Location).WithAliases(ImmutableArray.Create("global", "corlib")); + public static readonly MetadataReference SystemReference = MetadataReference.CreateFromFile(typeof(System.Diagnostics.Debug).GetTypeInfo().Assembly.Location).WithAliases(ImmutableArray.Create("global", "system")); + public static readonly MetadataReference SystemCoreReference = MetadataReference.CreateFromFile(typeof(Enumerable).GetTypeInfo().Assembly.Location); + public static readonly MetadataReference CSharpSymbolsReference = MetadataReference.CreateFromFile(typeof(CSharpCompilation).GetTypeInfo().Assembly.Location); + public static readonly MetadataReference CodeAnalysisReference = MetadataReference.CreateFromFile(typeof(Compilation).GetTypeInfo().Assembly.Location); + + public static readonly MetadataReference SystemRuntimeReference; + public static readonly MetadataReference SystemValueTupleReference; + + static MetadataReferences() + { + if (typeof(string).GetTypeInfo().Assembly.GetType("System.ValueTuple", false) != null) + { + // mscorlib contains ValueTuple, so no need to add a separate reference + SystemRuntimeReference = null; + SystemValueTupleReference = null; + } + else + { +#if !NETSTANDARD1_5 + Assembly systemRuntime = AppDomain.CurrentDomain.GetAssemblies().SingleOrDefault(x => x.GetName().Name == "System.Runtime"); + if (systemRuntime != null) + { + SystemRuntimeReference = MetadataReference.CreateFromFile(systemRuntime.Location); + } + + Assembly systemValueTuple = AppDomain.CurrentDomain.GetAssemblies().SingleOrDefault(x => x.GetName().Name == "System.ValueTuple"); + if (systemValueTuple != null) + { + SystemValueTupleReference = MetadataReference.CreateFromFile(systemValueTuple.Location); + } +#endif + } + } + } +} diff --git a/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj b/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj index 820c8806b..563b00066 100644 --- a/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj +++ b/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj @@ -1,14 +1,27 @@ - + - netstandard2.0 - $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)\..\..\bin\CSharp\Roslyn.UnitTestFramework')) + netstandard1.5;net452 + + + + 7.2 - - + + + + + + + + portable-net45+win8 + + + + diff --git a/samples/Shared/UnitTestFramework/TestDiagnosticProvider.cs b/samples/Shared/UnitTestFramework/TestDiagnosticProvider.cs new file mode 100644 index 000000000..bb258ebeb --- /dev/null +++ b/samples/Shared/UnitTestFramework/TestDiagnosticProvider.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeFixes; + +namespace Roslyn.UnitTestFramework +{ + internal sealed class TestDiagnosticProvider : FixAllContext.DiagnosticProvider + { + private readonly ImmutableArray _diagnostics; + + private TestDiagnosticProvider(ImmutableArray diagnostics) + { + _diagnostics = diagnostics; + } + + public override Task> GetAllDiagnosticsAsync(Project project, CancellationToken cancellationToken) + { + return Task.FromResult>(_diagnostics); + } + + public override Task> GetDocumentDiagnosticsAsync(Document document, CancellationToken cancellationToken) + { + return Task.FromResult(_diagnostics.Where(i => i.Location.GetLineSpan().Path == document.Name)); + } + + public override Task> GetProjectDiagnosticsAsync(Project project, CancellationToken cancellationToken) + { + return Task.FromResult(_diagnostics.Where(i => !i.Location.IsInSource)); + } + + internal static TestDiagnosticProvider Create(ImmutableArray diagnostics) + { + return new TestDiagnosticProvider(diagnostics); + } + } +} diff --git a/samples/Shared/UnitTestFramework/TestXmlReferenceResolver.cs b/samples/Shared/UnitTestFramework/TestXmlReferenceResolver.cs new file mode 100644 index 000000000..b15913929 --- /dev/null +++ b/samples/Shared/UnitTestFramework/TestXmlReferenceResolver.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.IO; +using System.Runtime.CompilerServices; +using System.Text; +using Microsoft.CodeAnalysis; + +namespace Roslyn.UnitTestFramework +{ + internal class TestXmlReferenceResolver : XmlReferenceResolver + { + public Dictionary XmlReferences { get; } = + new Dictionary(); + + public override bool Equals(object other) + { + return ReferenceEquals(this, other); + } + + public override int GetHashCode() + { + return RuntimeHelpers.GetHashCode(this); + } + + public override Stream OpenRead(string resolvedPath) + { + if (!XmlReferences.TryGetValue(resolvedPath, out string content)) + { + return null; + } + + return new MemoryStream(Encoding.UTF8.GetBytes(content)); + } + + public override string ResolveReference(string path, string baseFilePath) + { + return path; + } + } +} From e1ff60ddc51a814b8d80280df73fb6511a61288e Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 14 Aug 2018 07:49:31 -0500 Subject: [PATCH 2/9] Support filename verification during analyzer testing --- .../UnitTestFramework/GenericAnalyzerTest.cs | 121 +++++++----------- .../Roslyn.UnitTestFramework.csproj | 14 ++ .../UnitTestFramework/SourceFileList.cs | 23 ++++ 3 files changed, 84 insertions(+), 74 deletions(-) create mode 100644 samples/Shared/UnitTestFramework/SourceFileList.cs diff --git a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs index 4c8a406fd..d7efb503b 100644 --- a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs +++ b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs @@ -59,6 +59,13 @@ static GenericAnalyzerTest() } #endif + protected GenericAnalyzerTest() + { + TestSources = new SourceFileList(DefaultFilePathPrefix, Language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt); + FixedSources = new SourceFileList(DefaultFilePathPrefix, Language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt); + BatchFixedSources = new SourceFileList(DefaultFilePathPrefix, Language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt); + } + /// /// Gets the language name used for the test. /// @@ -82,7 +89,7 @@ public string TestCode } } - public List TestSources { get; } = new List(); + public SourceFileList TestSources { get; } public Dictionary XmlReferences { get; } = new Dictionary(); @@ -92,6 +99,8 @@ public string TestCode public List BatchRemainingDiagnostics { get; } = new List(); + public bool VerifyExclusions { get; set; } = true; + public List DisabledDiagnostics { get; } = new List(); public int? CodeFixIndex { get; set; } @@ -108,7 +117,7 @@ public string FixedCode } } - public List FixedSources { get; } = new List(); + public SourceFileList FixedSources { get; } public string BatchFixedCode { @@ -122,7 +131,7 @@ public string BatchFixedCode } } - public List BatchFixedSources { get; } = new List(); + public SourceFileList BatchFixedSources { get; } public int? NumberOfIncrementalIterations { get; set; } @@ -139,15 +148,15 @@ public async Task RunAsync(CancellationToken cancellationToken) Assert.NotEmpty(TestSources); DiagnosticResult[] expected = ExpectedDiagnostics.ToArray(); - await VerifyDiagnosticsAsync(TestSources.ToArray(), expected, filenames: null, cancellationToken).ConfigureAwait(false); + await VerifyDiagnosticsAsync(TestSources.ToArray(), expected, cancellationToken).ConfigureAwait(false); if (HasFixableDiagnostics()) { DiagnosticResult[] remainingDiagnostics = FixedSources.SequenceEqual(TestSources) ? expected : RemainingDiagnostics.ToArray(); - await VerifyDiagnosticsAsync(FixedSources.ToArray(), remainingDiagnostics, filenames: null, cancellationToken).ConfigureAwait(false); + await VerifyDiagnosticsAsync(FixedSources.ToArray(), remainingDiagnostics, cancellationToken).ConfigureAwait(false); if (BatchFixedSources.Any()) { DiagnosticResult[] batchRemainingDiagnostics = BatchFixedSources.SequenceEqual(TestSources) ? expected : BatchRemainingDiagnostics.ToArray(); - await VerifyDiagnosticsAsync(BatchFixedSources.ToArray(), batchRemainingDiagnostics, filenames: null, cancellationToken).ConfigureAwait(false); + await VerifyDiagnosticsAsync(BatchFixedSources.ToArray(), batchRemainingDiagnostics, cancellationToken).ConfigureAwait(false); } await VerifyFixAsync(cancellationToken: cancellationToken).ConfigureAwait(false); @@ -184,10 +193,10 @@ private bool HasFixableDiagnostics() } Assert.True(FixedSources.Count == 0 - || (FixedSources.Count == 1 && string.IsNullOrEmpty(FixedSources[0])) + || (FixedSources.Count == 1 && string.IsNullOrEmpty(FixedSources[0].content)) || FixedSources.SequenceEqual(TestSources)); Assert.True(BatchFixedSources.Count == 0 - || (BatchFixedSources.Count == 1 && string.IsNullOrEmpty(BatchFixedSources[0])) + || (BatchFixedSources.Count == 1 && string.IsNullOrEmpty(BatchFixedSources[0].content)) || BatchFixedSources.SequenceEqual(TestSources)); Assert.Empty(RemainingDiagnostics); Assert.Empty(BatchRemainingDiagnostics); @@ -227,16 +236,15 @@ private static bool IsSubjectToExclusion(DiagnosticResult result) /// An array of strings to create source documents from to run the analyzers on. /// A collection of s that should appear after the analyzer /// is run on the sources. - /// The filenames or null if the default filename should be used. /// The that the task will observe. /// A representing the asynchronous operation. - private async Task VerifyDiagnosticsAsync(string[] sources, DiagnosticResult[] expected, string[] filenames, CancellationToken cancellationToken) + private async Task VerifyDiagnosticsAsync((string filename, string content)[] sources, DiagnosticResult[] expected, CancellationToken cancellationToken) { ImmutableArray analyzers = GetDiagnosticAnalyzers().ToImmutableArray(); - VerifyDiagnosticResults(await GetSortedDiagnosticsAsync(sources, analyzers, filenames, cancellationToken).ConfigureAwait(false), analyzers, expected); + VerifyDiagnosticResults(await GetSortedDiagnosticsAsync(sources, analyzers, cancellationToken).ConfigureAwait(false), analyzers, expected); - // If filenames is null we want to test for exclusions too - if (filenames == null) + // Automatically test for exclusions + if (VerifyExclusions) { // Also check if the analyzer honors exclusions if (expected.Any(IsSubjectToExclusion)) @@ -249,7 +257,7 @@ private async Task VerifyDiagnosticsAsync(string[] sources, DiagnosticResult[] e .Select(x => x.WithLineOffset(1)) .ToArray(); - VerifyDiagnosticResults(await GetSortedDiagnosticsAsync(sources.Select(x => " // \r\n" + x).ToArray(), analyzers, null, cancellationToken).ConfigureAwait(false), analyzers, expectedResults); + VerifyDiagnosticResults(await GetSortedDiagnosticsAsync(sources.Select(x => (x.filename, " // \r\n" + x.content)).ToArray(), analyzers, cancellationToken).ConfigureAwait(false), analyzers, expectedResults); } } } @@ -324,13 +332,12 @@ private static Diagnostic[] SortDistinctDiagnostics(IEnumerable diag /// /// Classes in the form of strings. /// The analyzers to be run on the sources. - /// The filenames or if the default filename should be used. /// The that the task will observe. /// A collection of s that surfaced in the source code, sorted by /// . - private Task> GetSortedDiagnosticsAsync(string[] sources, ImmutableArray analyzers, string[] filenames, CancellationToken cancellationToken) + private Task> GetSortedDiagnosticsAsync((string filename, string content)[] sources, ImmutableArray analyzers, CancellationToken cancellationToken) { - return GetSortedDiagnosticsFromDocumentsAsync(analyzers, GetDocuments(sources, filenames), cancellationToken); + return GetSortedDiagnosticsFromDocumentsAsync(analyzers, GetDocuments(sources), cancellationToken); } /// @@ -338,16 +345,15 @@ private Task> GetSortedDiagnosticsAsync(string[] sour /// documents and spans of it. /// /// Classes in the form of strings. - /// The filenames or if the default filename should be used. /// A collection of s representing the sources. - private Document[] GetDocuments(string[] sources, string[] filenames) + private Document[] GetDocuments((string filename, string content)[] sources) { if (Language != LanguageNames.CSharp && Language != LanguageNames.VisualBasic) { throw new ArgumentException("Unsupported Language"); } - Project project = CreateProject(sources, Language, filenames); + Project project = CreateProject(sources, Language); Document[] documents = project.Documents.ToArray(); if (sources.Length != documents.Length) @@ -368,12 +374,11 @@ private Document[] GetDocuments(string[] sources, string[] filenames) /// Classes in the form of strings. /// The language the source classes are in. Values may be taken from the /// class. - /// The filenames or if the default filename should be used. /// A created out of the s created from the source /// strings. - protected Project CreateProject(string[] sources, string language = LanguageNames.CSharp, string[] filenames = null) + protected Project CreateProject((string filename, string content)[] sources, string language = LanguageNames.CSharp) { - Project project = CreateProjectImpl(sources, language, filenames); + Project project = CreateProjectImpl(sources, language); return ApplyCompilationOptions(project); } @@ -383,10 +388,9 @@ protected Project CreateProject(string[] sources, string language = LanguageName /// Classes in the form of strings. /// The language the source classes are in. Values may be taken from the /// class. - /// The filenames or if the default filename should be used. /// A created out of the s created from the source /// strings. - protected virtual Project CreateProjectImpl(string[] sources, string language, string[] filenames) + protected virtual Project CreateProjectImpl((string filename, string content)[] sources, string language) { string fileNamePrefix = DefaultFilePathPrefix; string fileExt = language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt; @@ -397,8 +401,7 @@ protected virtual Project CreateProjectImpl(string[] sources, string language, s int count = 0; for (int i = 0; i < sources.Length; i++) { - string source = sources[i]; - string newFileName = filenames?[i] ?? fileNamePrefix + count + "." + fileExt; + (string newFileName, string source) = sources[i]; DocumentId documentId = DocumentId.CreateNewId(projectId, debugName: newFileName); solution = solution.AddDocument(documentId, newFileName, SourceText.From(source)); count++; @@ -735,15 +738,13 @@ private static string FormatDiagnostics(ImmutableArray analy /// /// Called to test a C# code fix when applied on the input source as a string. /// - /// An array of file names in the project before the code fix was applied. - /// An array of file names in the project after the code fix was applied. /// The that the task will observe. /// A representing the asynchronous operation. - protected async Task VerifyFixAsync(string[] oldFileNames = null, string[] newFileNames = null, CancellationToken cancellationToken = default) + protected async Task VerifyFixAsync(CancellationToken cancellationToken) { - string[] oldSources = TestSources.ToArray(); - string[] newSources = FixedSources.ToArray(); - string[] batchNewSources = BatchFixedSources.Any() ? BatchFixedSources.ToArray() : newSources; + (string filename, string content)[] oldSources = TestSources.ToArray(); + (string filename, string content)[] newSources = FixedSources.ToArray(); + (string filename, string content)[] batchNewSources = BatchFixedSources.Any() ? BatchFixedSources.ToArray() : newSources; int numberOfIncrementalIterations; int numberOfFixAllIterations; @@ -753,7 +754,7 @@ protected async Task VerifyFixAsync(string[] oldFileNames = null, string[] newFi } else { - if (!HasAnyChange(oldSources, newSources, oldFileNames, newFileNames)) + if (!HasAnyChange(oldSources, newSources)) { numberOfIncrementalIterations = 0; } @@ -769,7 +770,7 @@ protected async Task VerifyFixAsync(string[] oldFileNames = null, string[] newFi } else { - if (!HasAnyChange(oldSources, batchNewSources, oldFileNames, newFileNames)) + if (!HasAnyChange(oldSources, batchNewSources)) { numberOfFixAllIterations = 0; } @@ -779,7 +780,7 @@ protected async Task VerifyFixAsync(string[] oldFileNames = null, string[] newFi } } - ConfiguredTaskAwaitable t1 = VerifyFixpublicAsync(Language, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, newSources, oldFileNames, newFileNames, numberOfIncrementalIterations, FixEachAnalyzerDiagnosticAsync, cancellationToken).ConfigureAwait(false); + ConfiguredTaskAwaitable t1 = VerifyFixpublicAsync(Language, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, newSources, numberOfIncrementalIterations, FixEachAnalyzerDiagnosticAsync, cancellationToken).ConfigureAwait(false); ImmutableArray fixAllProvider = GetCodeFixProviders().Select(codeFixProvider => codeFixProvider.GetFixAllProvider()).Where(codeFixProvider => codeFixProvider != null).ToImmutableArray(); @@ -794,19 +795,19 @@ protected async Task VerifyFixAsync(string[] oldFileNames = null, string[] newFi await t1; } - ConfiguredTaskAwaitable t2 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, oldFileNames, newFileNames, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInDocumentAsync, cancellationToken).ConfigureAwait(false); + ConfiguredTaskAwaitable t2 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInDocumentAsync, cancellationToken).ConfigureAwait(false); if (Debugger.IsAttached) { await t2; } - ConfiguredTaskAwaitable t3 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, oldFileNames, newFileNames, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInProjectAsync, cancellationToken).ConfigureAwait(false); + ConfiguredTaskAwaitable t3 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInProjectAsync, cancellationToken).ConfigureAwait(false); if (Debugger.IsAttached) { await t3; } - ConfiguredTaskAwaitable t4 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, oldFileNames, newFileNames, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInSolutionAsync, cancellationToken).ConfigureAwait(false); + ConfiguredTaskAwaitable t4 = VerifyFixpublicAsync(LanguageNames.CSharp, GetDiagnosticAnalyzers().ToImmutableArray(), GetCodeFixProviders().ToImmutableArray(), oldSources, batchNewSources ?? newSources, numberOfFixAllIterations, FixAllAnalyzerDiagnosticsInSolutionAsync, cancellationToken).ConfigureAwait(false); if (Debugger.IsAttached) { await t4; @@ -827,27 +828,13 @@ private async Task VerifyFixpublicAsync( string language, ImmutableArray analyzers, ImmutableArray codeFixProviders, - string[] oldSources, - string[] newSources, - string[] oldFileNames, - string[] newFileNames, + (string filename, string content)[] oldSources, + (string filename, string content)[] newSources, int numberOfIterations, Func, ImmutableArray, int?, Project, int, CancellationToken, Task> getFixedProject, CancellationToken cancellationToken) { - if (oldFileNames != null) - { - // Make sure the test case is consistent regarding the number of sources and file names before the code fix - Assert.Equal($"{oldSources.Length} old file names", $"{oldFileNames.Length} old file names"); - } - - if (newFileNames != null) - { - // Make sure the test case is consistent regarding the number of sources and file names after the code fix - Assert.Equal($"{newSources.Length} new file names", $"{newFileNames.Length} new file names"); - } - - Project project = CreateProject(oldSources, language, oldFileNames); + Project project = CreateProject(oldSources, language); ImmutableArray compilerDiagnostics = await GetCompilerDiagnosticsAsync(project, cancellationToken).ConfigureAwait(false); project = await getFixedProject(analyzers, codeFixProviders, CodeFixIndex, project, numberOfIterations, cancellationToken).ConfigureAwait(false); @@ -882,28 +869,14 @@ private async Task VerifyFixpublicAsync( for (int i = 0; i < updatedDocuments.Length; i++) { string actual = await GetStringFromDocumentAsync(updatedDocuments[i], cancellationToken).ConfigureAwait(false); - Assert.Equal(newSources[i], actual); - - if (newFileNames != null) - { - Assert.Equal(newFileNames[i], updatedDocuments[i].Name); - } + Assert.Equal(newSources[i].content, actual); + Assert.Equal(newSources[i].filename, updatedDocuments[i].Name); } } - private static bool HasAnyChange(string[] oldSources, string[] newSources, string[] oldFileNames, string[] newFileNames) + private static bool HasAnyChange((string filename, string content)[] oldSources, (string filename, string content)[] newSources) { - if (!oldSources.SequenceEqual(newSources)) - { - return true; - } - - if (oldFileNames != null && newFileNames != null && !oldFileNames.SequenceEqual(newFileNames)) - { - return true; - } - - return false; + return !oldSources.SequenceEqual(newSources); } private static async Task FixEachAnalyzerDiagnosticAsync(ImmutableArray analyzers, ImmutableArray codeFixProviders, int? codeFixIndex, Project project, int numberOfIterations, CancellationToken cancellationToken) diff --git a/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj b/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj index 563b00066..95f888662 100644 --- a/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj +++ b/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj @@ -4,6 +4,19 @@ netstandard1.5;net452 + + + $(NoWarn),1573,1591,1712 + true + + 7.2 @@ -12,6 +25,7 @@ + diff --git a/samples/Shared/UnitTestFramework/SourceFileList.cs b/samples/Shared/UnitTestFramework/SourceFileList.cs new file mode 100644 index 000000000..65243bd81 --- /dev/null +++ b/samples/Shared/UnitTestFramework/SourceFileList.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace Roslyn.UnitTestFramework +{ + public class SourceFileList : List<(string filename, string content)> + { + private readonly string _defaultPrefix; + private readonly string _defaultExtension; + + public SourceFileList(string defaultPrefix, string defaultExtension) + { + _defaultPrefix = defaultPrefix; + _defaultExtension = defaultExtension; + } + + public void Add(string content) + { + Add(($"{_defaultPrefix}{Count}.{_defaultExtension}", content)); + } + } +} From 7ed9da1afb2691648460d6afdf38a2bcbaf94a74 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 14 Aug 2018 08:31:04 -0500 Subject: [PATCH 3/9] Improved documentation for test setup --- .../UnitTestFramework/GenericAnalyzerTest.cs | 129 +++++++++++++++++- 1 file changed, 128 insertions(+), 1 deletion(-) diff --git a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs index d7efb503b..1a58434c0 100644 --- a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs +++ b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs @@ -77,6 +77,10 @@ public abstract string Language get; } + /// + /// Sets the input source file for analyzer or code fix testing. + /// + /// public string TestCode { set @@ -89,22 +93,61 @@ public string TestCode } } + /// + /// Gets the set of input source files for analyzer or code fix testing. Files may be added to this list using + /// one of the methods. + /// public SourceFileList TestSources { get; } + /// + /// Gets the collection of inputs to provide to the XML documentation resolver. + /// + /// + /// Files in this collection may be referenced via <include> elements in documentation + /// comments. + /// public Dictionary XmlReferences { get; } = new Dictionary(); + /// + /// Gets the list of diagnostics expected in the input source(s). + /// public List ExpectedDiagnostics { get; } = new List(); + /// + /// Gets the list of diagnostics expected after a code fix is applied. + /// public List RemainingDiagnostics { get; } = new List(); + /// + /// Gets the list of diagnostics expected after a Fix All operation. + /// + /// + /// By default, Fix All operations are expected to produce the same result as incremental fix operations. + /// This collection is only used when differs from + /// . + /// public List BatchRemainingDiagnostics { get; } = new List(); + /// + /// Gets or sets a value indicating whether exclusions for generated code should be tested automatically. The + /// default value is . + /// public bool VerifyExclusions { get; set; } = true; + /// + /// Gets a collection of diagnostics to explicitly disable in the for projects. + /// public List DisabledDiagnostics { get; } = new List(); + /// + /// Gets or sets the index of the code fix to apply. + /// public int? CodeFixIndex { get; set; } + /// + /// Sets the expected output source file for code fix testing. + /// + /// public string FixedCode { set @@ -117,8 +160,16 @@ public string FixedCode } } + /// + /// Gets the set of expected output files for code fix testing. Files may be added to this list using one of the + /// methods. + /// public SourceFileList FixedSources { get; } + /// + /// Sets the expected output source file after a Fix All operation is applied. + /// + /// public string BatchFixedCode { set @@ -131,16 +182,90 @@ public string BatchFixedCode } } + /// + /// Gets the set of expected output files after a Fix All operation is applied. Files may be added to this list + /// using one of the methods. + /// + /// + /// By default, Fix All operations are expected to produce the same result as incremental fix operations. + /// If this collection is not specified for the test, provides the expected test + /// results for both incremental and Fix All scenarios. + /// public SourceFileList BatchFixedSources { get; } + /// + /// Gets or sets the number of code fix iterations expected during code fix testing. + /// + /// + /// Code fixes are applied until one of the following conditions are met: + /// + /// + /// No diagnostics are reported in the input. + /// No code fixes are provided for the diagnostics reported in the input. + /// The code fix applied for the diagnostics does not produce a change in the source file(s). + /// The maximum number of allowed iterations is exceeded. + /// + /// + /// If the number of iterations is positive, it represents an exact number of iterations: code fix tests + /// will fail if the code fix required more or fewer iterations to complete. If the number of iterations is + /// negative, the negation of the number of iterations is treated as an upper bound on the number of allowed + /// iterations: code fix tests will fail only if the code fix required more iterations to complete. If the + /// number of iterations is zero, the code fix test will validate that no code fixes are offered for the set of + /// diagnostics reported in the original input. + /// + /// When the number of iterations is not specified, the value is automatically selected according to the + /// current test configuration: + /// + /// + /// If the expected code fix output equals the input sources, the default value is treated as 0. + /// Otherwise, the default value is treated as the negative of the number of fixable diagnostics appearing in the input source file(s). + /// + /// + /// + /// The default value for this property can be interpreted as "Iterative code fix operations are expected + /// to complete in at most one operation for each fixable diagnostic in the input source has been applied. + /// Completing in fewer iterations is acceptable." + /// + /// public int? NumberOfIncrementalIterations { get; set; } + /// + /// Gets or sets the number of code fix iterations expected during code fix testing for Fix All scenarios. + /// + /// + /// See the property for an overview of the behavior of this + /// property. If the number of Fix All iterations is not specified, the value is automatically selected + /// according to the current test configuration: + /// + /// + /// If the expected Fix All output equals the input sources, the default value is treated as 0. + /// Otherwise, the default value is treated as 1. + /// + /// + /// + /// The default value for this property can be interpreted as "Fix All operations are expected to complete + /// in the minimum number of iterations possible unless otherwise specified." + /// + /// + /// public int? NumberOfFixAllIterations { get; set; } + /// + /// Gets or sets a value indicating whether new compiler diagnostics are allowed to appear in code fix outputs. + /// The default value is . + /// public bool AllowNewCompilerDiagnostics { get; set; } = false; + /// + /// Gets a collection of transformation functions to apply to during diagnostic + /// or code fix test setup. + /// public List> OptionsTransforms { get; } = new List>(); + /// + /// Gets a collection of transformation functions to apply to a during diagnostic or code + /// fix test setup. + /// public List> SolutionTransforms { get; } = new List>(); public async Task RunAsync(CancellationToken cancellationToken) @@ -174,7 +299,7 @@ public async Task RunAsync(CancellationToken cancellationToken) /// /// Returns the code fixes being tested - to be implemented in non-abstract class. /// - /// The to be used for C# code. + /// The to be used. protected abstract IEnumerable GetCodeFixProviders(); private bool HasFixableDiagnostics() @@ -213,11 +338,13 @@ private static bool IsSubjectToExclusion(DiagnosticResult result) { if (result.Id.StartsWith("CS", StringComparison.Ordinal)) { + // This is a compiler diagnostic return false; } if (result.Id.StartsWith("AD", StringComparison.Ordinal)) { + // This diagnostic is reported by the analyzer infrastructure return false; } From c0f8098a6a648ee96bd4aa89b4049cd621345fff Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 14 Aug 2018 08:33:02 -0500 Subject: [PATCH 4/9] Code cleanup for items found in review * Make CancellationToken optional for RunAsync in test scenarios * Handle VB compiler diagnostics in the same manner as CS * Use IsEmpty instead of testing Length --- samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs index 1a58434c0..0518a3dd8 100644 --- a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs +++ b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs @@ -268,7 +268,7 @@ public string BatchFixedCode /// public List> SolutionTransforms { get; } = new List>(); - public async Task RunAsync(CancellationToken cancellationToken) + public async Task RunAsync(CancellationToken cancellationToken = default) { Assert.NotEmpty(TestSources); @@ -336,7 +336,8 @@ bool HasFixableDiagnosticsCore() private static bool IsSubjectToExclusion(DiagnosticResult result) { - if (result.Id.StartsWith("CS", StringComparison.Ordinal)) + if (result.Id.StartsWith("CS", StringComparison.Ordinal) + || result.Id.StartsWith("VB", StringComparison.Ordinal)) { // This is a compiler diagnostic return false; @@ -348,7 +349,7 @@ private static bool IsSubjectToExclusion(DiagnosticResult result) return false; } - if (result.Spans.Length == 0) + if (result.Spans.IsEmpty) { return false; } From 4a95fa0f098c1f97e7b73dfae21d59c56af5e523 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 14 Aug 2018 09:33:46 -0500 Subject: [PATCH 5/9] Add initial tests for DiagnosticVerifier --- Samples.sln | 7 + .../CSharpSyntaxTreeDiagnosticAnalyzer.cs | 86 ++++ .../DiagnosticVerifierTest.cs | 453 ++++++++++++++++++ .../Roslyn.UnitTestFramework.Test.csproj | 40 ++ .../UnitTestFramework/DiagnosticResult.cs | 10 +- 5 files changed, 591 insertions(+), 5 deletions(-) create mode 100644 samples/Shared/UnitTestFramework.Test/CSharpSyntaxTreeDiagnosticAnalyzer.cs create mode 100644 samples/Shared/UnitTestFramework.Test/DiagnosticVerifierTest.cs create mode 100644 samples/Shared/UnitTestFramework.Test/Roslyn.UnitTestFramework.Test.csproj diff --git a/Samples.sln b/Samples.sln index aa4a4cab1..4c03585e8 100644 --- a/Samples.sln +++ b/Samples.sln @@ -115,6 +115,8 @@ Project("{778DAE3C-4631-46EA-AA77-85C1314464D9}") = "VisualBasicToCSharpConverte EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SolutionExplorer", "samples\CSharp\SolutionExplorer\SolutionExplorer.csproj", "{B0474F4F-A6A9-4F10-BC49-CE2957201DBB}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Roslyn.UnitTestFramework.Test", "samples\Shared\UnitTestFramework.Test\Roslyn.UnitTestFramework.Test.csproj", "{04DABE2E-7230-4992-8E8B-F6BEACB11AA5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -281,6 +283,10 @@ Global {B0474F4F-A6A9-4F10-BC49-CE2957201DBB}.Debug|Any CPU.Build.0 = Debug|Any CPU {B0474F4F-A6A9-4F10-BC49-CE2957201DBB}.Release|Any CPU.ActiveCfg = Release|Any CPU {B0474F4F-A6A9-4F10-BC49-CE2957201DBB}.Release|Any CPU.Build.0 = Release|Any CPU + {04DABE2E-7230-4992-8E8B-F6BEACB11AA5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {04DABE2E-7230-4992-8E8B-F6BEACB11AA5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {04DABE2E-7230-4992-8E8B-F6BEACB11AA5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {04DABE2E-7230-4992-8E8B-F6BEACB11AA5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -339,6 +345,7 @@ Global {ECB83742-8023-4609-B139-D7B78DD66ED9} = {8E1C9AEC-6EF1-43A8-A378-52C5C0E40532} {5B7D7569-B5EE-4C01-9AFA-BC1958588160} = {8E1C9AEC-6EF1-43A8-A378-52C5C0E40532} {B0474F4F-A6A9-4F10-BC49-CE2957201DBB} = {C3FB27E9-C8EE-4F76-B0AA-7CD67A7E652B} + {04DABE2E-7230-4992-8E8B-F6BEACB11AA5} = {A54A1AB7-DBD6-4C31-A22E-C53674137C53} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B849838B-3D7A-4B6B-BE07-285DCB1588F4} diff --git a/samples/Shared/UnitTestFramework.Test/CSharpSyntaxTreeDiagnosticAnalyzer.cs b/samples/Shared/UnitTestFramework.Test/CSharpSyntaxTreeDiagnosticAnalyzer.cs new file mode 100644 index 000000000..90f95e168 --- /dev/null +++ b/samples/Shared/UnitTestFramework.Test/CSharpSyntaxTreeDiagnosticAnalyzer.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Immutable; +using System.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; + +namespace Roslyn.UnitTestFramework.Test +{ + [DiagnosticAnalyzer(LanguageNames.CSharp)] + internal class CSharpSyntaxTreeDiagnosticAnalyzer : DiagnosticAnalyzer + { + public const string DiagnosticId = "TEST01"; + public static readonly LocalizableString Title = "Test title"; + public static readonly LocalizableString MessageFormat = "Semicolons should be {0} by a space"; + public static readonly string Category = "Test"; + public static readonly DiagnosticSeverity DefaultSeverity = DiagnosticSeverity.Warning; + public static readonly bool IsEnabledByDefault = true; + public static readonly LocalizableString Description = "Test description"; + public static readonly string HelpLinkUri = "https://github.com/dotnet/roslyn-sdk"; + public static readonly ImmutableArray CustomTags = ImmutableArray.Create(WellKnownDiagnosticTags.Unnecessary); + + private static readonly DiagnosticDescriptor Descriptor = new DiagnosticDescriptor(DiagnosticId, Title, MessageFormat, Category, DefaultSeverity, IsEnabledByDefault, Description, HelpLinkUri, CustomTags.ToArray()); + + public override ImmutableArray SupportedDiagnostics { get; } + = ImmutableArray.Create(Descriptor); + + public override void Initialize(AnalysisContext context) + { + context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + context.EnableConcurrentExecution(); + + context.RegisterSyntaxTreeAction(HandleSyntaxTree); + } + + private static void HandleSyntaxTree(SyntaxTreeAnalysisContext context) + { + SyntaxNode root = context.Tree.GetCompilationUnitRoot(context.CancellationToken); + foreach (SyntaxToken token in root.DescendantTokens()) + { + switch (token.Kind()) + { + case SyntaxKind.SemicolonToken: + HandleSemicolonToken(context, token); + break; + + default: + break; + } + } + } + + private static void HandleSemicolonToken(SyntaxTreeAnalysisContext context, SyntaxToken token) + { + // check for a following space + bool missingFollowingSpace = true; + if (token.HasTrailingTrivia) + { + if (token.TrailingTrivia.First().IsKind(SyntaxKind.EndOfLineTrivia)) + { + missingFollowingSpace = false; + } + } + + if (missingFollowingSpace) + { + // semicolon should{} be {followed} by a space + context.ReportDiagnostic(Diagnostic.Create(Descriptor, token.GetLocation(), TokenSpacingProperties.InsertFollowing, "followed")); + } + } + + internal static class TokenSpacingProperties + { + internal const string LocationKey = "location"; + internal const string ActionKey = "action"; + internal const string LocationFollowing = "following"; + internal const string ActionInsert = "insert"; + + internal static ImmutableDictionary InsertFollowing { get; } = + ImmutableDictionary.Empty + .SetItem(LocationKey, LocationFollowing) + .SetItem(ActionKey, ActionInsert); + } + } +} diff --git a/samples/Shared/UnitTestFramework.Test/DiagnosticVerifierTest.cs b/samples/Shared/UnitTestFramework.Test/DiagnosticVerifierTest.cs new file mode 100644 index 000000000..bf042a78c --- /dev/null +++ b/samples/Shared/UnitTestFramework.Test/DiagnosticVerifierTest.cs @@ -0,0 +1,453 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.Diagnostics; +using Xunit; +using Xunit.Sdk; +using static Roslyn.UnitTestFramework.DiagnosticVerifier; + +namespace Roslyn.UnitTestFramework.Test +{ + /// + /// This class verifies that will correctly report failing tests. + /// + public class DiagnosticVerifierTest + { + [Fact] + public async Task TestExpectedDiagnosticMissingAsync() + { + string testCode = @" +class ClassName +{ + void MethodName() + { + ; + } +} +"; + + DiagnosticResult expected = Diagnostic(); + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Mismatch between number of diagnostics returned, expected \"1\" actual \"0\"", ex.Message); + } + + [Fact] + public async Task TestValidBehaviorAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithLocation(7, 33); + + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + } + + [Fact] + public async Task TestValidBehaviorUncheckedLineAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithLocation(0, 33); + + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + } + + [Fact] + public async Task TestValidBehaviorUncheckedColumnAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithLocation(7, 0); + + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + } + + [Fact] + public async Task TestValidBehaviorWithFullSpanAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithSpan(7, 33, 7, 34); + + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + } + + [Fact] + public async Task TestUnexpectedLocationForProjectDiagnosticAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + // By failing to include a location, the verified thinks we're only trying to verify a project diagnostic. + DiagnosticResult expected = Diagnostic().WithArguments("followed"); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Expected:\nA project diagnostic with No location\nActual:\n", ex.Message); + } + + [Fact] + public async Task TestUnexpectedMessageAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, EmptyDiagnosticResults, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Mismatch between number of diagnostics returned, expected \"0\" actual \"1\"", ex.Message); + Assert.Contains("warning TEST01", ex.Message); + } + + [Fact] + public async Task TestUnexpectedAnalyzerErrorAsync() + { + string testCode = @" +class ClassName +{ + void MethodName() + { + ; + } +} +"; + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await DiagnosticVerifier.VerifyCSharpDiagnosticAsync(testCode, EmptyDiagnosticResults, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Mismatch between number of diagnostics returned, expected \"0\" actual \"2\"", ex.Message); + Assert.Contains("error AD0001", ex.Message); + } + + [Fact] + public async Task TestUnexpectedCompilerErrorAsync() + { + string testCode = @" +class ClassName +{ + int property; + Int32 PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithLocation(7, 33); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Mismatch between number of diagnostics returned, expected \"1\" actual \"2\"", ex.Message); + Assert.Contains("error CS0246", ex.Message); + } + + [Fact] + public async Task TestUnexpectedCompilerWarningAsync() + { + string testCode = @" +class ClassName +{ + int property; + Int32 PropertyName + { + /// + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithLocation(8, 33); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Mismatch between number of diagnostics returned, expected \"1\" actual \"2\"", ex.Message); + Assert.Contains("error CS0246", ex.Message); + } + + [Fact] + public async Task TestInvalidDiagnosticIdAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticDescriptor descriptor = new DiagnosticDescriptor("SA9999", "Title", "Message", "Category", DiagnosticSeverity.Warning, isEnabledByDefault: true); + DiagnosticResult expected = Diagnostic(descriptor).WithLocation(7, 33); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith($"Expected diagnostic id to be \"SA9999\" was \"{CSharpSyntaxTreeDiagnosticAnalyzer.DiagnosticId}\"", ex.Message); + } + + [Fact] + public async Task TestInvalidSeverityAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithLocation(7, 33).WithSeverity(DiagnosticSeverity.Error); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Expected diagnostic severity to be \"Error\" was \"Warning\"", ex.Message); + } + + [Fact] + public async Task TestIncorrectLocationLine1Async() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithLocation(8, 33); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Expected diagnostic to start on line \"8\" was actually on line \"7\"", ex.Message); + } + + [Fact] + public async Task TestIncorrectLocationLine2Async() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + set{this.property = value;} + } +} +"; + + DiagnosticResult[] expected = + { + Diagnostic().WithArguments("followed").WithLocation(7, 33), + Diagnostic().WithArguments("followed").WithLocation(7, 34), + }; + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Expected diagnostic to start on line \"7\" was actually on line \"8\"", ex.Message); + } + + [Fact] + public async Task TestIncorrectLocationColumnAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithLocation(7, 34); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Expected diagnostic to start at column \"34\" was actually at column \"33\"", ex.Message); + } + + [Fact] + public async Task TestIncorrectLocationEndColumnAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("followed").WithSpan(7, 33, 7, 35); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Expected diagnostic to end at column \"35\" was actually at column \"34\"", ex.Message); + } + + [Fact] + public async Task TestIncorrectMessageAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("bogus argument").WithLocation(7, 33); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Expected diagnostic message to be ", ex.Message); + } + + [Fact] + public async Task TestIncorrectAdditionalLocationAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + } +} +"; + + DiagnosticResult expected = Diagnostic().WithArguments("bogus argument").WithLocation(7, 33).WithLocation(8, 34); + + XunitException ex = await Assert.ThrowsAnyAsync( + async () => + { + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + }).ConfigureAwait(false); + Assert.StartsWith("Expected 1 additional locations but got 0 for Diagnostic", ex.Message); + } + + private class ErrorThrowingAnalyzer : CSharpSyntaxTreeDiagnosticAnalyzer + { + private static readonly Action BlockAction = HandleBlock; + + /// + public override void Initialize(AnalysisContext context) + { + context.RegisterSyntaxNodeAction(BlockAction, SyntaxKind.Block); + } + + private static void HandleBlock(SyntaxNodeAnalysisContext context) + { + throw new NotImplementedException(); + } + } + } +} diff --git a/samples/Shared/UnitTestFramework.Test/Roslyn.UnitTestFramework.Test.csproj b/samples/Shared/UnitTestFramework.Test/Roslyn.UnitTestFramework.Test.csproj new file mode 100644 index 000000000..d969b67fc --- /dev/null +++ b/samples/Shared/UnitTestFramework.Test/Roslyn.UnitTestFramework.Test.csproj @@ -0,0 +1,40 @@ + + + + net452;netstandard1.5 + + + + + $(NoWarn),1573,1591,1712 + true + + + + 7.2 + + + + + + + + + + + + + + portable-net45+win8 + + + + + diff --git a/samples/Shared/UnitTestFramework/DiagnosticResult.cs b/samples/Shared/UnitTestFramework/DiagnosticResult.cs index 571f4064d..63a669dc5 100644 --- a/samples/Shared/UnitTestFramework/DiagnosticResult.cs +++ b/samples/Shared/UnitTestFramework/DiagnosticResult.cs @@ -145,12 +145,12 @@ public DiagnosticResult WithLineOffset(int offset) { DiagnosticResult result = this; ImmutableArray.Builder spansBuilder = result._spans.ToBuilder(); - for (int i = 0; i < result._spans.Length; i++) + for (int i = 0; i < result.Spans.Length; i++) { - LinePosition newStartLinePosition = new LinePosition(result._spans[i].StartLinePosition.Line + offset, result._spans[i].StartLinePosition.Character); - LinePosition newEndLinePosition = new LinePosition(result._spans[i].EndLinePosition.Line + offset, result._spans[i].EndLinePosition.Character); + LinePosition newStartLinePosition = new LinePosition(result.Spans[i].StartLinePosition.Line + offset, result.Spans[i].StartLinePosition.Character); + LinePosition newEndLinePosition = new LinePosition(result.Spans[i].EndLinePosition.Line + offset, result.Spans[i].EndLinePosition.Character); - spansBuilder[i] = new FileLinePositionSpan(result._spans[i].Path, newStartLinePosition, newEndLinePosition); + spansBuilder[i] = new FileLinePositionSpan(result.Spans[i].Path, newStartLinePosition, newEndLinePosition); } result._spans = spansBuilder.MoveToImmutable(); @@ -159,7 +159,7 @@ public DiagnosticResult WithLineOffset(int offset) private DiagnosticResult AppendSpan(FileLinePositionSpan span) { - ImmutableArray newSpans = _spans.Add(span); + ImmutableArray newSpans = Spans.Add(span); // clone the object, so that the fluent syntax will work on immutable objects. return new DiagnosticResult From 180fb4b34a42110677b8fac7a9b8ba3d4951eca1 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 14 Aug 2018 09:41:10 -0500 Subject: [PATCH 6/9] Add third party license notice for StyleCop Analyzers --- THIRD-PARTY-NOTICES.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 THIRD-PARTY-NOTICES.txt diff --git a/THIRD-PARTY-NOTICES.txt b/THIRD-PARTY-NOTICES.txt new file mode 100644 index 000000000..422f036f8 --- /dev/null +++ b/THIRD-PARTY-NOTICES.txt @@ -0,0 +1,25 @@ +Roslyn SDK uses third-party libraries or other resources that may be +distributed under licenses different than the Roslyn SDK software. + +In the event that we accidentally failed to list a required notice, please +bring it to our attention. Post an issue or email us: + + dotnet@microsoft.com + +The attached notices are provided for information only. + +License notice for StyleCop Analyzers +------------------------------------- + +Copyright (c) Tunnel Vision Laboratories, LLC. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +these files except in compliance with the License. You may obtain a copy of the +License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. From 56d48d941804cda46d4af0b7de329b03ab4149eb Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 14 Aug 2018 11:58:13 -0500 Subject: [PATCH 7/9] Support markup in test inputs and outputs --- .../DiagnosticVerifierTest.cs | 93 ++++++++++++++ .../UnitTestFramework/DiagnosticResult.cs | 22 ++-- .../UnitTestFramework/DictionaryExtensions.cs | 6 + .../UnitTestFramework/GenericAnalyzerTest.cs | 119 ++++++++++++++++-- 4 files changed, 220 insertions(+), 20 deletions(-) diff --git a/samples/Shared/UnitTestFramework.Test/DiagnosticVerifierTest.cs b/samples/Shared/UnitTestFramework.Test/DiagnosticVerifierTest.cs index bf042a78c..9adb250cf 100644 --- a/samples/Shared/UnitTestFramework.Test/DiagnosticVerifierTest.cs +++ b/samples/Shared/UnitTestFramework.Test/DiagnosticVerifierTest.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.CodeAnalysis; @@ -434,6 +435,98 @@ int PropertyName Assert.StartsWith("Expected 1 additional locations but got 0 for Diagnostic", ex.Message); } + [Fact] + public async Task TestDiagnosticsUnorderedAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + set{this.property = value;} + } +} +"; + + DiagnosticResult[] expected = + { + Diagnostic().WithArguments("followed").WithLocation(7, 33), + Diagnostic().WithArguments("followed").WithLocation(8, 34), + }; + + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + + expected = expected.Reverse().ToArray(); + + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + } + + [Fact] + public async Task TestMarkupThenExplicitAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property[|;|]} + set{this.property = value;} + } +} +"; + + DiagnosticResult[] expected = + { + Diagnostic().WithArguments("followed").WithLocation(8, 34), + }; + + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + } + + [Fact] + public async Task TestExplicitThenMarkupAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property;} + set{this.property = value[|;|]} + } +} +"; + + DiagnosticResult[] expected = + { + Diagnostic().WithArguments("followed").WithLocation(7, 33), + }; + + await VerifyCSharpDiagnosticAsync(testCode, expected, CancellationToken.None).ConfigureAwait(false); + } + + [Fact] + public async Task TestMarkupAsync() + { + string testCode = @" +class ClassName +{ + int property; + int PropertyName + { + get{return this.property[|;|]} + set{this.property = value[|;|]} + } +} +"; + + await VerifyCSharpDiagnosticAsync(testCode, EmptyDiagnosticResults, CancellationToken.None).ConfigureAwait(false); + } + private class ErrorThrowingAnalyzer : CSharpSyntaxTreeDiagnosticAnalyzer { private static readonly Action BlockAction = HandleBlock; diff --git a/samples/Shared/UnitTestFramework/DiagnosticResult.cs b/samples/Shared/UnitTestFramework/DiagnosticResult.cs index 63a669dc5..4cad14630 100644 --- a/samples/Shared/UnitTestFramework/DiagnosticResult.cs +++ b/samples/Shared/UnitTestFramework/DiagnosticResult.cs @@ -16,6 +16,7 @@ public struct DiagnosticResult private static readonly object[] EmptyArguments = new object[0]; private ImmutableArray _spans; + private bool _suppressMessage; private string _message; public DiagnosticResult(string id, DiagnosticSeverity severity) @@ -57,6 +58,11 @@ public string Message { get { + if (_suppressMessage) + { + return null; + } + if (_message != null) { return _message; @@ -109,6 +115,7 @@ public DiagnosticResult WithMessage(string message) { DiagnosticResult result = this; result._message = message; + result._suppressMessage = message is null; return result; } @@ -159,18 +166,9 @@ public DiagnosticResult WithLineOffset(int offset) private DiagnosticResult AppendSpan(FileLinePositionSpan span) { - ImmutableArray newSpans = Spans.Add(span); - - // clone the object, so that the fluent syntax will work on immutable objects. - return new DiagnosticResult - { - Id = Id, - _message = _message, - MessageFormat = MessageFormat, - MessageArguments = MessageArguments, - Severity = Severity, - _spans = newSpans, - }; + DiagnosticResult result = this; + result._spans = Spans.Add(span); + return result; } } } diff --git a/samples/Shared/UnitTestFramework/DictionaryExtensions.cs b/samples/Shared/UnitTestFramework/DictionaryExtensions.cs index 7d08828f0..2bde9b6f2 100644 --- a/samples/Shared/UnitTestFramework/DictionaryExtensions.cs +++ b/samples/Shared/UnitTestFramework/DictionaryExtensions.cs @@ -21,5 +21,11 @@ public static TValue GetOrAdd(this IDictionary dicti public static TValue GetOrAdd(this IDictionary dictionary, TKey key, Func function) => dictionary.GetOrAdd(key, _ => function()); + + public static void Deconstruct(this KeyValuePair pair, out TKey key, out TValue value) + { + key = pair.Key; + value = pair.Value; + } } } diff --git a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs index 0518a3dd8..cbb69f390 100644 --- a/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs +++ b/samples/Shared/UnitTestFramework/GenericAnalyzerTest.cs @@ -250,6 +250,27 @@ public string BatchFixedCode /// public int? NumberOfFixAllIterations { get; set; } + /// + /// Gets or sets a value indicating whether markup can be used to identify diagnostics within expected inputs + /// and outputs. The default value is . + /// + /// + /// Diagnostics expressed using markup are combined with explicitly-specified expected diagnostics. + /// + /// Supported markup syntax includes the following: + /// + /// + /// [|text|]: indicates that a diagnostic is reported for text. The diagnostic + /// descriptor is located via . This syntax may only be used when the first + /// analyzer provided by supports a single diagnostic. + /// {|ID1:text|}: indicates that a diagnostic with ID ID1 is reported for + /// text. The diagnostic descriptor for ID1 is located via . + /// If no matching descriptor is found, the diagnostic is assumed to be a compiler-reported diagnostic with the + /// specified ID and severity . + /// + /// + public bool AllowMarkup { get; set; } = true; + /// /// Gets or sets a value indicating whether new compiler diagnostics are allowed to appear in code fix outputs. /// The default value is . @@ -272,22 +293,104 @@ public async Task RunAsync(CancellationToken cancellationToken = default) { Assert.NotEmpty(TestSources); - DiagnosticResult[] expected = ExpectedDiagnostics.ToArray(); - await VerifyDiagnosticsAsync(TestSources.ToArray(), expected, cancellationToken).ConfigureAwait(false); + (DiagnosticResult[] expected, (string filename, string content)[] testSources) = ProcessMarkupSources(TestSources, ExpectedDiagnostics); + await VerifyDiagnosticsAsync(testSources, expected, cancellationToken).ConfigureAwait(false); if (HasFixableDiagnostics()) { - DiagnosticResult[] remainingDiagnostics = FixedSources.SequenceEqual(TestSources) ? expected : RemainingDiagnostics.ToArray(); - await VerifyDiagnosticsAsync(FixedSources.ToArray(), remainingDiagnostics, cancellationToken).ConfigureAwait(false); + (DiagnosticResult[] remainingDiagnostics, (string filename, string content)[] fixedSources) = FixedSources.SequenceEqual(TestSources) + ? (expected, testSources) + : ProcessMarkupSources(FixedSources, RemainingDiagnostics); + await VerifyDiagnosticsAsync(fixedSources, remainingDiagnostics, cancellationToken).ConfigureAwait(false); if (BatchFixedSources.Any()) { - DiagnosticResult[] batchRemainingDiagnostics = BatchFixedSources.SequenceEqual(TestSources) ? expected : BatchRemainingDiagnostics.ToArray(); - await VerifyDiagnosticsAsync(BatchFixedSources.ToArray(), batchRemainingDiagnostics, cancellationToken).ConfigureAwait(false); + (DiagnosticResult[] batchRemainingDiagnostics, (string filename, string content)[] batchFixedSources) = BatchFixedSources.SequenceEqual(TestSources) + ? (expected, testSources) + : ProcessMarkupSources(BatchFixedSources, BatchRemainingDiagnostics); + await VerifyDiagnosticsAsync(batchFixedSources, batchRemainingDiagnostics, cancellationToken).ConfigureAwait(false); } await VerifyFixAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } } + private (DiagnosticResult[], (string filename, string content)[]) ProcessMarkupSources(IEnumerable<(string filename, string content)> sources, IEnumerable explicitDiagnostics) + { + if (!AllowMarkup) + { + return (ToOrderedArray(explicitDiagnostics), sources.ToArray()); + } + + DiagnosticAnalyzer[] analyzers = GetDiagnosticAnalyzers().ToArray(); + List<(string filename, string content)> sourceFiles = new List<(string filename, string content)>(); + List diagnostics = new List(explicitDiagnostics); + foreach ((string filename, string content) in sources) + { + MarkupTestFile.GetSpans(content, out string output, out IDictionary> namedSpans); + sourceFiles.Add((filename, output)); + if (namedSpans.Count == 0) + { + // No markup notation in this input + continue; + } + + SourceText sourceText = SourceText.From(output); + foreach ((string name, IList spans) in namedSpans) + { + foreach (TextSpan span in spans) + { + diagnostics.Add(CreateDiagnosticForSpan(analyzers, name, filename, sourceText, span)); + } + } + } + + return (ToOrderedArray(diagnostics), sourceFiles.ToArray()); + } + + private DiagnosticResult CreateDiagnosticForSpan(DiagnosticAnalyzer[] analyzers, string diagnosticId, string filename, SourceText content, TextSpan span) + { + LinePositionSpan linePositionSpan = content.Lines.GetLinePositionSpan(span); + + DiagnosticResult diagnosticResult; + if (diagnosticId == "") + { + diagnosticResult = new DiagnosticResult(analyzers.First().SupportedDiagnostics.Single()); + } + else + { + DiagnosticDescriptor descriptor = analyzers.SelectMany(analyzer => analyzer.SupportedDiagnostics).SingleOrDefault(d => d.Id == diagnosticId); + if (descriptor != null) + { + diagnosticResult = new DiagnosticResult(descriptor); + } + else + { + // This must be a compiler error + diagnosticResult = new DiagnosticResult(diagnosticId, DiagnosticSeverity.Error); + } + } + + return diagnosticResult + .WithMessage(null) + .WithSpan( + filename, + linePositionSpan.Start.Line + 1, + linePositionSpan.Start.Character + 1, + linePositionSpan.End.Line + 1, + linePositionSpan.End.Character + 1); + } + + private static DiagnosticResult[] ToOrderedArray(IEnumerable diagnosticResults) + { + return diagnosticResults + .OrderBy(diagnosticResult => diagnosticResult.Spans.FirstOrDefault().Path, StringComparer.Ordinal) + .ThenBy(diagnosticResult => diagnosticResult.Spans.FirstOrDefault().Span.Start.Line) + .ThenBy(diagnosticResult => diagnosticResult.Spans.FirstOrDefault().Span.Start.Character) + .ThenBy(diagnosticResult => diagnosticResult.Spans.FirstOrDefault().Span.End.Line) + .ThenBy(diagnosticResult => diagnosticResult.Spans.FirstOrDefault().Span.End.Character) + .ThenBy(diagnosticResult => diagnosticResult.Id, StringComparer.Ordinal) + .ToArray(); + } + /// /// Gets the analyzers being tested. /// @@ -450,7 +553,7 @@ protected static async Task> GetSortedDiagnosticsFrom /// and . private static Diagnostic[] SortDistinctDiagnostics(IEnumerable diagnostics) { - return diagnostics.OrderBy(d => d.Location.SourceSpan.Start).ThenBy(d => d.Id).ToArray(); + return diagnostics.OrderBy(d => d.Location.SourceSpan.Start).ThenBy(d => d.Location.SourceSpan.End).ThenBy(d => d.Id).ToArray(); } /// @@ -732,7 +835,7 @@ private static void VerifyDiagnosticResults(IEnumerable actualResult Assert.True(false, message); } - if (actual.GetMessage() != expected.Message) + if (expected.Message != null && actual.GetMessage() != expected.Message) { string message = string.Format( From 8d4de6e87e8a1cbdd1167bfe84bdc093423b38c5 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Tue, 14 Aug 2018 12:06:12 -0500 Subject: [PATCH 8/9] Make CancellationToken optional in the public testing API --- samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs | 8 ++++---- samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs b/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs index 79c433fc1..82b4d7aa7 100644 --- a/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs +++ b/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs @@ -25,16 +25,16 @@ public static DiagnosticResult Diagnostic(DiagnosticDescriptor descriptor) public static DiagnosticResult CompilerError(string errorIdentifier) => DiagnosticVerifier.CompilerError(errorIdentifier); - public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult expected, CancellationToken cancellationToken) + public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult expected, CancellationToken cancellationToken = default) => DiagnosticVerifier.VerifyCSharpDiagnosticAsync(source, expected, cancellationToken); - public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult[] expected, CancellationToken cancellationToken) + public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult[] expected, CancellationToken cancellationToken = default) => DiagnosticVerifier.VerifyCSharpDiagnosticAsync(source, expected, cancellationToken); - public static Task VerifyCSharpFixAsync(string source, DiagnosticResult expected, string fixedSource, CancellationToken cancellationToken) + public static Task VerifyCSharpFixAsync(string source, DiagnosticResult expected, string fixedSource, CancellationToken cancellationToken = default) => VerifyCSharpFixAsync(source, new[] { expected }, fixedSource, cancellationToken); - public static Task VerifyCSharpFixAsync(string source, DiagnosticResult[] expected, string fixedSource, CancellationToken cancellationToken) + public static Task VerifyCSharpFixAsync(string source, DiagnosticResult[] expected, string fixedSource, CancellationToken cancellationToken = default) { CSharpTest test = new CSharpTest { diff --git a/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs b/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs index e4a37aa2e..c568ea725 100644 --- a/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs +++ b/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs @@ -40,10 +40,10 @@ public static DiagnosticResult CompilerError(string errorIdentifier) return new DiagnosticResult(errorIdentifier, DiagnosticSeverity.Error); } - public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult expected, CancellationToken cancellationToken) + public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult expected, CancellationToken cancellationToken = default) => VerifyCSharpDiagnosticAsync(source, new[] { expected }, cancellationToken); - public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult[] expected, CancellationToken cancellationToken) + public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult[] expected, CancellationToken cancellationToken = default) { CSharpTest test = new CSharpTest { From 497cb502a2690884c905cf73f58d82c75ce93933 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Wed, 15 Aug 2018 12:39:32 -0500 Subject: [PATCH 9/9] Add helpers for Visual Basic diagnostic and code fix testing --- .../UnitTestFramework/CodeFixVerifier`2.cs | 21 +++++++++++++++++++ .../UnitTestFramework/DiagnosticVerifier`1.cs | 14 +++++++++++++ 2 files changed, 35 insertions(+) diff --git a/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs b/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs index 82b4d7aa7..8033f6072 100644 --- a/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs +++ b/samples/Shared/UnitTestFramework/CodeFixVerifier`2.cs @@ -46,6 +46,27 @@ public static Task VerifyCSharpFixAsync(string source, DiagnosticResult[] expect return test.RunAsync(cancellationToken); } + public static Task VerifyVisualBasicDiagnosticAsync(string source, DiagnosticResult expected, CancellationToken cancellationToken = default) + => DiagnosticVerifier.VerifyVisualBasicDiagnosticAsync(source, expected, cancellationToken); + + public static Task VerifyVisualBasicDiagnosticAsync(string source, DiagnosticResult[] expected, CancellationToken cancellationToken = default) + => DiagnosticVerifier.VerifyVisualBasicDiagnosticAsync(source, expected, cancellationToken); + + public static Task VerifyVisualBasicFixAsync(string source, DiagnosticResult expected, string fixedSource, CancellationToken cancellationToken = default) + => VerifyVisualBasicFixAsync(source, new[] { expected }, fixedSource, cancellationToken); + + public static Task VerifyVisualBasicFixAsync(string source, DiagnosticResult[] expected, string fixedSource, CancellationToken cancellationToken = default) + { + VisualBasicTest test = new VisualBasicTest + { + TestCode = source, + FixedCode = fixedSource, + }; + + test.ExpectedDiagnostics.AddRange(expected); + return test.RunAsync(cancellationToken); + } + public class CSharpTest : DiagnosticVerifier.CSharpTest { protected override IEnumerable GetCodeFixProviders() diff --git a/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs b/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs index c568ea725..513e99de4 100644 --- a/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs +++ b/samples/Shared/UnitTestFramework/DiagnosticVerifier`1.cs @@ -54,6 +54,20 @@ public static Task VerifyCSharpDiagnosticAsync(string source, DiagnosticResult[] return test.RunAsync(cancellationToken); } + public static Task VerifyVisualBasicDiagnosticAsync(string source, DiagnosticResult expected, CancellationToken cancellationToken = default) + => VerifyVisualBasicDiagnosticAsync(source, new[] { expected }, cancellationToken); + + public static Task VerifyVisualBasicDiagnosticAsync(string source, DiagnosticResult[] expected, CancellationToken cancellationToken = default) + { + VisualBasicTest test = new VisualBasicTest + { + TestCode = source, + }; + + test.ExpectedDiagnostics.AddRange(expected); + return test.RunAsync(cancellationToken); + } + public class CSharpTest : GenericAnalyzerTest { public override string Language => LanguageNames.CSharp;