Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add unit test for 3rd party fixer formatting #896

Merged
merged 7 commits into from
Dec 11, 2020
Merged

Add unit test for 3rd party fixer formatting #896

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
eec63a8
Add a lock around loading a MSBuild workspace
JoeRobich Dec 11, 2020
cf045dc
Extract out a nuget restore helper
JoeRobich Dec 11, 2020
bc9060a
Add a project to store analyzer package references
JoeRobich Dec 11, 2020
b78dcd9
Add ability to add AnalyzerReferences during test formatting
JoeRobich Dec 11, 2020
1fb7f29
Add a 3rd party analyzer test class
JoeRobich Dec 11, 2020
e3d2ce1
Enhance other tests to log the formatter output on failure
JoeRobich Dec 11, 2020
a5387a9
Fix file encoding
JoeRobich Dec 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 32 additions & 20 deletions src/Workspaces/MSBuildWorkspaceLoader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ namespace Microsoft.CodeAnalysis.Tools.Workspaces
{
internal static class MSBuildWorkspaceLoader
{
private static readonly SemaphoreSlim s_guard = new SemaphoreSlim(1, 1);

public static async Task<Workspace?> LoadAsync(
string solutionOrProjectPath,
WorkspaceType workspaceType,
Expand All @@ -29,35 +31,45 @@ internal static class MSBuildWorkspaceLoader
{ "AlwaysCompileMarkupFilesInSeparateDomain", bool.FalseString },
};

var workspace = MSBuildWorkspace.Create(properties);
MSBuildWorkspace workspace;

Build.Framework.ILogger? binlog = null;
if (createBinaryLog)
await s_guard.WaitAsync();
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is needed because tests run in parallel but msbuild won't

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still don't quite understand this. Why can't each test create their own workspace?

Copy link
Member Author

@JoeRobich JoeRobich Dec 11, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the CodeFormatterTests is creating MSBuildWorkspaces and now we are creating a MSBuildWorkspace here, there is an InvalidOperationException "Cannot start a new build while a build is in progress", since they are running parallel.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we create a WorspaceFact that controls this intead of changing product code?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Took a stab at this in #899

try
{
binlog = new Build.Logging.BinaryLogger()
workspace = MSBuildWorkspace.Create(properties);

Build.Framework.ILogger? binlog = null;
if (createBinaryLog)
{
Parameters = Path.Combine(Environment.CurrentDirectory, "formatDiagnosticLog.binlog"),
Verbosity = Build.Framework.LoggerVerbosity.Diagnostic,
};
}
binlog = new Build.Logging.BinaryLogger()
{
Parameters = Path.Combine(Environment.CurrentDirectory, "formatDiagnosticLog.binlog"),
Verbosity = Build.Framework.LoggerVerbosity.Diagnostic,
};
}

if (workspaceType == WorkspaceType.Solution)
{
await workspace.OpenSolutionAsync(solutionOrProjectPath, msbuildLogger: binlog, cancellationToken: cancellationToken).ConfigureAwait(false);
}
else
{
try
if (workspaceType == WorkspaceType.Solution)
{
await workspace.OpenProjectAsync(solutionOrProjectPath, msbuildLogger: binlog, cancellationToken: cancellationToken).ConfigureAwait(false);
await workspace.OpenSolutionAsync(solutionOrProjectPath, msbuildLogger: binlog, cancellationToken: cancellationToken).ConfigureAwait(false);
}
catch (InvalidOperationException)
else
{
logger.LogError(Resources.Could_not_format_0_Format_currently_supports_only_CSharp_and_Visual_Basic_projects, solutionOrProjectPath);
workspace.Dispose();
return null;
try
{
await workspace.OpenProjectAsync(solutionOrProjectPath, msbuildLogger: binlog, cancellationToken: cancellationToken).ConfigureAwait(false);
}
catch (InvalidOperationException)
{
logger.LogError(Resources.Could_not_format_0_Format_currently_supports_only_CSharp_and_Visual_Basic_projects, solutionOrProjectPath);
workspace.Dispose();
return null;
}
}
}
finally
{
s_guard.Release();
}

LogWorkspaceDiagnostics(logger, logWorkspaceWarnings, workspace.Diagnostics);

Expand Down
6 changes: 6 additions & 0 deletions tests/Analyzers/CodeStyleAnalyzerFormatterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,19 @@
using Microsoft.CodeAnalysis.Tools.Formatters;
using Microsoft.CodeAnalysis.Tools.Tests.Formatters;
using Xunit;
using Xunit.Abstractions;

namespace Microsoft.CodeAnalysis.Tools.Tests.Analyzers
{
public class CodeStyleAnalyzerFormatterTests : CSharpFormatterTests
{
private protected override ICodeFormatter Formatter => AnalyzerFormatter.CodeStyleFormatter;

public CodeStyleAnalyzerFormatterTests(ITestOutputHelper output)
{
TestOutputHelper = output;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On failure of any tests in the class, the formatter log will be written to the test output.

}

[Fact]
public async Task TestUseVarCodeStyle_AppliesWhenNotUsingVar()
{
Expand Down
122 changes: 122 additions & 0 deletions tests/Analyzers/ThirdPartyAnalyzerFormatterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
// Copyright (c) Microsoft. All Rights Reserved. Licensed under the MIT license. See License.txt in the project root for license information.

using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.Diagnostics;
using Microsoft.CodeAnalysis.Tools.Analyzers;
using Microsoft.CodeAnalysis.Tools.Formatters;
using Microsoft.CodeAnalysis.Tools.Tests.Formatters;
using Microsoft.CodeAnalysis.Tools.Tests.Utilities;
using Microsoft.CodeAnalysis.Tools.Workspaces;
using Xunit;
using Xunit.Abstractions;

namespace Microsoft.CodeAnalysis.Tools.Tests.Analyzers
{
public class ThirdPartyAnalyzerFormatterTests : CSharpFormatterTests, IAsyncLifetime
{
private static readonly string s_analyzerProjectFilePath = Path.Combine("for_analyzer_formatter", "analyzer_project", "analyzer_project.csproj");

private protected override ICodeFormatter Formatter => AnalyzerFormatter.ThirdPartyFormatter;

private Project _analyzerReferencesProject;

public ThirdPartyAnalyzerFormatterTests(ITestOutputHelper output)
{
TestOutputHelper = output;
}

public async Task InitializeAsync()
{
var logger = new TestLogger();

try
{
// Restore the Analyzer packages that have been added to `for_analyzer_formatter/analyzer_project/analyzer_project.csproj`
var exitCode = await NuGetHelper.PerformRestore(s_analyzerProjectFilePath, TestOutputHelper);
Assert.Equal(0, exitCode);

// Load the analyzer_project into a MSBuildWorkspace.
var workspacePath = Path.Combine(TestProjectsPathHelper.GetProjectsDirectory(), s_analyzerProjectFilePath);

MSBuildRegistrar.RegisterInstance(logger);
var analyzerWorkspace = await MSBuildWorkspaceLoader.LoadAsync(workspacePath, WorkspaceType.Project, createBinaryLog: false, logWorkspaceWarnings: true, logger, CancellationToken.None);

// From this project we can get valid AnalyzerReferences to add to our test project.
_analyzerReferencesProject = analyzerWorkspace.CurrentSolution.Projects.Single();
}
catch
{
TestOutputHelper.WriteLine(logger.GetLog());
throw;
}
}

public Task DisposeAsync()
{
_analyzerReferencesProject = null;

return Task.CompletedTask;
}

private IEnumerable<AnalyzerReference> GetAnalyzerReferences(string prefix)
=> _analyzerReferencesProject.AnalyzerReferences.Where(reference => reference.Display.StartsWith(prefix));

[Fact]
public async Task TestStyleCopBlankLineFixer_RemovesUnnecessaryBlankLines()
{
var analyzerReferences = GetAnalyzerReferences("StyleCop");

var testCode = @"
class C
{

void M()

{

object obj = new object();


int count = 5;

}

}
";

var expectedCode = @"
class C
{
void M()
{
object obj = new object();

int count = 5;
}
}
";

var editorConfig = new Dictionary<string, string>()
{
// Turn off all diagnostics analyzers
["dotnet_analyzer_diagnostic.severity"] = "none",

// Two or more consecutive blank lines: Remove down to one blank line. SA1507
["dotnet_diagnostic.SA1507.severity"] = "error",

// Blank line immediately before or after a { line: remove it. SA1505, SA1509
["dotnet_diagnostic.SA1505.severity"] = "error",
["dotnet_diagnostic.SA1509.severity"] = "error",

// Blank line immediately before a } line: remove it. SA1508
["dotnet_diagnostic.SA1508.severity"] = "error",
};

await AssertCodeChangedAsync(testCode, expectedCode, editorConfig, fixCategory: FixCategory.Analyzers, analyzerReferences: analyzerReferences);
}
}
}
37 changes: 16 additions & 21 deletions tests/CodeFormatterTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ await TestFormatWorkspaceAsync(
[Fact]
public async Task NoFilesFormattedInCodeStyleSolution_WhenNotFixingCodeStyle()
{
var restoreExitCode = await PerformNuGetRestore(s_codeStyleSolutionFilePath);
var restoreExitCode = await NuGetHelper.PerformRestore(s_codeStyleSolutionFilePath, _output);
Assert.Equal(0, restoreExitCode);

await TestFormatWorkspaceAsync(
Expand All @@ -429,7 +429,7 @@ await TestFormatWorkspaceAsync(
[Fact]
public async Task NoFilesFormattedInCodeStyleSolution_WhenFixingCodeStyleErrors()
{
var restoreExitCode = await PerformNuGetRestore(s_codeStyleSolutionFilePath);
var restoreExitCode = await NuGetHelper.PerformRestore(s_codeStyleSolutionFilePath, _output);
Assert.Equal(0, restoreExitCode);

await TestFormatWorkspaceAsync(
Expand All @@ -447,7 +447,7 @@ await TestFormatWorkspaceAsync(
[Fact]
public async Task FilesFormattedInCodeStyleSolution_WhenFixingCodeStyleWarnings()
{
var restoreExitCode = await PerformNuGetRestore(s_codeStyleSolutionFilePath);
var restoreExitCode = await NuGetHelper.PerformRestore(s_codeStyleSolutionFilePath, _output);
Assert.Equal(0, restoreExitCode);

await TestFormatWorkspaceAsync(
Expand All @@ -465,7 +465,7 @@ await TestFormatWorkspaceAsync(
[Fact]
public async Task NoFilesFormattedInAnalyzersSolution_WhenNotFixingAnalyzers()
{
var restoreExitCode = await PerformNuGetRestore(s_analyzersSolutionFilePath);
var restoreExitCode = await NuGetHelper.PerformRestore(s_analyzersSolutionFilePath, _output);
Assert.Equal(0, restoreExitCode);

await TestFormatWorkspaceAsync(
Expand All @@ -482,7 +482,7 @@ await TestFormatWorkspaceAsync(
[Fact]
public async Task FilesFormattedInAnalyzersSolution_WhenFixingAnalyzerErrors()
{
var restoreExitCode = await PerformNuGetRestore(s_analyzersSolutionFilePath);
var restoreExitCode = await NuGetHelper.PerformRestore(s_analyzersSolutionFilePath, _output);
Assert.Equal(0, restoreExitCode);

await TestFormatWorkspaceAsync(
Expand All @@ -497,18 +497,6 @@ await TestFormatWorkspaceAsync(
analyzerSeverity: DiagnosticSeverity.Error);
}

internal async Task<int> PerformNuGetRestore(string workspaceFilePath)
{
var workspacePath = Path.Combine(TestProjectsPathHelper.GetProjectsDirectory(), workspaceFilePath);

var processInfo = ProcessRunner.CreateProcess("dotnet", $"restore \"{workspacePath}\"", captureOutput: true, displayWindow: false);
var restoreResult = await processInfo.Result;

_output.WriteLine(string.Join(Environment.NewLine, restoreResult.OutputLines));

return restoreResult.ExitCode;
}

internal async Task<string> TestFormatWorkspaceAsync(
string workspaceFilePath,
string[] include,
Expand Down Expand Up @@ -561,11 +549,18 @@ internal async Task<string> TestFormatWorkspaceAsync(
Environment.CurrentDirectory = currentDirectory;

var log = logger.GetLog();
_output.WriteLine(log);

Assert.Equal(expectedExitCode, formatResult.ExitCode);
Assert.Equal(expectedFilesFormatted, formatResult.FilesFormatted);
Assert.Equal(expectedFileCount, formatResult.FileCount);
try
{
Assert.Equal(expectedExitCode, formatResult.ExitCode);
Assert.Equal(expectedFilesFormatted, formatResult.FilesFormatted);
Assert.Equal(expectedFileCount, formatResult.FileCount);
}
catch
{
_output.WriteLine(log);
throw;
}

return log;
}
Expand Down
Loading