Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add generator driver cache to VBCSCompiler #55023

Merged
merged 7 commits into from
Aug 31, 2021
Merged

Add generator driver cache to VBCSCompiler #55023

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0f3dd68
Add generator drive cache to VBCSCompiler:
chsienki Jul 21, 2021
0e1ad7e
Update cache to invalidate when generators change
chsienki Jul 26, 2021
d400669
Disable the cache when there isn't an output file name
chsienki Jul 27, 2021
04a1bc9
Formatting
chsienki Jul 27, 2021
481391b
PR Feedback
chsienki Aug 13, 2021
33670e9
Add comment about correctness
chsienki Aug 30, 2021
0aafa26
Update src/Compilers/Core/Portable/CommandLine/CommonCompiler.cs
chsienki Aug 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 4 additions & 7 deletions src/Compilers/CSharp/Portable/CommandLine/CSharpCompiler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ internal abstract class CSharpCompiler : CommonCompiler
private readonly CommandLineDiagnosticFormatter _diagnosticFormatter;
private readonly string? _tempDirectory;

protected CSharpCompiler(CSharpCommandLineParser parser, string? responseFile, string[] args, BuildPaths buildPaths, string? additionalReferenceDirectories, IAnalyzerAssemblyLoader assemblyLoader)
: base(parser, responseFile, args, buildPaths, additionalReferenceDirectories, assemblyLoader)
protected CSharpCompiler(CSharpCommandLineParser parser, string? responseFile, string[] args, BuildPaths buildPaths, string? additionalReferenceDirectories, IAnalyzerAssemblyLoader assemblyLoader, GeneratorDriverCache? driverCache = null)
: base(parser, responseFile, args, buildPaths, additionalReferenceDirectories, assemblyLoader, driverCache)
{
_diagnosticFormatter = new CommandLineDiagnosticFormatter(buildPaths.WorkingDirectory, Arguments.PrintFullPaths, Arguments.ShouldIncludeErrorEndLocation);
_tempDirectory = buildPaths.TempDirectory;
Expand Down Expand Up @@ -372,12 +372,9 @@ protected override void ResolveEmbeddedFilesFromExternalSourceDirectives(
}
}

private protected override Compilation RunGenerators(Compilation input, ParseOptions parseOptions, ImmutableArray<ISourceGenerator> generators, AnalyzerConfigOptionsProvider analyzerConfigProvider, ImmutableArray<AdditionalText> additionalTexts, DiagnosticBag diagnostics)
private protected override GeneratorDriver CreateGeneratorDriver(ParseOptions parseOptions, ImmutableArray<ISourceGenerator> generators, AnalyzerConfigOptionsProvider analyzerConfigOptionsProvider, ImmutableArray<AdditionalText> additionalTexts)
{
var driver = CSharpGeneratorDriver.Create(generators, additionalTexts, (CSharpParseOptions)parseOptions, analyzerConfigProvider);
driver.RunGeneratorsAndUpdateCompilation(input, out var compilationOut, out var generatorDiagnostics);
diagnostics.AddRange(generatorDiagnostics);
return compilationOut;
return CSharpGeneratorDriver.Create(generators, additionalTexts, (CSharpParseOptions)parseOptions, analyzerConfigOptionsProvider);
}
}
}
8 changes: 4 additions & 4 deletions src/Compilers/CSharp/Test/CommandLine/CommandLineTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,15 @@ internal CSharpCommandLineArguments DefaultParse(IEnumerable<string> args, strin
return CSharpCommandLineParser.Default.Parse(args, baseDirectory, sdkDirectory, additionalReferenceDirectories);
}

internal MockCSharpCompiler CreateCSharpCompiler(string[] args, ImmutableArray<DiagnosticAnalyzer> analyzers = default, ImmutableArray<ISourceGenerator> generators = default, AnalyzerAssemblyLoader loader = null)
internal MockCSharpCompiler CreateCSharpCompiler(string[] args, ImmutableArray<DiagnosticAnalyzer> analyzers = default, ImmutableArray<ISourceGenerator> generators = default, AnalyzerAssemblyLoader loader = null, GeneratorDriverCache driverCache = null)
{
return CreateCSharpCompiler(null, WorkingDirectory, args, analyzers, generators, loader);
return CreateCSharpCompiler(null, WorkingDirectory, args, analyzers, generators, loader, driverCache);
}

internal MockCSharpCompiler CreateCSharpCompiler(string responseFile, string workingDirectory, string[] args, ImmutableArray<DiagnosticAnalyzer> analyzers = default, ImmutableArray<ISourceGenerator> generators = default, AnalyzerAssemblyLoader loader = null)
internal MockCSharpCompiler CreateCSharpCompiler(string responseFile, string workingDirectory, string[] args, ImmutableArray<DiagnosticAnalyzer> analyzers = default, ImmutableArray<ISourceGenerator> generators = default, AnalyzerAssemblyLoader loader = null, GeneratorDriverCache driverCache = null)
{
var buildPaths = RuntimeUtilities.CreateBuildPaths(workingDirectory, sdkDirectory: SdkDirectory);
return new MockCSharpCompiler(responseFile, buildPaths, args, analyzers, generators, loader);
return new MockCSharpCompiler(responseFile, buildPaths, args, analyzers, generators, loader, driverCache);
}
}
}
232 changes: 231 additions & 1 deletion src/Compilers/CSharp/Test/CommandLine/CommandLineTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9793,6 +9793,235 @@ string[] compileAndRun(string featureOpt)
};
}

[Fact]
public void Compiler_Uses_DriverCache()
{
var dir = Temp.CreateDirectory();
var src = dir.CreateFile("temp.cs").WriteAllText(@"
class C
{
}");
int sourceCallbackCount = 0;
var generator = new PipelineCallbackGenerator((ctx) =>
{
ctx.RegisterSourceOutput(ctx.ParseOptionsProvider, (spc, po) =>
{
sourceCallbackCount++;
});
});

// with no cache, we'll see the callback execute multiple times
RunWithNoCache();
Assert.Equal(1, sourceCallbackCount);

RunWithNoCache();
Assert.Equal(2, sourceCallbackCount);

RunWithNoCache();
Assert.Equal(3, sourceCallbackCount);

// now re-run with a cache
GeneratorDriverCache cache = new GeneratorDriverCache();
sourceCallbackCount = 0;

RunWithCache();
Assert.Equal(1, sourceCallbackCount);

RunWithCache();
Assert.Equal(1, sourceCallbackCount);

RunWithCache();
Assert.Equal(1, sourceCallbackCount);

// Clean up temp files
CleanupAllGeneratedFiles(src.Path);
Directory.Delete(dir.Path, true);

void RunWithNoCache() => VerifyOutput(dir, src, includeCurrentAssemblyAsAnalyzerReference: false, additionalFlags: new[] { "/langversion:preview" }, generators: new[] { generator.AsSourceGenerator() }, analyzers: null);

void RunWithCache() => VerifyOutput(dir, src, includeCurrentAssemblyAsAnalyzerReference: false, additionalFlags: new[] { "/langversion:preview" }, generators: new[] { generator.AsSourceGenerator() }, driverCache: cache, analyzers: null);
}

[Fact]
public void Compiler_Doesnt_Use_Cache_From_Other_Compilation()
{
var dir = Temp.CreateDirectory();
var src = dir.CreateFile("temp.cs").WriteAllText(@"
class C
{
}");
int sourceCallbackCount = 0;
var generator = new PipelineCallbackGenerator((ctx) =>
{
ctx.RegisterSourceOutput(ctx.ParseOptionsProvider, (spc, po) =>
{
sourceCallbackCount++;
});
});

// now re-run with a cache
GeneratorDriverCache cache = new GeneratorDriverCache();
sourceCallbackCount = 0;

RunWithCache("1.dll");
Assert.Equal(1, sourceCallbackCount);

RunWithCache("1.dll");
Assert.Equal(1, sourceCallbackCount);

// now emulate a new compilation, and check we were invoked, but only once
RunWithCache("2.dll");
Assert.Equal(2, sourceCallbackCount);

RunWithCache("2.dll");
Assert.Equal(2, sourceCallbackCount);

// now re-run our first compilation
RunWithCache("1.dll");
Assert.Equal(2, sourceCallbackCount);

// a new one
RunWithCache("3.dll");
Assert.Equal(3, sourceCallbackCount);

// and another old one
RunWithCache("2.dll");
Assert.Equal(3, sourceCallbackCount);

RunWithCache("1.dll");
Assert.Equal(3, sourceCallbackCount);

// Clean up temp files
CleanupAllGeneratedFiles(src.Path);
Directory.Delete(dir.Path, true);

void RunWithCache(string outputPath) => VerifyOutput(dir, src, includeCurrentAssemblyAsAnalyzerReference: false, additionalFlags: new[] { "/langversion:preview", "/out:" + outputPath }, generators: new[] { generator.AsSourceGenerator() }, driverCache: cache, analyzers: null);
}

[Fact]
public void Compiler_Can_Disable_DriverCache()
{
var dir = Temp.CreateDirectory();
var src = dir.CreateFile("temp.cs").WriteAllText(@"
class C
{
}");
int sourceCallbackCount = 0;
var generator = new PipelineCallbackGenerator((ctx) =>
{
ctx.RegisterSourceOutput(ctx.ParseOptionsProvider, (spc, po) =>
{
sourceCallbackCount++;
});
});

// run with the cache
GeneratorDriverCache cache = new GeneratorDriverCache();
sourceCallbackCount = 0;

RunWithCache();
Assert.Equal(1, sourceCallbackCount);

RunWithCache();
Assert.Equal(1, sourceCallbackCount);

RunWithCache();
Assert.Equal(1, sourceCallbackCount);

// now re-run with the cache disabled
sourceCallbackCount = 0;

RunWithCacheDisabled();
Assert.Equal(1, sourceCallbackCount);

RunWithCacheDisabled();
Assert.Equal(2, sourceCallbackCount);

RunWithCacheDisabled();
Assert.Equal(3, sourceCallbackCount);

// now clear the cache as well as disabling, and verify we don't put any entries into it either
cache = new GeneratorDriverCache();
sourceCallbackCount = 0;

RunWithCacheDisabled();
Assert.Equal(1, sourceCallbackCount);
Assert.Equal(0, cache.CacheSize);

RunWithCacheDisabled();
Assert.Equal(2, sourceCallbackCount);
Assert.Equal(0, cache.CacheSize);

RunWithCacheDisabled();
Assert.Equal(3, sourceCallbackCount);
Assert.Equal(0, cache.CacheSize);

// Clean up temp files
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

//

Consider clearing the cache and testing RunWithCacheDisabled(); then checking that the cache is still empty. (It wasn't clear from the calls above whether RunWithCacheDisabled() updates the cache even though it ignores the cache for getting a generator.)

CleanupAllGeneratedFiles(src.Path);
Directory.Delete(dir.Path, true);

void RunWithCache() => VerifyOutput(dir, src, includeCurrentAssemblyAsAnalyzerReference: false, additionalFlags: new[] { "/langversion:preview" }, generators: new[] { generator.AsSourceGenerator() }, driverCache: cache, analyzers: null);

void RunWithCacheDisabled() => VerifyOutput(dir, src, includeCurrentAssemblyAsAnalyzerReference: false, additionalFlags: new[] { "/langversion:preview", "/features:disable-generator-cache" }, generators: new[] { generator.AsSourceGenerator() }, driverCache: cache, analyzers: null);
}

[Fact]
public void Adding_Or_Removing_A_Generator_Invalidates_Cache()
{
var dir = Temp.CreateDirectory();
var src = dir.CreateFile("temp.cs").WriteAllText(@"
class C
{
}");
int sourceCallbackCount = 0;
int sourceCallbackCount2 = 0;
var generator = new PipelineCallbackGenerator((ctx) =>
{
ctx.RegisterSourceOutput(ctx.ParseOptionsProvider, (spc, po) =>
{
sourceCallbackCount++;
});
});

var generator2 = new PipelineCallbackGenerator2((ctx) =>
{
ctx.RegisterSourceOutput(ctx.ParseOptionsProvider, (spc, po) =>
{
sourceCallbackCount2++;
});
});

// run with the cache
GeneratorDriverCache cache = new GeneratorDriverCache();

RunWithOneGenerator();
Assert.Equal(1, sourceCallbackCount);
Assert.Equal(0, sourceCallbackCount2);

RunWithOneGenerator();
Assert.Equal(1, sourceCallbackCount);
Assert.Equal(0, sourceCallbackCount2);

RunWithTwoGenerators();
Assert.Equal(2, sourceCallbackCount);
Assert.Equal(1, sourceCallbackCount2);

RunWithTwoGenerators();
Assert.Equal(2, sourceCallbackCount);
Assert.Equal(1, sourceCallbackCount2);

// this seems counterintuitive, but when the only thing to change is the generator, we end up back at the state of the project when
// we just ran a single generator. Thus we already have an entry in the cache we can use (the one created by the original call to
// RunWithOneGenerator above) meaning we can use the previously cached results and not run.
RunWithOneGenerator();
Assert.Equal(2, sourceCallbackCount);
Assert.Equal(1, sourceCallbackCount2);

void RunWithOneGenerator() => VerifyOutput(dir, src, includeCurrentAssemblyAsAnalyzerReference: false, additionalFlags: new[] { "/langversion:preview" }, generators: new[] { generator.AsSourceGenerator() }, driverCache: cache, analyzers: null);

void RunWithTwoGenerators() => VerifyOutput(dir, src, includeCurrentAssemblyAsAnalyzerReference: false, additionalFlags: new[] { "/langversion:preview" }, generators: new[] { generator.AsSourceGenerator(), generator2.AsSourceGenerator() }, driverCache: cache, analyzers: null);
}

private static int OccurrenceCount(string source, string word)
{
var n = 0;
Expand All @@ -9814,6 +10043,7 @@ private string VerifyOutput(TempDirectory sourceDir, TempFile sourceFile,
int? expectedExitCode = null,
bool errorlog = false,
IEnumerable<ISourceGenerator> generators = null,
GeneratorDriverCache driverCache = null,
params DiagnosticAnalyzer[] analyzers)
{
var args = new[] {
Expand All @@ -9835,7 +10065,7 @@ private string VerifyOutput(TempDirectory sourceDir, TempFile sourceFile,
args = args.Append(additionalFlags);
}

var csc = CreateCSharpCompiler(null, sourceDir.Path, args, analyzers: analyzers.ToImmutableArrayOrEmpty(), generators: generators.ToImmutableArrayOrEmpty());
var csc = CreateCSharpCompiler(null, sourceDir.Path, args, analyzers: analyzers.ToImmutableArrayOrEmpty(), generators: generators.ToImmutableArrayOrEmpty(), driverCache: driverCache);
var outWriter = new StringWriter(CultureInfo.InvariantCulture);
var exitCode = csc.Run(outWriter);
var output = outWriter.ToString();
Expand Down
Loading