Skip to content

Commit

Permalink
Allow the incremental compiler to load modules
Browse files Browse the repository at this point in the history 
Change-Id: I63bbbdfc94ac6ab0b7b8cce104fe38c4ba592f34
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/95382
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Kevin Millikin <kmillikin@google.com>
  • Loading branch information
@jensjoha
jensjoha authored and commit-bot@chromium.org committed Mar 28, 2019
1 parent ef168de commit 83e6e9f
Show file tree
Hide file tree
Showing 4 changed files with 386 additions and 8 deletions.
2 changes: 1 addition & 1 deletion pkg/front_end/lib/src/fasta/dill/dill_library_builder.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ import 'dill_type_alias_builder.dart' show DillTypeAliasBuilder;
class DillLibraryBuilder extends LibraryBuilder<KernelTypeBuilder, Library> {
final Library library;

final DillLoader loader;
DillLoader loader;

/// Exports that can't be serialized.
///
Expand Down
76 changes: 72 additions & 4 deletions pkg/front_end/lib/src/fasta/incremental_compiler.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,8 @@ class IncrementalCompiler implements IncrementalKernelGenerator {

Set<Uri> invalidatedUris = this.invalidatedUris.toSet();

invalidateNotKeptUserBuilders(invalidatedUris);

ClassHierarchy hierarchy = userCode?.loader?.hierarchy;
Set<LibraryBuilder> notReusedLibraries = new Set<LibraryBuilder>();
List<LibraryBuilder> reusedLibraries = computeReusedLibraries(
Expand All @@ -225,6 +227,22 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
CompilerContext.current.uriToSource.remove(builder.fileUri);
}

if (hasToCheckPackageUris) {
// The package file was changed.
// Make sure the dill loader is on the same page.
DillTarget oldDillLoadedData = dillLoadedData;
dillLoadedData =
new DillTarget(ticker, uriTranslator, c.options.target);
for (DillLibraryBuilder library
in oldDillLoadedData.loader.builders.values) {
library.loader = dillLoadedData.loader;
dillLoadedData.loader.builders[library.uri] = library;
if (library.uri.scheme == "dart" && library.uri.path == "core") {
dillLoadedData.loader.coreLibrary = library;
}
}
}

for (LibraryBuilder builder in notReusedLibraries) {
Library lib = builder.target;
CompilerContext.current.uriToSource.remove(builder.fileUri);
Expand All @@ -250,6 +268,8 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
" of ${userCode.loader.builders.length} libraries");
}

await loadEnsureLoadedComponents(reusedLibraryUris, reusedLibraries);

KernelTarget userCodeOld = userCode;
userCode = new KernelTarget(
new HybridFileSystem(
Expand Down Expand Up @@ -342,13 +362,57 @@ class IncrementalCompiler implements IncrementalKernelGenerator {

/// Internal method.
void invalidateNotKeptUserBuilders(Set<Uri> invalidatedUris) {
throw UnimplementedError("Not implemented yet.");
if (modulesToLoad != null && userBuilders != null) {
Set<Library> loadedNotKept = new Set<Library>();
for (LibraryBuilder builder in userBuilders.values) {
loadedNotKept.add(builder.target);
}
for (Component module in modulesToLoad) {
loadedNotKept.removeAll(module.libraries);
}
for (Library lib in loadedNotKept) {
invalidatedUris.add(lib.importUri);
}
}
}

/// Internal method.
Future loadEnsureLoadedComponents(
Set<Uri> reusedLibraryUris, List<LibraryBuilder> reusedLibraries) async {
throw UnimplementedError("Not implemented yet.");
if (modulesToLoad != null) {
bool loadedAnything = false;
for (Component module in modulesToLoad) {
bool usedComponent = false;
for (Library lib in module.libraries) {
if (!reusedLibraryUris.contains(lib.importUri)) {
dillLoadedData.loader.libraries.add(lib);
dillLoadedData.addLibrary(lib);
reusedLibraries.add(dillLoadedData.loader.read(lib.importUri, -1));
usedComponent = true;
}
}
if (usedComponent) {
dillLoadedData.uriToSource.addAll(module.uriToSource);
loadedAnything = true;
}
}
if (loadedAnything) {
await dillLoadedData.buildOutlines();
userBuilders = <Uri, LibraryBuilder>{};
platformBuilders = <LibraryBuilder>[];
dillLoadedData.loader.builders.forEach((uri, builder) {
if (builder.uri.scheme == "dart") {
platformBuilders.add(builder);
} else {
userBuilders[uri] = builder;
}
});
if (userBuilders.isEmpty) {
userBuilders = null;
}
}
modulesToLoad = null;
}
}

/// Internal method.
Expand Down Expand Up @@ -849,12 +913,16 @@ class IncrementalCompiler implements IncrementalKernelGenerator {

@override
void invalidateAllSources() {
throw UnimplementedError("Not implemented yet.");
if (userCode != null) {
Set<Uri> uris = new Set<Uri>.from(userCode.loader.builders.keys);
uris.removeAll(dillLoadedData.loader.builders.keys);
invalidatedUris.addAll(uris);
}
}

@override
void setModulesToLoadOnNextComputeDelta(List<Component> components) {
throw UnimplementedError("Not implemented yet.");
modulesToLoad = components.toList();
}

/// Internal method.
Expand Down
169 changes: 166 additions & 3 deletions pkg/front_end/test/incremental_load_from_dill_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@ import 'package:front_end/src/base/processed_options.dart'
import 'package:front_end/src/fasta/compiler_context.dart' show CompilerContext;

import 'package:front_end/src/api_prototype/compiler_options.dart'
show CompilerOptions, DiagnosticMessage;
show CompilerOptions;

import 'package:front_end/src/api_prototype/diagnostic_message.dart'
show DiagnosticMessage, getMessageCodeObject;

import "package:front_end/src/api_prototype/memory_file_system.dart"
show MemoryFileSystem;
Expand All @@ -27,7 +30,10 @@ import 'package:front_end/src/fasta/incremental_compiler.dart'

import 'package:front_end/src/fasta/severity.dart' show Severity;

import 'package:kernel/kernel.dart' show Component, Library;
import 'package:kernel/binary/ast_from_binary.dart' show BinaryBuilder;

import 'package:kernel/kernel.dart'
show Class, Component, Field, Library, Procedure;

import 'package:kernel/target/targets.dart' show TargetFlags;

Expand Down Expand Up @@ -111,6 +117,7 @@ class RunCompilations extends Step<TestData, TestData, Context> {
await newWorldTest(
map["strong"],
map["worlds"],
map["modules"],
map["omitPlatform"],
);
break;
Expand Down Expand Up @@ -180,7 +187,74 @@ Future<Null> basicTest(YamlMap sourceFiles, String entryPoint, bool strong,
checkIsEqual(normalDillData, initializedDillData);
}

Future<Null> newWorldTest(bool strong, List worlds, bool omitPlatform) async {
Future<Map<String, List<int>>> createModules(
Map module, final List<int> sdkSummaryData, bool strong) async {
final Uri base = Uri.parse("org-dartlang-test:///");
final Uri sdkSummary = base.resolve("vm_platform.dill");

MemoryFileSystem fs = new MemoryFileSystem(base);
fs.entityForUri(sdkSummary).writeAsBytesSync(sdkSummaryData);

// Setup all sources
for (Map moduleSources in module.values) {
for (String filename in moduleSources.keys) {
String data = moduleSources[filename];
Uri uri = base.resolve(filename);
if (await fs.entityForUri(uri).exists())
throw "More than one entry for $filename";
fs.entityForUri(uri).writeAsStringSync(data);
}
}

Map<String, List<int>> moduleResult = new Map<String, List<int>>();

for (String moduleName in module.keys) {
List<Uri> moduleSources = new List<Uri>();
Uri packagesUri;
for (String filename in module[moduleName].keys) {
Uri uri = base.resolve(filename);
if (uri.pathSegments.last == ".packages") {
packagesUri = uri;
} else {
moduleSources.add(uri);
}
}
CompilerOptions options = getOptions(strong);
options.fileSystem = fs;
options.sdkRoot = null;
options.sdkSummary = sdkSummary;
options.omitPlatform = true;
options.onDiagnostic = (DiagnosticMessage message) {
if (getMessageCodeObject(message)?.name == "InferredPackageUri") return;
throw message.ansiFormatted;
};
if (packagesUri != null) {
options.packagesFileUri = packagesUri;
}
TestIncrementalCompiler compiler =
new TestIncrementalCompiler(options, moduleSources.first, null);
Component c = await compiler.computeDelta(entryPoints: moduleSources);
c.computeCanonicalNames();
List<Library> wantedLibs = new List<Library>();
for (Library lib in c.libraries) {
if (moduleSources.contains(lib.importUri) ||
moduleSources.contains(lib.fileUri)) {
wantedLibs.add(lib);
}
}
if (wantedLibs.length != moduleSources.length) {
throw "Module probably not setup right.";
}
Component result = new Component(libraries: wantedLibs);
List<int> resultBytes = util.postProcess(result);
moduleResult[moduleName] = resultBytes;
}

return moduleResult;
}

Future<Null> newWorldTest(
bool strong, List worlds, Map modules, bool omitPlatform) async {
final Uri sdkRoot = computePlatformBinariesLocation(forceBuildDir: true);
final Uri base = Uri.parse("org-dartlang-test:///");
final Uri sdkSummary = base.resolve("vm_platform.dill");
Expand All @@ -200,7 +274,49 @@ Future<Null> newWorldTest(bool strong, List worlds, bool omitPlatform) async {
Map<String, String> sourceFiles;
CompilerOptions options;
TestIncrementalCompiler compiler;

Map<String, List<int>> moduleData;
Map<String, Component> moduleComponents;
Component sdk;
if (modules != null) {
moduleData = await createModules(modules, sdkSummaryData, strong);
sdk = newestWholeComponent = new Component();
new BinaryBuilder(sdkSummaryData, filename: null, disableLazyReading: false)
.readComponent(newestWholeComponent);
}

for (YamlMap world in worlds) {
List<Component> modulesToUse;
if (world["modules"] != null) {
moduleComponents ??= new Map<String, Component>();

sdk.adoptChildren();
for (Component c in moduleComponents.values) {
c.adoptChildren();
}
sdk.unbindCanonicalNames();
sdk.computeCanonicalNames();

modulesToUse = new List<Component>();
for (String moduleName in world["modules"]) {
Component moduleComponent = moduleComponents[moduleName];
if (moduleComponent != null) {
moduleComponent.computeCanonicalNames();
modulesToUse.add(moduleComponent);
}
}
for (String moduleName in world["modules"]) {
Component moduleComponent = moduleComponents[moduleName];
if (moduleComponent == null) {
moduleComponent = new Component(nameRoot: sdk.root);
new BinaryBuilder(moduleData[moduleName],
filename: null, disableLazyReading: false)
.readComponent(moduleComponent);
moduleComponents[moduleName] = moduleComponent;
modulesToUse.add(moduleComponent);
}
}
}
bool brandNewWorld = true;
if (world["worldType"] == "updated") {
brandNewWorld = false;
Expand Down Expand Up @@ -308,6 +424,11 @@ Future<Null> newWorldTest(bool strong, List worlds, bool omitPlatform) async {
}
}

if (modulesToUse != null) {
compiler.setModulesToLoadOnNextComputeDelta(modulesToUse);
compiler.invalidateAllSources();
}

Stopwatch stopwatch = new Stopwatch()..start();
Component component = await compiler.computeDelta(
entryPoints: entries,
Expand All @@ -318,6 +439,9 @@ Future<Null> newWorldTest(bool strong, List worlds, bool omitPlatform) async {
util.throwOnEmptyMixinBodies(component);
util.throwOnInsufficientUriToSource(component);
print("Compile took ${stopwatch.elapsedMilliseconds} ms");

checkExpectedContent(world, component);

newestWholeComponentData = util.postProcess(component);
newestWholeComponent = component;
print("*****\n\ncomponent:\n"
Expand Down Expand Up @@ -435,6 +559,45 @@ Future<Null> newWorldTest(bool strong, List worlds, bool omitPlatform) async {
}
}

void checkExpectedContent(YamlMap world, Component component) {
if (world["expectedContent"] != null) {
Map<String, Set<String>> actualContent = new Map<String, Set<String>>();
for (Library lib in component.libraries) {
Set<String> libContent =
actualContent[lib.importUri.toString()] = new Set<String>();
for (Class c in lib.classes) {
libContent.add("Class ${c.name}");
}
for (Procedure p in lib.procedures) {
libContent.add("Procedure ${p.name}");
}
for (Field f in lib.fields) {
libContent.add("Field ${f.name}");
}
}

Map expectedContent = world["expectedContent"];

doThrow() {
throw "Expected and actual content not the same.\n"
"Expected $expectedContent.\n"
"Got $actualContent";
}

if (actualContent.length != expectedContent.length) doThrow();
Set<String> missingKeys = actualContent.keys.toSet()
..removeAll(expectedContent.keys);
if (missingKeys.isNotEmpty) doThrow();
for (String key in expectedContent.keys) {
Set<String> expected = new Set<String>.from(expectedContent[key]);
Set<String> actual = actualContent[key].toSet();
if (expected.length != actual.length) doThrow();
actual.removeAll(expected);
if (actual.isNotEmpty) doThrow();
}
}
}

String componentToStringSdkFiltered(Component node) {
Component c = new Component();
List<Uri> dartUris = new List<Uri>();
Expand Down
Loading

0 comments on commit 83e6e9f

Please sign in to comment.