From 83e6e9f8e6fb649e1f92fd31ebdcefb309c6c24f Mon Sep 17 00:00:00 2001 From: Jens Johansen Date: Thu, 28 Mar 2019 08:39:41 +0000 Subject: [PATCH] Allow the incremental compiler to load modules Change-Id: I63bbbdfc94ac6ab0b7b8cce104fe38c4ba592f34 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/95382 Commit-Queue: Jens Johansen Reviewed-by: Kevin Millikin --- .../src/fasta/dill/dill_library_builder.dart | 2 +- .../lib/src/fasta/incremental_compiler.dart | 76 +++++++- .../test/incremental_load_from_dill_test.dart | 169 +++++++++++++++++- .../changing_modules.yaml | 147 +++++++++++++++ 4 files changed, 386 insertions(+), 8 deletions(-) create mode 100644 pkg/front_end/testcases/incremental_initialize_from_dill/changing_modules.yaml diff --git a/pkg/front_end/lib/src/fasta/dill/dill_library_builder.dart b/pkg/front_end/lib/src/fasta/dill/dill_library_builder.dart index 610dcea81dc39..c76a2bc18ae80 100644 --- a/pkg/front_end/lib/src/fasta/dill/dill_library_builder.dart +++ b/pkg/front_end/lib/src/fasta/dill/dill_library_builder.dart @@ -53,7 +53,7 @@ import 'dill_type_alias_builder.dart' show DillTypeAliasBuilder; class DillLibraryBuilder extends LibraryBuilder { final Library library; - final DillLoader loader; + DillLoader loader; /// Exports that can't be serialized. /// diff --git a/pkg/front_end/lib/src/fasta/incremental_compiler.dart b/pkg/front_end/lib/src/fasta/incremental_compiler.dart index 862775f610510..c2ac1800c823e 100644 --- a/pkg/front_end/lib/src/fasta/incremental_compiler.dart +++ b/pkg/front_end/lib/src/fasta/incremental_compiler.dart @@ -211,6 +211,8 @@ class IncrementalCompiler implements IncrementalKernelGenerator { Set invalidatedUris = this.invalidatedUris.toSet(); + invalidateNotKeptUserBuilders(invalidatedUris); + ClassHierarchy hierarchy = userCode?.loader?.hierarchy; Set notReusedLibraries = new Set(); List reusedLibraries = computeReusedLibraries( @@ -225,6 +227,22 @@ class IncrementalCompiler implements IncrementalKernelGenerator { CompilerContext.current.uriToSource.remove(builder.fileUri); } + if (hasToCheckPackageUris) { + // The package file was changed. + // Make sure the dill loader is on the same page. + DillTarget oldDillLoadedData = dillLoadedData; + dillLoadedData = + new DillTarget(ticker, uriTranslator, c.options.target); + for (DillLibraryBuilder library + in oldDillLoadedData.loader.builders.values) { + library.loader = dillLoadedData.loader; + dillLoadedData.loader.builders[library.uri] = library; + if (library.uri.scheme == "dart" && library.uri.path == "core") { + dillLoadedData.loader.coreLibrary = library; + } + } + } + for (LibraryBuilder builder in notReusedLibraries) { Library lib = builder.target; CompilerContext.current.uriToSource.remove(builder.fileUri); @@ -250,6 +268,8 @@ class IncrementalCompiler implements IncrementalKernelGenerator { " of ${userCode.loader.builders.length} libraries"); } + await loadEnsureLoadedComponents(reusedLibraryUris, reusedLibraries); + KernelTarget userCodeOld = userCode; userCode = new KernelTarget( new HybridFileSystem( @@ -342,13 +362,57 @@ class IncrementalCompiler implements IncrementalKernelGenerator { /// Internal method. void invalidateNotKeptUserBuilders(Set invalidatedUris) { - throw UnimplementedError("Not implemented yet."); + if (modulesToLoad != null && userBuilders != null) { + Set loadedNotKept = new Set(); + for (LibraryBuilder builder in userBuilders.values) { + loadedNotKept.add(builder.target); + } + for (Component module in modulesToLoad) { + loadedNotKept.removeAll(module.libraries); + } + for (Library lib in loadedNotKept) { + invalidatedUris.add(lib.importUri); + } + } } /// Internal method. Future loadEnsureLoadedComponents( Set reusedLibraryUris, List reusedLibraries) async { - throw UnimplementedError("Not implemented yet."); + if (modulesToLoad != null) { + bool loadedAnything = false; + for (Component module in modulesToLoad) { + bool usedComponent = false; + for (Library lib in module.libraries) { + if (!reusedLibraryUris.contains(lib.importUri)) { + dillLoadedData.loader.libraries.add(lib); + dillLoadedData.addLibrary(lib); + reusedLibraries.add(dillLoadedData.loader.read(lib.importUri, -1)); + usedComponent = true; + } + } + if (usedComponent) { + dillLoadedData.uriToSource.addAll(module.uriToSource); + loadedAnything = true; + } + } + if (loadedAnything) { + await dillLoadedData.buildOutlines(); + userBuilders = {}; + platformBuilders = []; + dillLoadedData.loader.builders.forEach((uri, builder) { + if (builder.uri.scheme == "dart") { + platformBuilders.add(builder); + } else { + userBuilders[uri] = builder; + } + }); + if (userBuilders.isEmpty) { + userBuilders = null; + } + } + modulesToLoad = null; + } } /// Internal method. @@ -849,12 +913,16 @@ class IncrementalCompiler implements IncrementalKernelGenerator { @override void invalidateAllSources() { - throw UnimplementedError("Not implemented yet."); + if (userCode != null) { + Set uris = new Set.from(userCode.loader.builders.keys); + uris.removeAll(dillLoadedData.loader.builders.keys); + invalidatedUris.addAll(uris); + } } @override void setModulesToLoadOnNextComputeDelta(List components) { - throw UnimplementedError("Not implemented yet."); + modulesToLoad = components.toList(); } /// Internal method. diff --git a/pkg/front_end/test/incremental_load_from_dill_test.dart b/pkg/front_end/test/incremental_load_from_dill_test.dart index 5b01e0f0c1043..4ee035ff73475 100644 --- a/pkg/front_end/test/incremental_load_from_dill_test.dart +++ b/pkg/front_end/test/incremental_load_from_dill_test.dart @@ -14,7 +14,10 @@ import 'package:front_end/src/base/processed_options.dart' import 'package:front_end/src/fasta/compiler_context.dart' show CompilerContext; import 'package:front_end/src/api_prototype/compiler_options.dart' - show CompilerOptions, DiagnosticMessage; + show CompilerOptions; + +import 'package:front_end/src/api_prototype/diagnostic_message.dart' + show DiagnosticMessage, getMessageCodeObject; import "package:front_end/src/api_prototype/memory_file_system.dart" show MemoryFileSystem; @@ -27,7 +30,10 @@ import 'package:front_end/src/fasta/incremental_compiler.dart' import 'package:front_end/src/fasta/severity.dart' show Severity; -import 'package:kernel/kernel.dart' show Component, Library; +import 'package:kernel/binary/ast_from_binary.dart' show BinaryBuilder; + +import 'package:kernel/kernel.dart' + show Class, Component, Field, Library, Procedure; import 'package:kernel/target/targets.dart' show TargetFlags; @@ -111,6 +117,7 @@ class RunCompilations extends Step { await newWorldTest( map["strong"], map["worlds"], + map["modules"], map["omitPlatform"], ); break; @@ -180,7 +187,74 @@ Future basicTest(YamlMap sourceFiles, String entryPoint, bool strong, checkIsEqual(normalDillData, initializedDillData); } -Future newWorldTest(bool strong, List worlds, bool omitPlatform) async { +Future>> createModules( + Map module, final List sdkSummaryData, bool strong) async { + final Uri base = Uri.parse("org-dartlang-test:///"); + final Uri sdkSummary = base.resolve("vm_platform.dill"); + + MemoryFileSystem fs = new MemoryFileSystem(base); + fs.entityForUri(sdkSummary).writeAsBytesSync(sdkSummaryData); + + // Setup all sources + for (Map moduleSources in module.values) { + for (String filename in moduleSources.keys) { + String data = moduleSources[filename]; + Uri uri = base.resolve(filename); + if (await fs.entityForUri(uri).exists()) + throw "More than one entry for $filename"; + fs.entityForUri(uri).writeAsStringSync(data); + } + } + + Map> moduleResult = new Map>(); + + for (String moduleName in module.keys) { + List moduleSources = new List(); + Uri packagesUri; + for (String filename in module[moduleName].keys) { + Uri uri = base.resolve(filename); + if (uri.pathSegments.last == ".packages") { + packagesUri = uri; + } else { + moduleSources.add(uri); + } + } + CompilerOptions options = getOptions(strong); + options.fileSystem = fs; + options.sdkRoot = null; + options.sdkSummary = sdkSummary; + options.omitPlatform = true; + options.onDiagnostic = (DiagnosticMessage message) { + if (getMessageCodeObject(message)?.name == "InferredPackageUri") return; + throw message.ansiFormatted; + }; + if (packagesUri != null) { + options.packagesFileUri = packagesUri; + } + TestIncrementalCompiler compiler = + new TestIncrementalCompiler(options, moduleSources.first, null); + Component c = await compiler.computeDelta(entryPoints: moduleSources); + c.computeCanonicalNames(); + List wantedLibs = new List(); + for (Library lib in c.libraries) { + if (moduleSources.contains(lib.importUri) || + moduleSources.contains(lib.fileUri)) { + wantedLibs.add(lib); + } + } + if (wantedLibs.length != moduleSources.length) { + throw "Module probably not setup right."; + } + Component result = new Component(libraries: wantedLibs); + List resultBytes = util.postProcess(result); + moduleResult[moduleName] = resultBytes; + } + + return moduleResult; +} + +Future newWorldTest( + bool strong, List worlds, Map modules, bool omitPlatform) async { final Uri sdkRoot = computePlatformBinariesLocation(forceBuildDir: true); final Uri base = Uri.parse("org-dartlang-test:///"); final Uri sdkSummary = base.resolve("vm_platform.dill"); @@ -200,7 +274,49 @@ Future newWorldTest(bool strong, List worlds, bool omitPlatform) async { Map sourceFiles; CompilerOptions options; TestIncrementalCompiler compiler; + + Map> moduleData; + Map moduleComponents; + Component sdk; + if (modules != null) { + moduleData = await createModules(modules, sdkSummaryData, strong); + sdk = newestWholeComponent = new Component(); + new BinaryBuilder(sdkSummaryData, filename: null, disableLazyReading: false) + .readComponent(newestWholeComponent); + } + for (YamlMap world in worlds) { + List modulesToUse; + if (world["modules"] != null) { + moduleComponents ??= new Map(); + + sdk.adoptChildren(); + for (Component c in moduleComponents.values) { + c.adoptChildren(); + } + sdk.unbindCanonicalNames(); + sdk.computeCanonicalNames(); + + modulesToUse = new List(); + for (String moduleName in world["modules"]) { + Component moduleComponent = moduleComponents[moduleName]; + if (moduleComponent != null) { + moduleComponent.computeCanonicalNames(); + modulesToUse.add(moduleComponent); + } + } + for (String moduleName in world["modules"]) { + Component moduleComponent = moduleComponents[moduleName]; + if (moduleComponent == null) { + moduleComponent = new Component(nameRoot: sdk.root); + new BinaryBuilder(moduleData[moduleName], + filename: null, disableLazyReading: false) + .readComponent(moduleComponent); + moduleComponents[moduleName] = moduleComponent; + modulesToUse.add(moduleComponent); + } + } + } bool brandNewWorld = true; if (world["worldType"] == "updated") { brandNewWorld = false; @@ -308,6 +424,11 @@ Future newWorldTest(bool strong, List worlds, bool omitPlatform) async { } } + if (modulesToUse != null) { + compiler.setModulesToLoadOnNextComputeDelta(modulesToUse); + compiler.invalidateAllSources(); + } + Stopwatch stopwatch = new Stopwatch()..start(); Component component = await compiler.computeDelta( entryPoints: entries, @@ -318,6 +439,9 @@ Future newWorldTest(bool strong, List worlds, bool omitPlatform) async { util.throwOnEmptyMixinBodies(component); util.throwOnInsufficientUriToSource(component); print("Compile took ${stopwatch.elapsedMilliseconds} ms"); + + checkExpectedContent(world, component); + newestWholeComponentData = util.postProcess(component); newestWholeComponent = component; print("*****\n\ncomponent:\n" @@ -435,6 +559,45 @@ Future newWorldTest(bool strong, List worlds, bool omitPlatform) async { } } +void checkExpectedContent(YamlMap world, Component component) { + if (world["expectedContent"] != null) { + Map> actualContent = new Map>(); + for (Library lib in component.libraries) { + Set libContent = + actualContent[lib.importUri.toString()] = new Set(); + for (Class c in lib.classes) { + libContent.add("Class ${c.name}"); + } + for (Procedure p in lib.procedures) { + libContent.add("Procedure ${p.name}"); + } + for (Field f in lib.fields) { + libContent.add("Field ${f.name}"); + } + } + + Map expectedContent = world["expectedContent"]; + + doThrow() { + throw "Expected and actual content not the same.\n" + "Expected $expectedContent.\n" + "Got $actualContent"; + } + + if (actualContent.length != expectedContent.length) doThrow(); + Set missingKeys = actualContent.keys.toSet() + ..removeAll(expectedContent.keys); + if (missingKeys.isNotEmpty) doThrow(); + for (String key in expectedContent.keys) { + Set expected = new Set.from(expectedContent[key]); + Set actual = actualContent[key].toSet(); + if (expected.length != actual.length) doThrow(); + actual.removeAll(expected); + if (actual.isNotEmpty) doThrow(); + } + } +} + String componentToStringSdkFiltered(Component node) { Component c = new Component(); List dartUris = new List(); diff --git a/pkg/front_end/testcases/incremental_initialize_from_dill/changing_modules.yaml b/pkg/front_end/testcases/incremental_initialize_from_dill/changing_modules.yaml new file mode 100644 index 0000000000000..ba6a85f00a793 --- /dev/null +++ b/pkg/front_end/testcases/incremental_initialize_from_dill/changing_modules.yaml @@ -0,0 +1,147 @@ +# Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE.md file. + +# Compile an application with a number of modules. +# Compile again with changing modules. + +type: newworld +strong: true +modules: + example_0.1.0: + example_0.1.0/a.dart: | + a() { + la1(); + } + la1() { + print("Hello from v0.1.0"); + } + example_0.1.0/b.dart: | + import "a.dart"; + b() { + a(); + } + example_0.1.0/.packages: | + example:. + example_0.1.1: + example_0.1.1/b.dart: | + b() { + print("Hello from v0.1.1"); + } + bool example011 = true; + example_0.1.1/.packages: | + example:. + foo_1: + foo_1/foo.dart: | + import "package:example/b.dart"; + foo() { + print("Hello from foo"); + b(); + } + bool foo1 = true; + foo_1/.packages: | + foo:. + example:../example_0.1.0 + foo_2: + foo_2/foo.dart: | + import "package:example/b.dart"; + import "bar.dart"; + import "baz.dart"; + foo() { + print("Hello from foo 2"); + bar(); + baz(); + b(); + } + bool foo2 = true; + foo_2/bar.dart: | + bar() { + print("hello from bar"); + } + foo_2/baz.dart: | + baz() { + print("hello from baz"); + } + foo_2/.packages: | + foo:. + example:../example_0.1.1 +worlds: + - entry: main.dart + fromComponent: true + sources: + main.dart: | + import "package:example/b.dart"; + main() { + print("hello"); + b(); + } + .packages: example:example_0.1.0 + modules: + - example_0.1.0 + expectedLibraryCount: 3 + expectedContent: + org-dartlang-test:///main.dart: + - Procedure main + package:example/b.dart: + - Procedure b + package:example/a.dart: + - Procedure a + - Procedure la1 + - entry: main.dart + worldType: updated + expectInitializeFromDill: false + sources: + main.dart: | + import "package:foo/foo.dart"; + main() { + print("hello"); + foo(); + } + .packages: | + example:example_0.1.0 + foo:foo_1 + modules: + - example_0.1.0 + - foo_1 + expectedLibraryCount: 4 + expectedContent: + org-dartlang-test:///main.dart: + - Procedure main + package:example/b.dart: + - Procedure b + package:example/a.dart: + - Procedure a + - Procedure la1 + package:foo/foo.dart: + - Procedure foo + - Field foo1 + - entry: main.dart + worldType: updated + expectInitializeFromDill: false + sources: + main.dart: | + import "package:foo/foo.dart"; + main() { + print("hello"); + foo(); + } + .packages: | + example:example_0.1.1 + foo:foo_2 + modules: + - example_0.1.1 + - foo_2 + expectedLibraryCount: 5 + expectedContent: + org-dartlang-test:///main.dart: + - Procedure main + package:example/b.dart: + - Procedure b + - Field example011 + package:foo/foo.dart: + - Procedure foo + - Field foo2 + package:foo/bar.dart: + - Procedure bar + package:foo/baz.dart: + - Procedure baz