From c25bfb5101c598d0125ac8dbed6c1f8ab9201ac0 Mon Sep 17 00:00:00 2001 From: PikachuHy Date: Mon, 20 May 2024 17:09:52 +0800 Subject: [PATCH] support C++20 Modules, construct build graph --- .../lib/rules/cpp/CcCompilationHelper.java | 465 +++++++++++++++++- 1 file changed, 463 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java index 9e2a98a550531d..7c0c6baf652012 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java @@ -34,6 +34,7 @@ import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.packages.BuildType; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.rules.cpp.CcCommon.CoptsFilter; @@ -58,6 +59,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Starlark; @@ -863,8 +865,21 @@ public CompilationInfo compile(RuleContext ruleContext) // Create compile actions (both PIC and no-PIC). try { - CcCompilationOutputs ccOutputs = createCcCompileActions(); - + CcCompilationOutputs ccOutputs; + if (featureConfiguration.isEnabled(CppRuleClasses.CPP20_MODULE)) { + // Handle C++20 Module compile + ccOutputs = createCcCompileActionsWithCpp20Module(); + publicCompilationContext = + CcCompilationContext.createWithCpp20Modules( + publicCompilationContext, + ccOutputs.getPcmFiles(false), + ccOutputs.getPcmFiles(true), + ccOutputs.getModulesInfoFiles(false), + ccOutputs.getModulesInfoFiles(true)); + } else { + // Create compile actions (both PIC and no-PIC). + ccOutputs = createCcCompileActions(); + } if (cppConfiguration.processHeadersInDependencies()) { return new CompilationInfo( CcCompilationContext.createWithExtraHeaderTokens( @@ -990,6 +1005,429 @@ private ImmutableSet getSourceArtifactsByType( return result.build(); } + private CcCompilationOutputs createCcCompileActionsWithCpp20Module() + throws RuleErrorException, EvalException, InterruptedException { + Preconditions.checkState( + featureConfiguration.isEnabled(CppRuleClasses.CPP20_MODULE), + "to use C++20 Modules, the feature cpp20_module must be enabled"); + Preconditions.checkNotNull(ccCompilationContext); + CcCompilationOutputs.Builder result = CcCompilationOutputs.builder(); + // merge module interfaces and ordinary sources + Map sourcesMap = new LinkedHashMap<>(); + sourcesMap.putAll(compilationUnitSources); + sourcesMap.putAll(moduleInterfaceSources); + ImmutableMap outputNameMap = + calculateOutputNameMapByType(sourcesMap, /* prefixDir= */ null); + if (generateNoPicAction) { + createCcCompileActionsWithCpp20ModuleHelper(result, /* usePic= */ false, outputNameMap); + } + if (generatePicAction) { + createCcCompileActionsWithCpp20ModuleHelper(result, /* usePic= */ true, outputNameMap); + } + return result.build(); + } + + private void createCcCompileActionsWithCpp20ModuleHelper( + CcCompilationOutputs.Builder result, + boolean usePic, + ImmutableMap outputNameMap) + throws RuleErrorException, EvalException, InterruptedException { + NestedSetBuilder pcmSetBuilder = NestedSetBuilder.stableOrder(); + NestedSetBuilder ddiSetBuilder = NestedSetBuilder.stableOrder(); + ImmutableList.Builder> pcmAndDdiPairListBuilder = + ImmutableList.builder(); + + // declare .CXXModules.json forward + // all modules information is put here + String modulesInfoFileName = label.getName(); + if (usePic) { + modulesInfoFileName = + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.PIC_FILE, modulesInfoFileName); + } + Artifact modulesInfoFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.CPP20_MODULES_INFO, modulesInfoFileName), + configuration); + if (usePic) { + result.addPicModulesInfoFile(modulesInfoFile); + } else { + result.addModulesInfoFile(modulesInfoFile); + } + // the builder list contains + // 1. compile c++ source (e.g. .cc -> .o) compilationUnitSources.size() + // 2. compile c++ module (e.g. .cppm -> .pcm) moduleInterfaceSources.size() + // 3. compile c++ module (e.g. .cpm -> .o) moduleInterfaceSources.size() + List builderList = + new ArrayList<>(compilationUnitSources.size() + 2 * moduleInterfaceSources.size()); + + createCppCompileActionBuilder( + result, + usePic, + outputNameMap, + pcmSetBuilder, + ddiSetBuilder, + pcmAndDdiPairListBuilder, + modulesInfoFile, + builderList, + compilationUnitSources, + /* isModuleInterface= */ false); + createCppCompileActionBuilder( + result, + usePic, + outputNameMap, + pcmSetBuilder, + ddiSetBuilder, + pcmAndDdiPairListBuilder, + modulesInfoFile, + builderList, + moduleInterfaceSources, + /* isModuleInterface= */ true); + // create .CXXModules.json + var modulesInfoAction = + new Cpp20ModulesInfoAction( + actionConstructionContext.getActionOwner(), + ddiSetBuilder.build(), + pcmAndDdiPairListBuilder.build(), + ccCompilationContext.getModulesInfoFiles(usePic), + modulesInfoFile); + actionConstructionContext.registerAction(modulesInfoAction); + var depPcmFiles = ccCompilationContext.getPcmFiles(usePic); + if (depPcmFiles != null) { + pcmSetBuilder.addTransitive(depPcmFiles); + } + var pcmFiles = pcmSetBuilder.build(); + for (CppCompileActionBuilder builder : builderList) { + builder.setPcmFiles(pcmFiles); + CppCompileAction cppCompileAction = builder.buildOrThrowRuleError(ruleErrorConsumer); + actionConstructionContext.registerAction(cppCompileAction); + } + } + + private void createCppCompileActionBuilder( + CcCompilationOutputs.Builder result, + boolean usePic, + ImmutableMap outputNameMap, + NestedSetBuilder pcmSetBuilder, + NestedSetBuilder ddiSetBuilder, + ImmutableList.Builder> pcmAndDdiPairListBuilder, + Artifact modulesInfoFile, + List builderList, + Map sourceMap, + boolean isModuleInterface) + throws RuleErrorException, EvalException, InterruptedException { + for (CppSource source : sourceMap.values()) { + CppCompileActionBuilder builder; + Label sourceLabel = source.getLabel(); + Artifact sourceArtifact = source.getSource(); + PathFragment sourcePath = sourceArtifact.getExecPath(); + String outputName = outputNameMap.get(sourceArtifact); + if (usePic) { + outputName = + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.PIC_FILE, outputName); + } + builder = initializeCompileAction(sourceArtifact); + // declare .d file forward + // reuse .d file produced by scan dependencies (c++20-deps-scanning) + Artifact dotdFile; + if (builder.dotdFilesEnabled() + && builder.useDotdFile(sourceArtifact)) { + String dotdFileName = CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.INCLUDED_FILE_LIST, outputName); + dotdFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, label, dotdFileName, configuration); + } else { + dotdFile = null; + } + // scan dependencies + // both module source files and ordinary c++ source files are needed + // others skip (e.g. C or assembler) + if (isModuleInterface || CppFileTypes.CPP_SOURCE.matches(sourcePath)) { + // dependencies information are put in .ddi file + // the format is https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html + Artifact ddiFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.CPP20_MODULE_DEP, outputName), + configuration); + ddiSetBuilder.add(ddiFile); + // all -fmodule-file== flags are put in .modmap file + var modmapFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.CPP20_MODULE_MAP, outputName), + configuration); + // all path/to/bmi are put in .modmap.input file, + // which is convenient to get all bmi in CppCompileAction + var modmapInputFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.CPP20_MODULE_MAP_INPUT, outputName), + configuration); + Cpp20ModuleDepMapAction cpp20ModuleDepMapAction = + new Cpp20ModuleDepMapAction( + actionConstructionContext.getActionOwner(), + ddiFile, + modulesInfoFile, + modmapFile, + modmapInputFile); + actionConstructionContext.registerAction(cpp20ModuleDepMapAction); + + if (isModuleInterface) { + // use two-phase compilation + // e.g. + // 1. .cppm -> .pcm + // 2. .pcm -> .o + // echo phase need one builder + // so the builder will change + // and the new builder return + builder = + createCpp20ModuleCompileActionBuilder( + builder, + sourceLabel, + outputName, + result, + sourceArtifact, + usePic, + pcmSetBuilder, + pcmAndDdiPairListBuilder, + builderList, + ddiFile, + dotdFile, + modmapFile, + modmapInputFile); + } else { + createCompileActionBuilder( + builder, + sourceLabel, + outputName, + result, + sourceArtifact, + usePic, + ddiFile, + dotdFile, + modmapFile, + /* isCpp20Module= */ false); + } + builder.setModmapFile(modmapFile); + builder.setModmapInputFile(modmapInputFile); + } else { + createCompileActionBuilder( + builder, + sourceLabel, + outputName, + result, + sourceArtifact, + usePic, + /* ddiFile= */ null, + dotdFile, + /* modmapFile= */ null, + /* isCpp20Module= */ false); + } + semantics.finalizeCompileActionBuilder( + configuration, featureConfiguration, builder); + builderList.add(builder); + } + } + + private CppCompileActionBuilder createCpp20ModuleCompileActionBuilder( + CppCompileActionBuilder moduleCompileBuilder, + Label sourceLabel, + String outputName, + CcCompilationOutputs.Builder result, + Artifact sourceArtifact, + boolean usePic, + NestedSetBuilder pcmSetBuilder, + ImmutableList.Builder> pcmAndDdiPairListBuilder, + List builderList, + Artifact ddiFile, + Artifact dotdFile, + Artifact modmapFile, + Artifact modmapInputFile) + throws RuleErrorException, EvalException, InterruptedException { + Artifact diagnosticsFile; + var outputCategory = ArtifactCategory.CPP_MODULE; + if (moduleCompileBuilder.serializedDiagnosticsFilesEnabled()) { + String diagnosticsFileName = + CppHelper.getDiagnosticsFileName(ccToolchain, outputCategory, outputName); + diagnosticsFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, label, diagnosticsFileName, configuration); + } else { + diagnosticsFile = null; + } + String pcmFileName = + CppHelper.getArtifactNameForCategory(ccToolchain, outputCategory, outputName); + PathFragment objectDir = + CppHelper.getObjDirectory(label, configuration.isSiblingRepositoryLayout()); + // the DerivedArtifact type is required when using restart mechanism + Artifact.DerivedArtifact pcmFile = + actionConstructionContext.getDerivedArtifact( + objectDir.getRelative(pcmFileName), + configuration.getBinDirectory(label.getRepository())); + pcmSetBuilder.add(pcmFile); + pcmAndDdiPairListBuilder.add(Pair.of(pcmFile, ddiFile)); + if (usePic) { + result.addPicPcmFile(pcmFile); + } else { + result.addPcmFile(pcmFile); + } + moduleCompileBuilder.setOutputs(pcmFile, dotdFile, diagnosticsFile); + moduleCompileBuilder.setModmapFile(modmapFile); + moduleCompileBuilder.setModmapInputFile(modmapInputFile); + moduleCompileBuilder.setActionName(CppActionNames.CPP20_MODULE_COMPILE); + PathFragment ccRelativeName = sourceArtifact.getRootRelativePath(); + var variables = + setupCompileBuildVariables( + moduleCompileBuilder, + sourceLabel, + usePic, + /* needsFdoBuildVariables= */ ccRelativeName != null, + cppModuleMap, + /* enableCoverage= */ false, + null, + false, + null, + null, + /* additionalBuildVariables= */ ImmutableMap.of( + CompileBuildVariables.CPP20_MODMAP_FILE.getVariableName(), + modmapFile.getExecPathString())); + moduleCompileBuilder.setVariables(variables); + semantics.finalizeCompileActionBuilder( + configuration, featureConfiguration, moduleCompileBuilder); + builderList.add(moduleCompileBuilder); + createScanDepsAction(variables, sourceArtifact, ddiFile, dotdFile); + // after compile module interface file to pcm file + // we compile pcm file to object file + var moduleCodegenBuilder = initializeCompileAction(pcmFile); + moduleCodegenBuilder.addMandatoryInputs(List.of(sourceArtifact)); + createCompileActionBuilder( + moduleCodegenBuilder, + sourceLabel, + outputName, + result, + pcmFile, + usePic, + /* ddiFile= */ null, + dotdFile, + modmapFile, + /* isCpp20Module= */ true); + return moduleCodegenBuilder; + } + + private void createCompileActionBuilder( + CppCompileActionBuilder builder, + Label sourceLabel, + String outputName, + CcCompilationOutputs.Builder result, + Artifact sourceArtifact, + boolean usePic, + Artifact ddiFile, + Artifact dotdFile, + Artifact modmapFile, + boolean isCpp20Module) + throws RuleErrorException, EvalException, InterruptedException { + if (isCpp20Module) { + builder.setActionName(CppActionNames.CPP20_MODULE_CODEGEN); + } else { + // do nothing + // fallback to default action name + } + boolean enableCoverage = isCodeCoverageEnabled; + boolean generateDwo = + ccToolchain.shouldCreatePerObjectDebugInfo(featureConfiguration, cppConfiguration); + boolean bitcodeOutput = + featureConfiguration.isEnabled(CppRuleClasses.THIN_LTO) + && CppFileTypes.LTO_SOURCE.matches(sourceArtifact.getFilename()); + PathFragment ccRelativeName = sourceArtifact.getRootRelativePath(); + + Artifact outputFile = CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory(ccToolchain, ArtifactCategory.OBJECT_FILE, outputName), + configuration + ); + Artifact diagnosticsFile; + if (builder.serializedDiagnosticsFilesEnabled()) { + String diagnosticsFileName = + CppHelper.getDiagnosticsFileName(ccToolchain, ArtifactCategory.OBJECT_FILE, outputName); + diagnosticsFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, label, diagnosticsFileName, configuration); + } + else { + diagnosticsFile = null; + } + builder.setOutputs(outputFile, dotdFile, diagnosticsFile); + String gcnoFileName = + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.COVERAGE_DATA_FILE, outputName); + Artifact gcnoFile = + enableCoverage + ? CppHelper.getCompileOutputArtifact( + actionConstructionContext, label, gcnoFileName, configuration) + : null; + Artifact dwoFile = generateDwo && !bitcodeOutput ? getDwoFile(outputFile) : null; + Artifact ltoIndexingFile = bitcodeOutput ? getLtoIndexingFile(outputFile) : null; + // compile arguments are produced by the same CompileBuildVariables + var variables = + setupCompileBuildVariables( + builder, + sourceLabel, + /* usePic= */ usePic, + /* needsFdoBuildVariables= */ ccRelativeName != null, + cppModuleMap, + /* enableCoverage= */ false, + gcnoFile, + generateDwo, + dwoFile, + ltoIndexingFile, + /* additionalBuildVariables= */ modmapFile == null + ? ImmutableMap.of() + : ImmutableMap.of( + CompileBuildVariables.CPP20_MODMAP_FILE.getVariableName(), + modmapFile.getExecPathString())); + + if (ddiFile != null) { + createScanDepsAction(variables, sourceArtifact, ddiFile, dotdFile); + } + builder.setVariables(variables); + builder.setGcnoFile(gcnoFile); + builder.setDwoFile(dwoFile); + builder.setLtoIndexingFile(ltoIndexingFile); + + result.addTemps( + createTempsActions( + sourceArtifact, sourceLabel, outputName, builder, usePic, ccRelativeName)); + if (usePic) { + result.addPicObjectFile(outputFile); + if (dwoFile != null) { + result.addPicDwoFile(dwoFile); + } + if (gcnoFile != null) { + result.addPicGcnoFile(gcnoFile); + } + } else { + result.addObjectFile(outputFile); + if (dwoFile != null) { + result.addDwoFile(dwoFile); + } + if (gcnoFile != null) { + result.addGcnoFile(gcnoFile); + } + } + } /** * Constructs the C++ compiler actions. It generally creates one action for every specified source * file. It takes into account coverage, and PIC, in addition to using the settings specified on @@ -997,6 +1435,9 @@ private ImmutableSet getSourceArtifactsByType( */ private CcCompilationOutputs createCcCompileActions() throws RuleErrorException, EvalException, InterruptedException { + Preconditions.checkState( + moduleInterfaceSources.isEmpty(), + "to use C++20 Modules, the feature cpp20_module must be enabled"); CcCompilationOutputs.Builder result = CcCompilationOutputs.builder(); Preconditions.checkNotNull(ccCompilationContext); @@ -1516,6 +1957,22 @@ private void createHeaderAction( result.addHeaderTokenFile(tokenFile); } + private void createScanDepsAction( + CcToolchainVariables variables, Artifact sourceArtifact, Artifact ddiFile, Artifact dotdFile) + throws RuleErrorException, EvalException { + var scanDepsBuilder = initializeCompileAction(sourceArtifact); + scanDepsBuilder.setActionName(CppActionNames.CPP20_DEPS_SCANNING); + scanDepsBuilder.setOutputs(ddiFile, dotdFile, null); + // only c++20-deps-scanning add .d file + var buildVariables = CcToolchainVariables.builder(variables) + .addStringVariable(CompileBuildVariables.DEPENDENCY_FILE.getVariableName(), dotdFile.getExecPathString()); + scanDepsBuilder.setVariables(buildVariables.build()); + semantics.finalizeCompileActionBuilder( + configuration, featureConfiguration, scanDepsBuilder); + var scanDepsAction = scanDepsBuilder.buildOrThrowRuleError(ruleErrorConsumer); + actionConstructionContext.registerAction(scanDepsAction); + } + private ImmutableList createModuleAction( CcCompilationOutputs.Builder result, CppModuleMap cppModuleMap) throws RuleErrorException, EvalException, InterruptedException { @@ -1865,4 +2322,8 @@ private ImmutableList createTempsActions( return ImmutableList.of(dAction.getPrimaryOutput(), sdAction.getPrimaryOutput()); } + private NestedSet getModuleInterfaceSourceFiles() { + var moduleInterfaceFiles = moduleInterfaceSources.values().stream().map(CppSource::getSource).collect(Collectors.toList()); + return NestedSetBuilder.wrap(Order.STABLE_ORDER, moduleInterfaceFiles); + } }