New sky function that does the following:

1) call BzlLoadFunction to load the content of PROJECT.scl file given the provided scl_config name
2) call ParsedFlagsFunction to parse the list of options
3) define a patch transition and applies the transition to the targetOptions which was used for creating topLevel configuration
4) the returned adjusted BuildOptions are then used in AnalysisPhaseRunner and BuildTool instead of the original BuildOptions.

PiperOrigin-RevId: 621913271
Change-Id: If0620fdaca6f292a96150faa34f1b3145ec54a23

src/main/java/com/google/devtools/build/lib/analysis/BUILD

@@ -602,13 +602,19 @@ java_library(
     name = "projects",
     srcs = ["Project.java"],
     deps = [
+        ":config/build_options",
+        ":config/core_options",
+        ":config/invalid_configuration_exception",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/skyframe:containing_package_lookup_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:package_lookup_function",
         "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
+        "//src/main/java/com/google/devtools/build/lib/skyframe/config",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
+        "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//src/main/protobuf:failure_details_java_proto",
         "//third_party:guava",
     ],
 )

src/main/java/com/google/devtools/build/lib/analysis/Project.java

@@ -24,14 +24,21 @@
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.CoreOptions;
+import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.cmdline.PackageIdentifier;
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.server.FailureDetails.BuildConfiguration.Code;
 import com.google.devtools.build.lib.skyframe.ContainingPackageLookupValue;
 import com.google.devtools.build.lib.skyframe.PackageLookupFunction;
 import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
+import com.google.devtools.build.lib.skyframe.config.FlagSetValue;
 import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.skyframe.EvaluationResult;
 import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -180,4 +187,34 @@ public static ImmutableMultimap<Label, PathFragment> findProjectFiles(
     }
     return ans.build();
   }
+
+  /**
+   * applies {@link CoreOptions.sclConfig} to the top-level {@link BuildOptions}
+   *
+   * <p>given an existing PROJECT.scl file and an {@link CoreOptions.sclConfig}, the method creates
+   * a {@link SkyKey} containing the {@link PathFragment} of the scl file and the config name which
+   * is evaluated by the {@link FlagSetFunction}
+   *
+   * @return {@link FlagSetValue} which has the effective top-level {@link BuildOptions} after
+   *     project file resolution.
+   */
+  public static FlagSetValue modifyBuildOptionsWithFlagSets(
+      PathFragment projectFile,
+      BuildOptions targetOptions,
+      ExtendedEventHandler eventHandler,
+      SkyframeExecutor skyframeExecutor)
+      throws InvalidConfigurationException {
+
+    FlagSetValue.Key flagSetKey =
+        FlagSetValue.Key.create(
+            projectFile, targetOptions.get(CoreOptions.class).sclConfig, targetOptions);
+
+    EvaluationResult<SkyValue> result =
+        skyframeExecutor.evaluateSkyKeys(
+            eventHandler, ImmutableList.of(flagSetKey), /* keepGoing= */ false);
+    if (result.hasError()) {
+      throw new InvalidConfigurationException("Cannot parse options", Code.INVALID_BUILD_OPTIONS);
+    }
+    return (FlagSetValue) result.get(flagSetKey);
+  }
 }

src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java

@@ -66,6 +66,17 @@ public class CoreOptions extends FragmentOptions implements Cloneable {
   public static final OptionDefinition CPU =
       OptionsParser.getOptionDefinitionByName(CoreOptions.class, "cpu");
 
+  @Option(
+      name = "scl_config",
+      defaultValue = "null",
+      documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      metadataTags = {OptionMetadataTag.EXPERIMENTAL},
+      help =
+          "Name of the scl config defined in PROJECT.scl. Note that this feature is still under"
+              + " development b/324119879.")
+  public String sclConfig;
+
   @Option(
       name = "incompatible_merge_genfiles_directory",
       defaultValue = "true",

src/main/java/com/google/devtools/build/lib/buildtool/AnalysisPhaseRunner.java

@@ -83,11 +83,16 @@ private AnalysisPhaseRunner() {}
   public static AnalysisResult execute(
       CommandEnvironment env,
       BuildRequest request,
-      BuildOptions buildOptions,
+      BuildOptions buildOptionsBeforeFlagSets,
       TargetValidator validator)
-      throws BuildFailedException, InterruptedException, ViewCreationFailedException,
-          TargetParsingException, LoadingFailedException, AbruptExitException,
-          InvalidConfigurationException, RepositoryMappingResolutionException {
+      throws BuildFailedException,
+          InterruptedException,
+          ViewCreationFailedException,
+          TargetParsingException,
+          LoadingFailedException,
+          AbruptExitException,
+          InvalidConfigurationException,
+          RepositoryMappingResolutionException {
 
     // Target pattern evaluation.
     TargetPatternPhaseValue loadingResult;
@@ -97,10 +102,25 @@ public static AnalysisResult execute(
     }
     env.setWorkspaceName(loadingResult.getWorkspaceName());
 
-    // TODO: b/324127375 - Use with incoming --scl_config flag.
-    PathFragment unusedProjectFile =
-        BuildTool.getProjectFile(
-            loadingResult.getTargetLabels(), env.getSkyframeExecutor(), env.getReporter());
+    BuildOptions postFlagsetsBuildOptions;
+    String sclConfig = buildOptionsBeforeFlagSets.get(CoreOptions.class).sclConfig;
+    if (sclConfig != null && !sclConfig.isEmpty()) {
+      PathFragment projectFile =
+          BuildTool.getProjectFile(
+              loadingResult.getTargetLabels(), env.getSkyframeExecutor(), env.getReporter());
+      if (projectFile != null) {
+        postFlagsetsBuildOptions =
+            BuildTool.applySclConfigs(
+                buildOptionsBeforeFlagSets,
+                projectFile,
+                env.getSkyframeExecutor(),
+                env.getReporter());
+      } else {
+        postFlagsetsBuildOptions = buildOptionsBeforeFlagSets;
+      }
+    } else {
+      postFlagsetsBuildOptions = buildOptionsBeforeFlagSets;
+    }
 
     // Compute the heuristic instrumentation filter if needed.
     if (request.needsInstrumentationFilter()) {
@@ -115,7 +135,7 @@ public static AnalysisResult execute(
           // We're modifying the buildOptions in place, which is not ideal, but we also don't want
           // to pay the price for making a copy. Maybe reconsider later if this turns out to be a
           // problem (and the performance loss may not be a big deal).
-          buildOptions.get(CoreOptions.class).instrumentationFilter =
+          postFlagsetsBuildOptions.get(CoreOptions.class).instrumentationFilter =
               new RegexFilter.RegexFilterConverter().convert(instrumentationFilter);
         } catch (OptionsParsingException e) {
           throw new InvalidConfigurationException(Code.HEURISTIC_INSTRUMENTATION_FILTER_INVALID, e);
@@ -131,11 +151,11 @@ public static AnalysisResult execute(
       Profiler.instance().markPhase(ProfilePhase.ANALYZE);
 
       try (SilentCloseable c = Profiler.instance().profile("runAnalysisPhase")) {
-        analysisResult = runAnalysisPhase(env, request, loadingResult, buildOptions);
+        analysisResult = runAnalysisPhase(env, request, loadingResult, postFlagsetsBuildOptions);
       }
 
       for (BlazeModule module : env.getRuntime().getBlazeModules()) {
-        module.afterAnalysis(env, request, buildOptions, analysisResult);
+        module.afterAnalysis(env, request, postFlagsetsBuildOptions, analysisResult);
       }
 
       if (request.shouldRunTests()) {

src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java

@@ -38,6 +38,7 @@
 import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
 import com.google.devtools.build.lib.analysis.actions.TemplateExpansionException;
 import com.google.devtools.build.lib.analysis.config.BuildOptions;
+import com.google.devtools.build.lib.analysis.config.CoreOptions;
 import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
 import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType;
 import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader.UploadContext;
@@ -77,6 +78,7 @@
 import com.google.devtools.build.lib.skyframe.actiongraph.v2.AqueryOutputHandler;
 import com.google.devtools.build.lib.skyframe.actiongraph.v2.AqueryOutputHandler.OutputType;
 import com.google.devtools.build.lib.skyframe.actiongraph.v2.InvalidAqueryOutputFormatException;
+import com.google.devtools.build.lib.skyframe.config.FlagSetValue;
 import com.google.devtools.build.lib.util.AbruptExitException;
 import com.google.devtools.build.lib.util.CrashFailureDetails;
 import com.google.devtools.build.lib.util.DetailedExitCode;
@@ -292,10 +294,16 @@ private void buildTargetsWithMergedAnalysisExecution(
       BuildRequest request,
       BuildResult result,
       TargetValidator validator,
-      BuildOptions buildOptions)
-      throws InterruptedException, TargetParsingException, LoadingFailedException,
-          AbruptExitException, ViewCreationFailedException, BuildFailedException, TestExecException,
-          InvalidConfigurationException, RepositoryMappingResolutionException {
+      BuildOptions buildOptionsBeforeFlagSets)
+      throws InterruptedException,
+          TargetParsingException,
+          LoadingFailedException,
+          AbruptExitException,
+          ViewCreationFailedException,
+          BuildFailedException,
+          TestExecException,
+          InvalidConfigurationException,
+          RepositoryMappingResolutionException {
     // Target pattern evaluation.
     TargetPatternPhaseValue loadingResult;
     Profiler.instance().markPhase(ProfilePhase.TARGET_PATTERN_EVAL);
@@ -305,9 +313,21 @@ private void buildTargetsWithMergedAnalysisExecution(
     }
     env.setWorkspaceName(loadingResult.getWorkspaceName());
 
-    // TODO: b/324127375 - Use with incoming --scl_config flag.
-    PathFragment unusedProjectFile =
-        getProjectFile(loadingResult.getTargetLabels(), env.getSkyframeExecutor(), getReporter());
+    BuildOptions postFlagSetsBuildOptions;
+    String sclConfig = buildOptionsBeforeFlagSets.get(CoreOptions.class).sclConfig;
+    if (sclConfig != null && !sclConfig.isEmpty()) {
+      PathFragment projectFile =
+          getProjectFile(loadingResult.getTargetLabels(), env.getSkyframeExecutor(), getReporter());
+      if (projectFile != null) {
+        postFlagSetsBuildOptions =
+            applySclConfigs(
+                buildOptionsBeforeFlagSets, projectFile, env.getSkyframeExecutor(), getReporter());
+      } else {
+        postFlagSetsBuildOptions = buildOptionsBeforeFlagSets;
+      }
+    } else {
+      postFlagSetsBuildOptions = buildOptionsBeforeFlagSets;
+    }
 
     // See https://github.com/bazelbuild/rules_nodejs/issues/3693.
     env.getSkyframeExecutor().clearSyscallCache();
@@ -326,7 +346,7 @@ private void buildTargetsWithMergedAnalysisExecution(
           AnalysisAndExecutionPhaseRunner.execute(
               env,
               request,
-              buildOptions,
+              postFlagSetsBuildOptions,
               loadingResult,
               () -> executionTool.prepareForExecution(executionTimer),
               result::setBuildConfiguration,
@@ -461,7 +481,7 @@ private void dumpSkyframeStateAfterBuild(
               /* includeSchedulingDependencies= */ true,
               /* actionFilters= */ null,
               /* includeParamFiles= */ false,
-              /* includeFileWriteContents */ false,
+              /* includeFileWriteContents= */ false,
               aqueryOutputHandler,
               getReporter());
       AqueryProcessor.dumpActionGraph(env, aqueryOutputHandler, actionGraphDump);
@@ -809,6 +829,22 @@ static PathFragment getProjectFile(
     }
   }
 
+  /** Creates a BuildOptions class for the given options taken from an {@link OptionsProvider}. */
+  public static BuildOptions applySclConfigs(
+      BuildOptions buildOptionsBeforeFlagSets,
+      PathFragment projectFile,
+      SkyframeExecutor skyframeExecutor,
+      ExtendedEventHandler eventHandler)
+      throws InvalidConfigurationException {
+
+    FlagSetValue flagSetValue =
+        Project.modifyBuildOptionsWithFlagSets(
+            projectFile, buildOptionsBeforeFlagSets, eventHandler, skyframeExecutor);
+
+    // BuildOptions after Flagsets
+    return flagSetValue.getTopLevelBuildOptions();
+  }
+
   private Reporter getReporter() {
     return env.getReporter();
   }

src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java

@@ -167,6 +167,8 @@ public final class SkyFunctions {
   public static final SkyFunctionName MODULE_EXTENSION_REPO_MAPPING_ENTRIES =
       SkyFunctionName.createHermetic("MODULE_EXTENSION_REPO_MAPPING_ENTRIES");
 
+  public static final SkyFunctionName FLAG_SET = SkyFunctionName.createHermetic("FLAG_SET");
+
   public static Predicate<SkyKey> isSkyFunction(SkyFunctionName functionName) {
     return key -> key.functionName().equals(functionName);
   }

src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java

@@ -197,6 +197,7 @@
 import com.google.devtools.build.lib.skyframe.config.BuildConfigurationKey;
 import com.google.devtools.build.lib.skyframe.config.BuildConfigurationKeyFunction;
 import com.google.devtools.build.lib.skyframe.config.BuildConfigurationKeyValue;
+import com.google.devtools.build.lib.skyframe.config.FlagSetFunction;
 import com.google.devtools.build.lib.skyframe.config.NativeAndStarlarkFlags;
 import com.google.devtools.build.lib.skyframe.config.ParsedFlagsFunction;
 import com.google.devtools.build.lib.skyframe.config.ParsedFlagsValue;
@@ -766,6 +767,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) {
         new ArtifactNestedSetFunction(this::getConsumedArtifactsTracker));
     BuildDriverFunction buildDriverFunction = newBuildDriverFunction();
     map.put(SkyFunctions.BUILD_DRIVER, buildDriverFunction);
+    FlagSetFunction flagSetFunction = new FlagSetFunction();
+    map.put(SkyFunctions.FLAG_SET, flagSetFunction);
     this.buildDriverFunction = buildDriverFunction;
 
     map.putAll(extraSkyFunctions);

src/main/java/com/google/devtools/build/lib/skyframe/config/BUILD

@@ -17,6 +17,7 @@ java_library(
         "BaselineOptionsFunction.java",
         "BuildConfigurationFunction.java",
         "BuildConfigurationKeyFunction.java",
+        "FlagSetFunction.java",
         "ParsedFlagsFunction.java",
         "PlatformMappingFunction.java",
     ],
@@ -43,6 +44,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:platform_options",
         "//src/main/java/com/google/devtools/build/lib/analysis/producers:build_configuration_key_producer",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
+        "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/packages",
         "//src/main/java/com/google/devtools/build/lib/packages/semantics",
         "//src/main/java/com/google/devtools/build/lib/pkgcache",
@@ -84,6 +86,7 @@ java_library(
     srcs = [
         "BuildConfigurationKey.java",
         "BuildConfigurationKeyValue.java",
+        "FlagSetValue.java",
         "NativeAndStarlarkFlags.java",
         "ParsedFlagsValue.java",
         "PlatformMappingValue.java",