Skip to content

Commit

Permalink
Add basic incompatible target skipping
Browse files Browse the repository at this point in the history
This patch aims to implement a basic version of incompatible target
skipping outlined here:
https://docs.google.com/document/d/12n5QNHmFSkuh5yAbdEex64ot4hRgR-moL1zRimU7wHQ/edit?usp=sharing

The implementation in this patch supports only "top level" target
skipping. In other words we only filter out targets if they are
globbed as part of an invocation like "bazel build //...".

The following features are not yet implemented:
- Filtering targets based on transitive dependencies. For example, if
  //a:b depends on //c:d an invocation of "bazel build //a/..." will
  still build //c:d even if it is incompatible with the current
  platform.
- Being able to select() on constraint values. For example, it's not
  possible to select() based on whether or not GCC is being used.
  Discussion of this aspect of the proposal is ongoing:
  https://groups.google.com/d/msg/bazel-dev/xK7diubpljQ/s3KSRwTWAQAJ

The goal is to implement the missing features in follow-up patches.
  • Loading branch information
philsc committed Aug 31, 2020
1 parent 4be7554 commit e4df959
Show file tree
Hide file tree
Showing 20 changed files with 696 additions and 15 deletions.
12 changes: 12 additions & 0 deletions src/main/java/com/google/devtools/build/lib/analysis/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,7 @@ java_library(
":extra/extra_action_info_file_write_action",
":extra_action_artifacts_provider",
":file_provider",
":incompatible_platform_provider",
":inconsistent_aspect_order_exception",
":label_and_location",
":label_expander",
Expand Down Expand Up @@ -378,6 +379,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/actions:fileset_output_symlink",
"//src/main/java/com/google/devtools/build/lib/actions:localhost_capacity",
"//src/main/java/com/google/devtools/build/lib/analysis/platform",
"//src/main/java/com/google/devtools/build/lib/analysis/platform:utils",
"//src/main/java/com/google/devtools/build/lib/analysis/stringtemplate",
"//src/main/java/com/google/devtools/build/lib/bugreport",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
Expand Down Expand Up @@ -778,6 +780,15 @@ java_library(
],
)

java_library(
name = "incompatible_platform_provider",
srcs = ["IncompatiblePlatformProvider.java"],
deps = [
":transitive_info_provider",
"//src/main/java/com/google/devtools/build/lib/concurrent",
],
)

java_library(
name = "inconsistent_aspect_order_exception",
srcs = ["InconsistentAspectOrderException.java"],
Expand Down Expand Up @@ -1982,6 +1993,7 @@ java_library(
srcs = ["constraints/TopLevelConstraintSemantics.java"],
deps = [
":analysis_cluster",
":incompatible_platform_provider",
":config/build_configuration",
":configured_target",
":constraints/constraint_semantics",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.google.devtools.build.lib.analysis.config.HostTransition;
import com.google.devtools.build.lib.analysis.config.RunUnder;
import com.google.devtools.build.lib.analysis.constraints.ConstraintConstants;
import com.google.devtools.build.lib.analysis.platform.ConstraintValueInfo;
import com.google.devtools.build.lib.analysis.test.TestConfiguration;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.packages.Attribute;
Expand Down Expand Up @@ -327,6 +328,14 @@ public static RuleClass.Builder commonCoreAndStarlarkAttributes(RuleClass.Builde
.dontCheckConstraints()
.nonconfigurable(
"special logic for constraints and select: see ConstraintSemantics"))
/* <!-- #BLAZE_RULE(toolchain).ATTRIBUTE(target_compatible_with) -->
A list of <code>constraint_value</code>s that must be satisfied by the target platform in
order for this toolchain to be selected for a target building for that platform.
<!-- #END_BLAZE_RULE.ATTRIBUTE --> */
.add(
attr("target_compatible_with", LABEL_LIST)
.mandatoryProviders(ConstraintValueInfo.PROVIDER.id())
.allowedFileTypes(FileTypeSet.NO_FILE))
.add(
attr(RuleClass.CONFIG_SETTING_DEPS_ATTRIBUTE, LABEL_LIST)
.nonconfigurable("stores configurability keys"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,7 @@ public AnalysisResult update(
TargetPatternPhaseValue loadingResult,
BuildOptions targetOptions,
Set<String> multiCpu,
List<String> requestedTargetPatterns,
List<String> aspects,
AnalysisOptions viewOptions,
boolean keepGoing,
Expand Down Expand Up @@ -420,6 +421,22 @@ public AnalysisResult update(
skyframeBuildView.clearInvalidatedConfiguredTargets();
}

TopLevelConstraintSemantics topLevelConstraintSemantics =
new TopLevelConstraintSemantics(
skyframeExecutor.getPackageManager(),
input -> skyframeExecutor.getConfiguration(eventHandler, input),
eventHandler);

TopLevelConstraintSemantics.PlatformRestrictionsResult platformRestrictions =
topLevelConstraintSemantics.checkPlatformRestrictions(
skyframeAnalysisResult.getConfiguredTargets(), requestedTargetPatterns, keepGoing);

if (platformRestrictions.targetsWithErrors.size() > 0) {
skyframeAnalysisResult =
skyframeAnalysisResult.withAdditionalErroredTargets(
ImmutableSet.copyOf(platformRestrictions.targetsWithErrors));
}

int numTargetsToAnalyze = topLevelTargetsWithConfigs.size();
int numSuccessful = skyframeAnalysisResult.getConfiguredTargets().size();
if (0 < numSuccessful && numSuccessful < numTargetsToAnalyze) {
Expand All @@ -430,11 +447,11 @@ public AnalysisResult update(
}

Set<ConfiguredTarget> targetsToSkip =
new TopLevelConstraintSemantics(
skyframeExecutor.getPackageManager(),
input -> skyframeExecutor.getConfiguration(eventHandler, input),
eventHandler)
.checkTargetEnvironmentRestrictions(skyframeAnalysisResult.getConfiguredTargets());
ImmutableSet.copyOf(
Sets.union(
topLevelConstraintSemantics.checkTargetEnvironmentRestrictions(
skyframeAnalysisResult.getConfiguredTargets()),
platformRestrictions.targetsToSkip));

AnalysisResult result =
createResult(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
import com.google.devtools.build.lib.analysis.configuredtargets.OutputFileConfiguredTarget;
import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget;
import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
import com.google.devtools.build.lib.analysis.platform.ConstraintValueInfo;
import com.google.devtools.build.lib.analysis.platform.PlatformProviderUtils;
import com.google.devtools.build.lib.analysis.starlark.StarlarkRuleConfiguredTargetUtil;
import com.google.devtools.build.lib.analysis.test.AnalysisFailure;
import com.google.devtools.build.lib.analysis.test.AnalysisFailureInfo;
Expand Down Expand Up @@ -318,6 +320,12 @@ private ConfiguredTarget createRule(
prerequisiteMap.values()))
.build();

ConfiguredTarget incompatibleTarget =
incompatibleConfiguredTarget(ruleContext, prerequisiteMap);
if (incompatibleTarget != null) {
return incompatibleTarget;
}

List<NestedSet<AnalysisFailure>> analysisFailures = depAnalysisFailures(ruleContext);
if (!analysisFailures.isEmpty()) {
return erroredConfiguredTargetWithFailures(ruleContext, analysisFailures);
Expand Down Expand Up @@ -372,6 +380,89 @@ private ConfiguredTarget createRule(
}
}

// TODO(phil): Document this.
// TODO(phil): Move this into ConstraintSemantics somehow.
@Nullable
private ConfiguredTarget incompatibleConfiguredTarget(
RuleContext ruleContext,
OrderedSetMultimap<DependencyKind, ConfiguredTargetAndData> prerequisiteMap)
throws ActionConflictException {
if (!"toolchain".equals(ruleContext.getRule().getRuleClass())) {
boolean isIncompatible = false;

// This is incompatible if one of the dependencies is as well.
for (ConfiguredTargetAndData infoCollection : prerequisiteMap.values()) {
ConfiguredTarget dependency = infoCollection.getConfiguredTarget();
if (dependency instanceof OutputFileConfiguredTarget) {
// For generated files, we want to query the generating rule for providers. genrule() for
// example doesn't attach providers like IncompatiblePlatformProvider to its outputs.
dependency = ((OutputFileConfiguredTarget)dependency).getGeneratingRule();
}
if (dependency.getProvider(IncompatiblePlatformProvider.class) != null) {
isIncompatible = true;
}
}

// This is incompatible if explicitly specified to be.
if (!isIncompatible && ruleContext.attributes().has("target_compatible_with")) {
Iterable<ConstraintValueInfo> constraintValues =
PlatformProviderUtils.constraintValues(
ruleContext.getPrerequisites("target_compatible_with", TransitionMode.DONT_CHECK));
for (ConstraintValueInfo constraintValue : constraintValues) {
if (!ruleContext.targetPlatformHasConstraint(constraintValue)) {
isIncompatible = true;
}
}
}

if (isIncompatible) {
// Create a dummy ConfiguredTarget that has the IncompatiblePlatformProvider set.
ImmutableList<Artifact> outputArtifacts = ruleContext.getOutputArtifacts();

if (ruleContext.isTestTarget() && outputArtifacts.size() == 0) {
// TODO(philsc): Figure out how to avoid this. Right now some test targets require the
// RunfilesSupport to be added to the RuleConfiguredTargetBuilder. E.g. sh_test requires
// this while cc_test does not.
outputArtifacts = ImmutableList.of(ruleContext.createOutputArtifact());
}

NestedSet<Artifact> filesToBuild =
NestedSetBuilder.<Artifact>stableOrder().addAll(outputArtifacts).build();

Runfiles.Builder runfilesBuilder =
new Runfiles.Builder(
ruleContext.getWorkspaceName(),
ruleContext.getConfiguration().legacyExternalRunfiles());
Runfiles runfiles =
runfilesBuilder
.addTransitiveArtifacts(filesToBuild)
.addRunfiles(ruleContext, RunfilesProvider.DEFAULT_RUNFILES)
.build();

RuleConfiguredTargetBuilder builder = new RuleConfiguredTargetBuilder(ruleContext);
builder.setFilesToBuild(filesToBuild);
builder.add(IncompatiblePlatformProvider.class, IncompatiblePlatformProvider.simple());
builder.add(RunfilesProvider.class, RunfilesProvider.simple(runfiles));
if (outputArtifacts.size() > 0) {
Artifact executable = outputArtifacts.get(0);
RunfilesSupport runfilesSupport =
RunfilesSupport.withExecutable(ruleContext, runfiles, executable);
builder.setRunfilesSupport(runfilesSupport, executable);

ruleContext.registerAction(
new FailAction(
ruleContext.getActionOwner(),
outputArtifacts,
"Can't build this. This target is incompatible."));
}
return builder.build();
}
}

// No incompatibilities found.
return null;
}

private List<NestedSet<AnalysisFailure>> depAnalysisFailures(RuleContext ruleContext) {
if (ruleContext.getConfiguration().allowAnalysisFailures()) {
ImmutableList.Builder<NestedSet<AnalysisFailure>> analysisFailures = ImmutableList.builder();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.analysis;

import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;

// TODO(phil): Document this.
@Immutable
public final class IncompatiblePlatformProvider implements TransitiveInfoProvider {
IncompatiblePlatformProvider() {
// Nothing to do.
}

public static IncompatiblePlatformProvider simple() {
return new IncompatiblePlatformProvider();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.IncompatiblePlatformProvider;
import com.google.devtools.build.lib.analysis.TransitiveInfoCollection;
import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
Expand Down Expand Up @@ -57,6 +58,19 @@ public class TopLevelConstraintSemantics {
private final Function<Key, BuildConfiguration> configurationProvider;
private final ExtendedEventHandler eventHandler;

// TODO(phil): Document this.
public class PlatformRestrictionsResult {
public ImmutableSet<ConfiguredTarget> targetsToSkip;
public ImmutableSet<ConfiguredTarget> targetsWithErrors;

public PlatformRestrictionsResult(
ImmutableSet<ConfiguredTarget> targetsToSkip,
ImmutableSet<ConfiguredTarget> targetsWithErrors) {
this.targetsToSkip = targetsToSkip;
this.targetsWithErrors = targetsWithErrors;
}
};

/**
* Constructor with helper classes for loading targets.
*
Expand Down Expand Up @@ -85,6 +99,42 @@ private MissingEnvironment(Label environment, RemovedEnvironmentCulprit culprit)
}
}

// TODO(phil): Document this.
public PlatformRestrictionsResult checkPlatformRestrictions(
ImmutableList<ConfiguredTarget> topLevelTargets,
List<String> requestedTargetPatterns,
boolean keepGoing)
throws ViewCreationFailedException {
ImmutableSet.Builder<ConfiguredTarget> incompatibleTargets = ImmutableSet.builder();
ImmutableSet.Builder<ConfiguredTarget> incompatibleButRequestedTargets = ImmutableSet.builder();

for (ConfiguredTarget target : topLevelTargets) {
if (target.getProvider(IncompatiblePlatformProvider.class) != null) {
String labelString = target.getLabel().toString();
if (requestedTargetPatterns.contains(labelString)) {
if (!keepGoing) {
throw new ViewCreationFailedException(
"Target "
+ labelString
+ " is incompatible and cannot be built, but was explicitly requested.");
}
this.eventHandler.handle(
Event.warn(
"Incompatible target "
+ labelString
+ " specifically requested: it will not be built"));
incompatibleButRequestedTargets.add(target);
} else {
incompatibleTargets.add(target);
}
}
}

return new PlatformRestrictionsResult(
ImmutableSet.copyOf(incompatibleTargets.build()),
ImmutableSet.copyOf(incompatibleButRequestedTargets.build()));
}

/**
* Checks that if this is an environment-restricted build, all top-level targets support expected
* top-level environments. Expected top-level environments can be declared explicitly through
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -543,6 +543,7 @@ enum TestStatus {
REMOTE_FAILURE = 6;
FAILED_TO_BUILD = 7;
TOOL_HALTED_BEFORE_TESTING = 8;
SKIPPED = 9;
}

// Payload on events reporting about individual test action.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ private static AnalysisResult runAnalysisPhase(
loadingResult,
targetOptions,
multiCpu,
request.getTargets(),
request.getAspects(),
request.getViewOptions(),
request.getKeepGoing(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,16 +86,22 @@ void showBuildResult(
if (targetsToPrint.size() > request.getBuildOptions().maxResultTargets) {
return;
}
// Filter the targets we care about into two buckets:
// Filter the targets we care about into three buckets:
Collection<ConfiguredTarget> succeeded = new ArrayList<>();
Collection<ConfiguredTarget> failed = new ArrayList<>();
Collection<ConfiguredTarget> skipped = new ArrayList<>();
Collection<ConfiguredTarget> successfulTargets = result.getSuccessfulTargets();
for (ConfiguredTarget target : targetsToPrint) {
(successfulTargets.contains(target) ? succeeded : failed).add(target);
if (configuredTargetsToSkip.contains(target)) {
skipped.add(target);
} else {
(successfulTargets.contains(target) ? succeeded : failed).add(target);
}
}

// TODO(bazel-team): convert these to a new "SKIPPED" status when ready: b/62191890.
failed.addAll(configuredTargetsToSkip);
for (ConfiguredTarget target : skipped) {
outErr.printErr("Target " + target.getLabel() + " was skipped\n");
}

TopLevelArtifactContext context = request.getTopLevelArtifactContext();
for (ConfiguredTarget target : succeeded) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,8 @@ public RuleErrorException(String message, Throwable cause) {
*/
public static final String COMPATIBLE_ENVIRONMENT_ATTR = "compatible_with";

public static final String TARGET_RESTRICTED_TO_ATTR = "target_compatible_with";

/**
* For Bazel's constraint system: the implicit attribute used to store rule class restriction
* defaults as specified by {@link Builder#restrictedTo}.
Expand Down Expand Up @@ -1411,6 +1413,7 @@ public <TYPE> Builder exemptFromConstraintChecking(String reason) {
this.supportsConstraintChecking = false;
attributes.remove(RuleClass.COMPATIBLE_ENVIRONMENT_ATTR);
attributes.remove(RuleClass.RESTRICTED_ENVIRONMENT_ATTR);
attributes.remove(RuleClass.TARGET_RESTRICTED_TO_ATTR);
return this;
}

Expand Down
Loading

0 comments on commit e4df959

Please sign in to comment.