Skip to content

Commit

Permalink
Add a new abstraction for build output groups.
Browse files Browse the repository at this point in the history
Use an enum with values for each output group that the `build_dependencies.bzl` aspect can produce. Use the enum values both when requesting output groups in the build invocation, and when handing them after the fact. Add a new `OutputGroupArtifacts` class to do this.i

Derive the set of output groups required from the language classes of the build targets we're building in `DependencyTracker`.

This makes adding new output groups easier, whcih will help when adding C++ support as that will require more than one new output group from the build.

PiperOrigin-RevId: 569116586
  • Loading branch information
Googler authored and copybara-github committed Sep 28, 2023
1 parent 85e1a12 commit 3fd5d6e
Show file tree
Hide file tree
Showing 11 changed files with 205 additions and 96 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import static java.util.stream.Collectors.joining;

import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.devtools.intellij.qsync.ArtifactTrackerData.BuildArtifacts;
Expand All @@ -36,10 +36,7 @@
import com.google.idea.blaze.base.command.BlazeInvocationContext;
import com.google.idea.blaze.base.command.buildresult.BlazeArtifact;
import com.google.idea.blaze.base.command.buildresult.BuildResultHelper;
import com.google.idea.blaze.base.command.buildresult.LocalFileOutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.RemoteOutputArtifact;
import com.google.idea.blaze.base.model.primitives.WorkspacePath;
import com.google.idea.blaze.base.model.primitives.WorkspaceRoot;
import com.google.idea.blaze.base.projectview.ProjectViewManager;
import com.google.idea.blaze.base.projectview.ProjectViewSet;
Expand All @@ -49,18 +46,19 @@
import com.google.idea.blaze.exception.BuildException;
import com.google.idea.blaze.qsync.BlazeQueryParser;
import com.google.idea.blaze.qsync.project.ProjectDefinition;
import com.google.idea.blaze.qsync.project.ProjectDefinition.LanguageClass;
import com.google.protobuf.TextFormat;
import com.intellij.ide.plugins.PluginManager;
import com.intellij.openapi.extensions.PluginDescriptor;
import com.intellij.openapi.project.Project;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Collection;
import java.util.List;
import java.util.Set;

Expand All @@ -86,8 +84,19 @@ public BazelDependencyBuilder(
this.handledRuleKinds = handledRuleKinds;
}

private static final ImmutableMultimap<LanguageClass, OutputGroup> OUTPUT_GROUPS_BY_LANGUAGE =
ImmutableMultimap.<LanguageClass, OutputGroup>builder()
.putAll(
LanguageClass.JAVA,
OutputGroup.JARS,
OutputGroup.AARS,
OutputGroup.GENSRCS,
OutputGroup.ARTIFACT_INFO_FILE)
.build();

@Override
public OutputInfo build(BlazeContext context, Set<Label> buildTargets)
public OutputInfo build(
BlazeContext context, Set<Label> buildTargets, Set<LanguageClass> languages)
throws IOException, BuildException {
BuildInvoker invoker = buildSystem.getDefaultInvoker(project, context);
try (BuildResultHelper buildResultHelper = invoker.createBuildResultHelper()) {
Expand All @@ -104,6 +113,12 @@ public OutputInfo build(BlazeContext context, Set<Label> buildTargets)
Sets.difference(BlazeQueryParser.ALWAYS_BUILD_RULE_KINDS, handledRuleKinds);
String alwaysBuildParam = Joiner.on(",").join(ruleKindsToBuild);

ImmutableSet<OutputGroup> outputGroups =
languages.stream()
.map(OUTPUT_GROUPS_BY_LANGUAGE::get)
.flatMap(Collection::stream)
.collect(ImmutableSet.toImmutableSet());

ProjectViewSet projectViewSet = ProjectViewManager.getInstance(project).getProjectViewSet();
// TODO This is not SYNC_CONTEXT, but also not OTHER_CONTEXT, we need to decide what kind
// of flags need to be passed here.
Expand All @@ -129,12 +144,9 @@ public OutputInfo build(BlazeContext context, Set<Label> buildTargets)
.addBlazeFlags(
String.format("--aspects_parameters=always_build_rules=%s", alwaysBuildParam))
.addBlazeFlags("--aspects_parameters=generate_aidl_classes=True")
.addBlazeFlags("--output_groups=qsync_jars")
.addBlazeFlags("--output_groups=qsync_aars")
.addBlazeFlags("--output_groups=qsync_gensrcs")
.addBlazeFlags("--output_groups=artifact_info_file")
.addBlazeFlags("--noexperimental_run_validations")
.addBlazeFlags("--keep_going");
outputGroups.stream().map(OutputGroup::asBuildFlag).forEach(builder::addBlazeFlags);

BlazeBuildOutputs outputs =
invoker.getCommandRunner().run(project, builder, buildResultHelper, context);
Expand All @@ -144,7 +156,7 @@ public OutputInfo build(BlazeContext context, Set<Label> buildTargets)
ThrowOption.ALLOW_PARTIAL_SUCCESS,
ThrowOption.ALLOW_BUILD_FAILURE);

return createOutputInfo(outputs);
return createOutputInfo(outputs, outputGroups);
}
}

Expand All @@ -168,56 +180,24 @@ protected String prepareAspect(BlazeContext context) throws IOException, BuildEx
return "//:.aswb.bzl";
}

private OutputInfo createOutputInfo(BlazeBuildOutputs blazeBuildOutputs) throws BuildException {
ImmutableList<OutputArtifact> jars =
translateOutputArtifacts(
blazeBuildOutputs.getOutputGroupArtifacts(s -> s.contains("qsync_jars")));
ImmutableList<OutputArtifact> aars =
translateOutputArtifacts(
blazeBuildOutputs.getOutputGroupArtifacts(s -> s.contains("qsync_aars")));
ImmutableList<OutputArtifact> generatedSources =
translateOutputArtifacts(
blazeBuildOutputs.getOutputGroupArtifacts(s -> s.contains("qsync_gensrcs")));
ImmutableList<OutputArtifact> artifactInfoFiles =
translateOutputArtifacts(
blazeBuildOutputs.getOutputGroupArtifacts(s -> s.contains("artifact_info_file")));
private OutputInfo createOutputInfo(
BlazeBuildOutputs blazeBuildOutputs, Set<OutputGroup> outputGroups) throws BuildException {
OutputGroupArtifacts allArtifacts = new OutputGroupArtifacts(blazeBuildOutputs, outputGroups);
ImmutableSet.Builder<BuildArtifacts> artifactInfoFilesBuilder = ImmutableSet.builder();
for (OutputArtifact artifactInfoFile : artifactInfoFiles) {

for (OutputArtifact artifactInfoFile : allArtifacts.get(OutputGroup.ARTIFACT_INFO_FILE)) {
artifactInfoFilesBuilder.add(readArtifactInfoFile(artifactInfoFile));
}
return OutputInfo.create(
allArtifacts,
artifactInfoFilesBuilder.build(),
jars,
aars,
generatedSources,
blazeBuildOutputs.getTargetsWithErrors().stream()
.map(Object::toString)
.map(Label::of)
.collect(toImmutableSet()),
blazeBuildOutputs.buildResult.exitCode);
}

private ImmutableList<OutputArtifact> translateOutputArtifacts(
ImmutableList<OutputArtifact> artifacts) {
return artifacts.stream()
.map(BazelDependencyBuilder::translateOutputArtifact)
.collect(ImmutableList.toImmutableList());
}

private static OutputArtifact translateOutputArtifact(OutputArtifact it) {
if (!(it instanceof RemoteOutputArtifact)) {
return it;
}
RemoteOutputArtifact remoteOutputArtifact = (RemoteOutputArtifact) it;
String hashId = remoteOutputArtifact.getHashId();
if (!(hashId.startsWith("/google_src") || hashId.startsWith("/google/src"))) {
return it;
}
File srcfsArtifact = new File(hashId.replaceFirst("/google_src", "/google/src"));
return new LocalFileOutputArtifact(
srcfsArtifact, it.getRelativePath(), it.getConfigurationMnemonic(), it.getDigest());
}

private BuildArtifacts readArtifactInfoFile(BlazeArtifact file) throws BuildException {
try (InputStream inputStream = file.getInputStream()) {
BuildArtifacts.Builder builder = BuildArtifacts.newBuilder();
Expand All @@ -228,8 +208,4 @@ private BuildArtifacts readArtifactInfoFile(BlazeArtifact file) throws BuildExce
throw new BuildException(e);
}
}

private static String directoryToLabel(WorkspacePath directory) {
return String.format("//%s", directory);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@
import com.google.idea.blaze.base.scope.BlazeContext;
import com.google.idea.blaze.common.Label;
import com.google.idea.blaze.exception.BuildException;
import com.google.idea.blaze.qsync.project.ProjectDefinition.LanguageClass;
import java.io.IOException;
import java.util.Set;

/** A query sync service that knows how to build dependencies for given targets */
public interface DependencyBuilder {

OutputInfo build(BlazeContext context, Set<Label> buildTargets)
OutputInfo build(BlazeContext context, Set<Label> buildTargets, Set<LanguageClass> languages)
throws IOException, BuildException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
import com.google.idea.blaze.qsync.BlazeProject;
import com.google.idea.blaze.qsync.project.BlazeProjectSnapshot;
import com.google.idea.blaze.qsync.project.ProjectDefinition;
import com.google.idea.blaze.qsync.project.ProjectDefinition.LanguageClass;
import com.google.idea.blaze.qsync.project.ProjectTarget;
import com.google.idea.blaze.qsync.project.TargetTree;
import com.intellij.openapi.application.ex.ApplicationEx;
import com.intellij.openapi.application.ex.ApplicationManagerEx;
Expand Down Expand Up @@ -191,7 +193,13 @@ private void buildDependencies(
throws IOException, BuildException {
BuildDepsStatsScope.fromContext(context)
.ifPresent(stats -> stats.setBuildTargets(requestedTargets.buildTargets));
OutputInfo outputInfo = builder.build(context, requestedTargets.buildTargets);
ImmutableSet<LanguageClass> targetLanguages =
requestedTargets.buildTargets.stream()
.map(snapshot.graph().targetMap()::get)
.map(ProjectTarget::languages)
.reduce((a, b) -> Sets.union(a, b).immutableCopy())
.orElse(ImmutableSet.of());
OutputInfo outputInfo = builder.build(context, requestedTargets.buildTargets, targetLanguages);
reportErrorsAndWarnings(context, snapshot, outputInfo);

ImmutableSet<Path> updatedFiles =
Expand Down
38 changes: 38 additions & 0 deletions base/src/com/google/idea/blaze/base/qsync/OutputGroup.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright 2023 The Bazel Authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.idea.blaze.base.qsync;

/** Represents an output group produced by the {@code build_dependencies.bzl} aspect. */
public enum OutputGroup {
JARS("qsync_jars"),
AARS("qsync_aars"),
GENSRCS("qsync_gensrcs"),
ARTIFACT_INFO_FILE("artifact_info_file");

private final String name;

OutputGroup(String name) {
this.name = name;
}

public String outputGroupName() {
return name;
}

public String asBuildFlag() {
return "--output_groups=" + outputGroupName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Copyright 2023 The Bazel Authors. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.idea.blaze.base.qsync;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
import com.google.idea.blaze.base.command.buildresult.LocalFileOutputArtifact;
import com.google.idea.blaze.base.command.buildresult.OutputArtifact;
import com.google.idea.blaze.base.command.buildresult.RemoteOutputArtifact;
import com.google.idea.blaze.base.sync.aspects.BlazeBuildOutputs;
import java.io.File;
import java.util.Set;

/** Output artifacts from the build per {@link OutputGroup}. */
public class OutputGroupArtifacts {

public static final OutputGroupArtifacts EMPTY = new OutputGroupArtifacts();

private final ImmutableListMultimap<OutputGroup, OutputArtifact> outputArtifacts;

private OutputGroupArtifacts() {
outputArtifacts = ImmutableListMultimap.of();
}

@VisibleForTesting
public OutputGroupArtifacts(ImmutableListMultimap<OutputGroup, OutputArtifact> map) {
this.outputArtifacts = map;
}

public OutputGroupArtifacts(BlazeBuildOutputs buildOutputs, Set<OutputGroup> outputGroups) {
ImmutableListMultimap.Builder<OutputGroup, OutputArtifact> builder =
ImmutableListMultimap.builder();
for (OutputGroup group : outputGroups) {
ImmutableList<OutputArtifact> artifacts =
translateOutputArtifacts(
buildOutputs.getOutputGroupArtifacts(group.outputGroupName()::equals));
builder.putAll(group, artifacts);
}
outputArtifacts = builder.build();
}

public ImmutableList<OutputArtifact> get(OutputGroup group) {
return outputArtifacts.get(group);
}

public boolean isEmpty() {
return outputArtifacts.isEmpty();
}

private static ImmutableList<OutputArtifact> translateOutputArtifacts(
ImmutableList<OutputArtifact> artifacts) {
return artifacts.stream()
.map(OutputGroupArtifacts::translateOutputArtifact)
.collect(ImmutableList.toImmutableList());
}

private static OutputArtifact translateOutputArtifact(OutputArtifact it) {
if (!(it instanceof RemoteOutputArtifact)) {
return it;
}
RemoteOutputArtifact remoteOutputArtifact = (RemoteOutputArtifact) it;
String hashId = remoteOutputArtifact.getHashId();
if (!(hashId.startsWith("/google_src") || hashId.startsWith("/google/src"))) {
return it;
}
File srcfsArtifact = new File(hashId.replaceFirst("/google_src", "/google/src"));
return new LocalFileOutputArtifact(
srcfsArtifact, it.getRelativePath(), it.getConfigurationMnemonic(), it.getDigest());
}
}
Loading

0 comments on commit 3fd5d6e

Please sign in to comment.