Skip to content

Commit

Permalink
[remote/downloader] Don't include headers in FetchBlobRequest
Browse files Browse the repository at this point in the history 
Including the headers in the request is very inefficient as credentials should never change the content of the downloaded archive. In fact, given that Bazel verifies the checksum of the downloaded file, the credentials cannot possibly used in a way where they influence the outcome of the download (other than deciding whether or not the user is allowed to download the blob at all). Hence, the credentials should not be included in the request.

Users that need to send credentials to the remote downloader should do so by passing the credentials as metadata to the gRPC call.

Note that the remote downloader is behind an experimental flag, so this change does not need to go thorugh the incompatible change process.

Closes #16595.

PiperOrigin-RevId: 485576157
Change-Id: I8afc7c818e4eed63ac1f70c3e4c2115a1d365830
  • Loading branch information
@Yannic @copybara-github
Yannic authored and copybara-github committed Nov 2, 2022
1 parent 9d250ed commit 9296068
Show file tree
Hide file tree
Showing 3 changed files with 3 additions and 141 deletions.
1 change: 0 additions & 1 deletion src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/remote/util",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
"//third_party/grpc-java:grpc-jar",
Expand Down
65 changes: 2 additions & 63 deletions src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@
import build.bazel.remote.execution.v2.RequestMetadata;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
import com.google.devtools.build.lib.bazel.repository.downloader.HashOutputStream;
Expand All @@ -38,9 +36,6 @@
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
import com.google.devtools.build.lib.remote.util.Utils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import io.grpc.CallCredentials;
import io.grpc.Channel;
import io.grpc.StatusRuntimeException;
Expand All @@ -51,7 +46,6 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
Expand Down Expand Up @@ -82,7 +76,6 @@ public class GrpcRemoteDownloader implements AutoCloseable, Downloader {
// supported by Bazel.
private static final String QUALIFIER_CHECKSUM_SRI = "checksum.sri";
private static final String QUALIFIER_CANONICAL_ID = "bazel.canonical_id";
private static final String QUALIFIER_AUTH_HEADERS = "bazel.auth_headers";

public GrpcRemoteDownloader(
String buildRequestId,
Expand Down Expand Up @@ -131,13 +124,7 @@ public void download(
RemoteActionExecutionContext.create(metadata);

final FetchBlobRequest request =
newFetchBlobRequest(
options.remoteInstanceName,
urls,
authHeaders,
checksum,
canonicalId,
options.remoteDownloaderSendAllHeaders);
newFetchBlobRequest(options.remoteInstanceName, urls, checksum, canonicalId);
try {
FetchBlobResponse response =
retrier.execute(
Expand Down Expand Up @@ -175,10 +162,8 @@ public void download(
static FetchBlobRequest newFetchBlobRequest(
String instanceName,
List<URL> urls,
Map<URI, Map<String, List<String>>> authHeaders,
com.google.common.base.Optional<Checksum> checksum,
String canonicalId,
boolean includeAllHeaders) {
String canonicalId) {
FetchBlobRequest.Builder requestBuilder =
FetchBlobRequest.newBuilder().setInstanceName(instanceName);
for (URL url : urls) {
Expand All @@ -195,13 +180,6 @@ static FetchBlobRequest newFetchBlobRequest(
requestBuilder.addQualifiers(
Qualifier.newBuilder().setName(QUALIFIER_CANONICAL_ID).setValue(canonicalId).build());
}
if (!authHeaders.isEmpty()) {
requestBuilder.addQualifiers(
Qualifier.newBuilder()
.setName(QUALIFIER_AUTH_HEADERS)
.setValue(authHeadersJson(urls, authHeaders, includeAllHeaders))
.build());
}

return requestBuilder.build();
}
Expand All @@ -224,43 +202,4 @@ private OutputStream newOutputStream(
}
return out;
}

private static String authHeadersJson(
List<URL> urls, Map<URI, Map<String, List<String>>> authHeaders, boolean includeAllHeaders) {
ImmutableSet<String> hostSet =
urls.stream().map(URL::getHost).collect(ImmutableSet.toImmutableSet());
Map<String, JsonObject> subObjects = new TreeMap<>();
for (Map.Entry<URI, Map<String, List<String>>> entry : authHeaders.entrySet()) {
URI uri = entry.getKey();
// Only add headers that are relevant to the hosts.
if (!hostSet.contains(uri.getHost())) {
continue;
}

JsonObject subObject = new JsonObject();
Map<String, List<String>> orderedHeaders = new TreeMap<>(entry.getValue());
for (Map.Entry<String, List<String>> subEntry : orderedHeaders.entrySet()) {
if (includeAllHeaders) {
JsonArray values = new JsonArray(subEntry.getValue().size());
for (String value : subEntry.getValue()) {
values.add(value);
}
subObject.add(subEntry.getKey(), values);
} else {
String value = Iterables.getFirst(subEntry.getValue(), null);
if (value != null) {
subObject.addProperty(subEntry.getKey(), value);
}
}
}
subObjects.put(uri.toString(), subObject);
}

JsonObject authHeadersJson = new JsonObject();
for (Map.Entry<String, JsonObject> entry : subObjects.entrySet()) {
authHeadersJson.add(entry.getKey(), entry.getValue());
}

return new Gson().toJson(authHeadersJson);
}
}
78 changes: 1 addition & 77 deletions src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,82 +350,10 @@ public void testFetchBlobRequest() throws Exception {
new URL("http://example.com/a"),
new URL("http://example.com/b"),
new URL("file:/not/limited/to/http")),
ImmutableMap.of(
new URI("http://example.com"),
ImmutableMap.of(
"Some-Header", ImmutableList.of("some header content"),
"Another-Header",
ImmutableList.of("another header content", "even more header content")),
new URI("http://example.org"),
ImmutableMap.of(
"Org-Header",
ImmutableList.of("org header content", "and a second one", "and a third one"))),
com.google.common.base.Optional.<Checksum>of(
Checksum.fromSubresourceIntegrity(
"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")),
"canonical ID",
/* includeAllHeaders= */ false);

final String expectedAuthHeadersJson =
"{"
+ "\"http://example.com\":{"
+ "\"Another-Header\":\"another header content\","
+ "\"Some-Header\":\"some header content\""
+ "}"
+ "}";

assertThat(request)
.isEqualTo(
FetchBlobRequest.newBuilder()
.setInstanceName("instance name")
.addUris("http://example.com/a")
.addUris("http://example.com/b")
.addUris("file:/not/limited/to/http")
.addQualifiers(
Qualifier.newBuilder()
.setName("checksum.sri")
.setValue("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="))
.addQualifiers(
Qualifier.newBuilder().setName("bazel.canonical_id").setValue("canonical ID"))
.addQualifiers(
Qualifier.newBuilder()
.setName("bazel.auth_headers")
.setValue(expectedAuthHeadersJson))
.build());
}

@Test
public void testFetchBlobRequestWithAllHeaders() throws Exception {
FetchBlobRequest request =
GrpcRemoteDownloader.newFetchBlobRequest(
"instance name",
ImmutableList.of(
new URL("http://example.com/a"),
new URL("http://example.com/b"),
new URL("file:/not/limited/to/http")),
ImmutableMap.of(
new URI("http://example.com"),
ImmutableMap.of(
"Some-Header", ImmutableList.of("some header content"),
"Another-Header",
ImmutableList.of("another header content", "even more header content")),
new URI("http://example.org"),
ImmutableMap.of(
"Org-Header",
ImmutableList.of("org header content", "and a second one", "and a third one"))),
com.google.common.base.Optional.<Checksum>of(
Checksum.fromSubresourceIntegrity(
"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")),
"canonical ID",
/* includeAllHeaders= */ true);

final String expectedAuthHeadersJson =
"{"
+ "\"http://example.com\":{"
+ "\"Another-Header\":[\"another header content\",\"even more header content\"],"
+ "\"Some-Header\":[\"some header content\"]"
+ "}"
+ "}";
"canonical ID");

assertThat(request)
.isEqualTo(
Expand All @@ -440,10 +368,6 @@ public void testFetchBlobRequestWithAllHeaders() throws Exception {
.setValue("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="))
.addQualifiers(
Qualifier.newBuilder().setName("bazel.canonical_id").setValue("canonical ID"))
.addQualifiers(
Qualifier.newBuilder()
.setName("bazel.auth_headers")
.setValue(expectedAuthHeadersJson))
.build());
}
}

0 comments on commit 9296068

Please sign in to comment.