From e5a73895068767e6af12767455bdc4ae67b60a6c Mon Sep 17 00:00:00 2001 From: Yannic Bonenberger Date: Thu, 27 Oct 2022 23:21:40 +0200 Subject: [PATCH] [remote/downloader] Don't include headers in `FetchBlobRequest` Including the headers in the request is very inefficient as credentials should never change the content of the downloaded archive. In fact, given that Bazel verifies the checksum of the downloaded file, the credentials cannot possibly used in a way where they influence the outcome of the download (other than deciding whether or not the user is allowed to download the blob at all). Hence, the credentials should not be included in the request. Users that need to send credentials to the remote downloader should do so by passing the credentials as metadata to the gRPC call. Note that the remote downloader is behind an experimental flag, so this change does not need to go thorugh the incompatible change process. --- .../build/lib/remote/downloader/BUILD | 1 - .../downloader/GrpcRemoteDownloader.java | 63 +-------------- .../downloader/GrpcRemoteDownloaderTest.java | 78 +------------------ 3 files changed, 3 insertions(+), 139 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD b/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD index d980d2a6e0bc2c..1804656458feb5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/downloader/BUILD @@ -22,7 +22,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/remote/options", "//src/main/java/com/google/devtools/build/lib/remote/util", "//src/main/java/com/google/devtools/build/lib/vfs", - "//third_party:gson", "//third_party:guava", "//third_party:jsr305", "//third_party/grpc-java:grpc-jar", diff --git a/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java b/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java index d94efdc0c60899..8715bd05672a2d 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java @@ -23,8 +23,6 @@ import build.bazel.remote.execution.v2.RequestMetadata; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.Downloader; import com.google.devtools.build.lib.bazel.repository.downloader.HashOutputStream; @@ -38,9 +36,6 @@ import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; import com.google.devtools.build.lib.remote.util.Utils; import com.google.devtools.build.lib.vfs.Path; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.StatusRuntimeException; @@ -131,13 +126,7 @@ public void download( RemoteActionExecutionContext.create(metadata); final FetchBlobRequest request = - newFetchBlobRequest( - options.remoteInstanceName, - urls, - authHeaders, - checksum, - canonicalId, - options.remoteDownloaderSendAllHeaders); + newFetchBlobRequest(options.remoteInstanceName, urls, checksum, canonicalId); try { FetchBlobResponse response = retrier.execute( @@ -175,10 +164,8 @@ public void download( static FetchBlobRequest newFetchBlobRequest( String instanceName, List urls, - Map>> authHeaders, com.google.common.base.Optional checksum, - String canonicalId, - boolean includeAllHeaders) { + String canonicalId) { FetchBlobRequest.Builder requestBuilder = FetchBlobRequest.newBuilder().setInstanceName(instanceName); for (URL url : urls) { @@ -195,13 +182,6 @@ static FetchBlobRequest newFetchBlobRequest( requestBuilder.addQualifiers( Qualifier.newBuilder().setName(QUALIFIER_CANONICAL_ID).setValue(canonicalId).build()); } - if (!authHeaders.isEmpty()) { - requestBuilder.addQualifiers( - Qualifier.newBuilder() - .setName(QUALIFIER_AUTH_HEADERS) - .setValue(authHeadersJson(urls, authHeaders, includeAllHeaders)) - .build()); - } return requestBuilder.build(); } @@ -224,43 +204,4 @@ private OutputStream newOutputStream( } return out; } - - private static String authHeadersJson( - List urls, Map>> authHeaders, boolean includeAllHeaders) { - ImmutableSet hostSet = - urls.stream().map(URL::getHost).collect(ImmutableSet.toImmutableSet()); - Map subObjects = new TreeMap<>(); - for (Map.Entry>> entry : authHeaders.entrySet()) { - URI uri = entry.getKey(); - // Only add headers that are relevant to the hosts. - if (!hostSet.contains(uri.getHost())) { - continue; - } - - JsonObject subObject = new JsonObject(); - Map> orderedHeaders = new TreeMap<>(entry.getValue()); - for (Map.Entry> subEntry : orderedHeaders.entrySet()) { - if (includeAllHeaders) { - JsonArray values = new JsonArray(subEntry.getValue().size()); - for (String value : subEntry.getValue()) { - values.add(value); - } - subObject.add(subEntry.getKey(), values); - } else { - String value = Iterables.getFirst(subEntry.getValue(), null); - if (value != null) { - subObject.addProperty(subEntry.getKey(), value); - } - } - } - subObjects.put(uri.toString(), subObject); - } - - JsonObject authHeadersJson = new JsonObject(); - for (Map.Entry entry : subObjects.entrySet()) { - authHeadersJson.add(entry.getKey(), entry.getValue()); - } - - return new Gson().toJson(authHeadersJson); - } } diff --git a/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java index 9ca60e4710bbae..c08093d8b4f081 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java @@ -350,82 +350,10 @@ public void testFetchBlobRequest() throws Exception { new URL("http://example.com/a"), new URL("http://example.com/b"), new URL("file:/not/limited/to/http")), - ImmutableMap.of( - new URI("http://example.com"), - ImmutableMap.of( - "Some-Header", ImmutableList.of("some header content"), - "Another-Header", - ImmutableList.of("another header content", "even more header content")), - new URI("http://example.org"), - ImmutableMap.of( - "Org-Header", - ImmutableList.of("org header content", "and a second one", "and a third one"))), com.google.common.base.Optional.of( Checksum.fromSubresourceIntegrity( "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")), - "canonical ID", - /* includeAllHeaders= */ false); - - final String expectedAuthHeadersJson = - "{" - + "\"http://example.com\":{" - + "\"Another-Header\":\"another header content\"," - + "\"Some-Header\":\"some header content\"" - + "}" - + "}"; - - assertThat(request) - .isEqualTo( - FetchBlobRequest.newBuilder() - .setInstanceName("instance name") - .addUris("http://example.com/a") - .addUris("http://example.com/b") - .addUris("file:/not/limited/to/http") - .addQualifiers( - Qualifier.newBuilder() - .setName("checksum.sri") - .setValue("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")) - .addQualifiers( - Qualifier.newBuilder().setName("bazel.canonical_id").setValue("canonical ID")) - .addQualifiers( - Qualifier.newBuilder() - .setName("bazel.auth_headers") - .setValue(expectedAuthHeadersJson)) - .build()); - } - - @Test - public void testFetchBlobRequestWithAllHeaders() throws Exception { - FetchBlobRequest request = - GrpcRemoteDownloader.newFetchBlobRequest( - "instance name", - ImmutableList.of( - new URL("http://example.com/a"), - new URL("http://example.com/b"), - new URL("file:/not/limited/to/http")), - ImmutableMap.of( - new URI("http://example.com"), - ImmutableMap.of( - "Some-Header", ImmutableList.of("some header content"), - "Another-Header", - ImmutableList.of("another header content", "even more header content")), - new URI("http://example.org"), - ImmutableMap.of( - "Org-Header", - ImmutableList.of("org header content", "and a second one", "and a third one"))), - com.google.common.base.Optional.of( - Checksum.fromSubresourceIntegrity( - "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")), - "canonical ID", - /* includeAllHeaders= */ true); - - final String expectedAuthHeadersJson = - "{" - + "\"http://example.com\":{" - + "\"Another-Header\":[\"another header content\",\"even more header content\"]," - + "\"Some-Header\":[\"some header content\"]" - + "}" - + "}"; + "canonical ID"); assertThat(request) .isEqualTo( @@ -440,10 +368,6 @@ public void testFetchBlobRequestWithAllHeaders() throws Exception { .setValue("sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")) .addQualifiers( Qualifier.newBuilder().setName("bazel.canonical_id").setValue("canonical ID")) - .addQualifiers( - Qualifier.newBuilder() - .setName("bazel.auth_headers") - .setValue(expectedAuthHeadersJson)) .build()); } }