From 00d74ff737cccd60305ee58d85313556a077152a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 21 Mar 2022 02:09:55 -0700 Subject: [PATCH] Support decompressing zstd tar archives for repository rules. This is mostly a rebase of https://github.com/bazelbuild/bazel/pull/11968 with a few tweaks of my own. Fixes https://github.com/bazelbuild/bazel/issues/10342. Co-authored-by: Grzegorz Lukasik RELNOTES[new]: `repository_ctx.extract`, and thus `http_archive`, can now decompress zstandard-compressed archives. Closes #15026. PiperOrigin-RevId: 436146779 --- .../devtools/build/lib/bazel/repository/BUILD | 1 + .../bazel/repository/DecompressorValue.java | 6 ++- .../lib/bazel/repository/TarZstFunction.java | 38 ++++++++++++++++++ .../repository/DecompressorValueTest.java | 4 ++ src/test/shell/bazel/BUILD | 5 ++- .../shell/bazel/external_integration_test.sh | 16 ++++++++ src/test/shell/bazel/testdata/BUILD | 8 ++-- .../bazel/testdata/zstd_test_archive.tar.zst | Bin 0 -> 135 bytes 8 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/repository/TarZstFunction.java create mode 100644 src/test/shell/bazel/testdata/zstd_test_archive.tar.zst diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD index a663cbb48c9879..b9e0e97309750d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD @@ -54,6 +54,7 @@ java_library( "//third_party:java-diff-utils", "//third_party:jsr305", "//third_party:xz", + "@zstd-jni", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java index 760cfc4a4d0908..49ff20b73b3498 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java @@ -103,13 +103,15 @@ static Decompressor getDecompressor(Path archivePath) return TarGzFunction.INSTANCE; } else if (baseName.endsWith(".tar.xz") || baseName.endsWith(".txz")) { return TarXzFunction.INSTANCE; + } else if (baseName.endsWith(".tar.zst") || baseName.endsWith(".tzst")) { + return TarZstFunction.INSTANCE; } else if (baseName.endsWith(".tar.bz2")) { return TarBz2Function.INSTANCE; } else { throw new RepositoryFunctionException( Starlark.errorf( - "Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz, " - + "or .tar.bz2 suffix (got %s)", + "Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz," + + " .tar.zst, .tzst, or .tar.bz2 suffix (got %s)", archivePath), Transience.PERSISTENT); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/TarZstFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarZstFunction.java new file mode 100644 index 00000000000000..bb0e4a4c5afa6c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarZstFunction.java @@ -0,0 +1,38 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.github.luben.zstd.ZstdInputStreamNoFinalizer; +import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor; +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** Creates a repository by unarchiving a zstandard-compressed .tar file. */ +final class TarZstFunction extends CompressedTarFunction { + static final Decompressor INSTANCE = new TarZstFunction(); + private static final int BUFFER_SIZE = 32 * 1024; + + private TarZstFunction() {} + + @Override + protected InputStream getDecompressorStream(DecompressorDescriptor descriptor) + throws IOException { + return new ZstdInputStreamNoFinalizer( + new BufferedInputStream( + new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE)); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java index b35588470d2dba..032da6fa321f39 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java @@ -50,6 +50,10 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception { DecompressorDescriptor.builder().setArchivePath(path).build(); path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.txz"); DecompressorDescriptor.builder().setArchivePath(path).build(); + path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.zst"); + DecompressorDescriptor.builder().setArchivePath(path).build(); + path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tzst"); + DecompressorDescriptor.builder().setArchivePath(path).build(); path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2"); DecompressorDescriptor.builder().setArchivePath(path).build(); } diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 6484ef0fa885bb..79fdfb5693f099 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -657,7 +657,10 @@ sh_test( name = "external_integration_test", size = "large", srcs = ["external_integration_test.sh"], - data = [":test-deps"], + data = [ + ":test-deps", + "//src/test/shell/bazel/testdata:zstd_test_archive.tar.zst", + ], shard_count = 15, tags = [ "no_windows", diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh index 59919684364954..8f4835e8d900a1 100755 --- a/src/test/shell/bazel/external_integration_test.sh +++ b/src/test/shell/bazel/external_integration_test.sh @@ -205,6 +205,22 @@ function test_http_archive_tar_xz() { http_archive_helper tar_xz_up } +function test_http_archive_tar_zstd() { + cat >> $(create_workspace_with_default_repos WORKSPACE) <> $(create_workspace_with_default_repos WORKSPACE) <wJ-f-04L1@08+&f7O*D;X^(wx*+$Y