From fefd9596d7997c197b26c5f1ea31e9774ad1732e Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Thu, 8 Aug 2019 14:07:36 -0400 Subject: [PATCH 01/16] Progress --- .../jib/builder/steps/ExtractTarStep.java | 68 +++++++++++++++++++ .../jib/builder/steps/SaveDockerStep.java | 45 ++++++++++++ .../tools/jib/builder/steps/StepsRunner.java | 14 ++++ .../google/cloud/tools/jib/cache/Cache.java | 11 +++ 4 files changed, 138 insertions(+) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java new file mode 100644 index 0000000000..dc2f839ffa --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; +import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate; +import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate; +import com.google.cloud.tools.jib.image.json.JsonToImageTranslator; +import com.google.cloud.tools.jib.json.JsonTemplateMapper; +import com.google.cloud.tools.jib.tar.TarExtractor; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.Callable; + +public class ExtractTarStep implements Callable { + + static class LocalImage { + Image baseImage; + List layers; + + LocalImage(Image baseImage, List layers) { + this.baseImage = baseImage; + this.layers = layers; + } + } + + private final Path tarPath; + private final BuildConfiguration buildConfiguration; + + ExtractTarStep(Path tarPath, BuildConfiguration buildConfiguration) { + this.tarPath = tarPath; + this.buildConfiguration = buildConfiguration; + } + + @Override + public LocalImage call() throws IOException { + Path destination = buildConfiguration.getBaseImageLayersCache().getTemporaryDirectory(); + TarExtractor.extract(tarPath, destination); + DockerManifestEntryTemplate manifest = + JsonTemplateMapper.readJsonFromFile( + destination.resolve("manifest.json"), DockerManifestEntryTemplate.class); + ContainerConfigurationTemplate configuration = + JsonTemplateMapper.readJsonFromFile( + destination.resolve(manifest.getConfig()), ContainerConfigurationTemplate.class); + + Image image = JsonToImageTranslator.toImage(manifest, configuration); + List layers; + + return new LocalImage(image, layers); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java new file mode 100644 index 0000000000..c43d6f09b2 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java @@ -0,0 +1,45 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.ImageReference; +import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.docker.DockerClient; +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.Callable; + +/** Saves an image from the docker daemon. */ +public class SaveDockerStep implements Callable { + + private final BuildConfiguration buildConfiguration; + private final DockerClient dockerClient; + + SaveDockerStep(BuildConfiguration buildConfiguration, DockerClient dockerClient) { + this.buildConfiguration = buildConfiguration; + this.dockerClient = dockerClient; + } + + @Override + public Path call() throws IOException, InterruptedException { + Path outputPath = + buildConfiguration.getBaseImageLayersCache().getTemporaryDirectory().resolve("out.tar"); + ImageReference imageReference = buildConfiguration.getBaseImageConfiguration().getImage(); + dockerClient.save(imageReference, outputPath); + return outputPath; + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 1a73e959c2..1509e5189c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -19,6 +19,7 @@ import com.google.cloud.tools.jib.api.Credential; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; @@ -59,6 +60,7 @@ private static Future failedFuture() { new IllegalStateException("invalid usage; required step not configured")); } + private Future dockerSaveTar = failedFuture(); private Future baseImageAndAuth = failedFuture(); private Future>> baseImageLayers = failedFuture(); @Nullable private List> applicationLayers; @@ -184,6 +186,18 @@ public BuildResult run() throws ExecutionException, InterruptedException { } } + private void saveDocker() { + DockerClient dockerClient = new DockerClient(); + results.dockerSaveTar = + executorService.submit(() -> new SaveDockerStep(buildConfiguration, dockerClient).call()); + } + + private void extractTar(Future extractionPath) { + Future localImage = + executorService.submit(() -> new ExtractTarStep(extractionPath.get(), buildConfiguration).call()); + + } + private void retrieveTargetRegistryCredentials() { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java index 6a5c946eed..25dd873478 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java @@ -53,10 +53,12 @@ public static Cache withDirectory(Path cacheDirectory) throws IOException { private final CacheStorageWriter cacheStorageWriter; private final CacheStorageReader cacheStorageReader; + private final Path temporaryDirectory; private Cache(CacheStorageFiles cacheStorageFiles) { this.cacheStorageWriter = new CacheStorageWriter(cacheStorageFiles); this.cacheStorageReader = new CacheStorageReader(cacheStorageFiles); + this.temporaryDirectory = cacheStorageFiles.getTemporaryDirectory(); } /** @@ -160,4 +162,13 @@ public Optional retrieve(DescriptorDigest layerDigest) throws IOException, CacheCorruptedException { return cacheStorageReader.retrieve(layerDigest); } + + /** + * Returns the cache's temporary directory. + * + * @return the cache's temporary directory + */ + public Path getTemporaryDirectory() { + return temporaryDirectory; + } } From 6c12fd9ce8be9e53fe6423e9d74f7fbf96b83f62 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Mon, 12 Aug 2019 15:30:22 -0400 Subject: [PATCH 02/16] Progress --- .../jib/builder/steps/ExtractTarStep.java | 101 +++++++++++++++++- .../tools/jib/builder/steps/StepsRunner.java | 13 --- .../cloud/tools/jib/cache/CachedLayer.java | 14 +-- .../json/ContainerConfigurationTemplate.java | 8 +- 4 files changed, 108 insertions(+), 28 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index dc2f839ffa..3c7ffab35a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -16,19 +16,35 @@ package com.google.cloud.tools.jib.builder.steps; +import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; +import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate; import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.image.LayerCountMismatchException; +import com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException; import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate; import com.google.cloud.tools.jib.image.json.JsonToImageTranslator; +import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.tar.TarExtractor; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.io.ByteStreams; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +/** Extracts a tar file base image. */ public class ExtractTarStep implements Callable { static class LocalImage { @@ -49,20 +65,95 @@ static class LocalImage { this.buildConfiguration = buildConfiguration; } + // TODO: Future<> stuff @Override - public LocalImage call() throws IOException { + public LocalImage call() + throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException { + // TODO: Use a non-overlapping temp directory for better concurrency Path destination = buildConfiguration.getBaseImageLayersCache().getTemporaryDirectory(); TarExtractor.extract(tarPath, destination); - DockerManifestEntryTemplate manifest = + DockerManifestEntryTemplate loadManifest = JsonTemplateMapper.readJsonFromFile( destination.resolve("manifest.json"), DockerManifestEntryTemplate.class); ContainerConfigurationTemplate configuration = JsonTemplateMapper.readJsonFromFile( - destination.resolve(manifest.getConfig()), ContainerConfigurationTemplate.class); + destination.resolve(loadManifest.getConfig()), ContainerConfigurationTemplate.class); - Image image = JsonToImageTranslator.toImage(manifest, configuration); - List layers; + if (configuration.getLayerCount() != loadManifest.getLayerFiles().size()) { + throw new LayerCountMismatchException( + "Invalid base image format: manifest contains " + + loadManifest.getLayerFiles().size() + + " layers, but container configuration contains " + + configuration.getLayerCount() + + " layers"); + } + + // Check the first layer to see if the layers are compressed already. 'docker save' output is + // uncompressed, but a jib-built tar has compressed layers. + // TODO: Skip this check for OCI? Apparently layers don't have any compression requirement. + // + // https://containers.gitbook.io/build-containers-the-hard-way/#registry-format-oci-image-manifest + boolean layersAreCompressed = false; + if (loadManifest.getLayerFiles().size() > 0) { + layersAreCompressed = isGzipped(destination.resolve(loadManifest.getLayerFiles().get(0))); + } + + // Convert v1.2 manifest to v2.2 manifest + List layers = new ArrayList<>(); + V22ManifestTemplate newManifest = new V22ManifestTemplate(); + for (int index = 0; index < loadManifest.getLayerFiles().size(); index++) { + Path file = destination.resolve(loadManifest.getLayerFiles().get(index)); + + Blob blob; + BlobDescriptor blobDescriptor; + if (layersAreCompressed) { + // If layers are compressed already, calculate digest/size as is + blob = Blobs.from(file); + blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); + } else { + // Compress uncompressed layers + // TODO: Consolidate with 'compress' method in CacheTest + blob = + Blobs.from( + outputStream -> { + try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) { + Blobs.from(file).writeTo(compressorStream); + } + }); + blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); + } + // 'manifest' contains the layer files in the same order as the diff ids in 'configuration' + // https://containers.gitbook.io/build-containers-the-hard-way/#docker-load-format + DescriptorDigest diffId = configuration.getLayerDiffId(index); + + CachedLayer layer = + CachedLayer.builder() + .setLayerBlob(blob) + .setLayerDiffId(diffId) + .setLayerDigest(blobDescriptor.getDigest()) + .setLayerSize(blobDescriptor.getSize()) + .build(); + // TODO: Check blob existence on target registry + layers.add(new PreparedLayer.Builder(layer).build()); + newManifest.addLayer(blobDescriptor.getSize(), blobDescriptor.getDigest()); + } + + BlobDescriptor configDescriptor = + Blobs.from(configuration).writeTo(ByteStreams.nullOutputStream()); + newManifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest()); + Image image = JsonToImageTranslator.toImage(newManifest, configuration); return new LocalImage(image, layers); } + + @VisibleForTesting + boolean isGzipped(Path path) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(2); + try (FileChannel channel = FileChannel.open(path)) { + for (int bytesRead = 0; bytesRead != -1 && buffer.hasRemaining(); ) { + bytesRead = channel.read(buffer); + } + } + return buffer.getInt() == GZIPInputStream.GZIP_MAGIC; + } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 1509e5189c..5c3dc2353e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -60,7 +60,6 @@ private static Future failedFuture() { new IllegalStateException("invalid usage; required step not configured")); } - private Future dockerSaveTar = failedFuture(); private Future baseImageAndAuth = failedFuture(); private Future>> baseImageLayers = failedFuture(); @Nullable private List> applicationLayers; @@ -186,18 +185,6 @@ public BuildResult run() throws ExecutionException, InterruptedException { } } - private void saveDocker() { - DockerClient dockerClient = new DockerClient(); - results.dockerSaveTar = - executorService.submit(() -> new SaveDockerStep(buildConfiguration, dockerClient).call()); - } - - private void extractTar(Future extractionPath) { - Future localImage = - executorService.submit(() -> new ExtractTarStep(extractionPath.get(), buildConfiguration).call()); - - } - private void retrieveTargetRegistryCredentials() { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java index 43a543693a..8efa2defb0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java @@ -27,7 +27,7 @@ public class CachedLayer implements Layer { /** Builds a {@link CachedLayer}. */ - static class Builder { + public static class Builder { @Nullable private DescriptorDigest layerDigest; @Nullable private DescriptorDigest layerDiffId; @@ -36,22 +36,22 @@ static class Builder { private Builder() {} - Builder setLayerDigest(DescriptorDigest layerDigest) { + public Builder setLayerDigest(DescriptorDigest layerDigest) { this.layerDigest = layerDigest; return this; } - Builder setLayerDiffId(DescriptorDigest layerDiffId) { + public Builder setLayerDiffId(DescriptorDigest layerDiffId) { this.layerDiffId = layerDiffId; return this; } - Builder setLayerSize(long layerSize) { + public Builder setLayerSize(long layerSize) { this.layerSize = layerSize; return this; } - Builder setLayerBlob(Blob layerBlob) { + public Builder setLayerBlob(Blob layerBlob) { this.layerBlob = layerBlob; return this; } @@ -60,7 +60,7 @@ boolean hasLayerBlob() { return layerBlob != null; } - CachedLayer build() { + public CachedLayer build() { return new CachedLayer( Preconditions.checkNotNull(layerDigest, "layerDigest required"), Preconditions.checkNotNull(layerDiffId, "layerDiffId required"), @@ -74,7 +74,7 @@ CachedLayer build() { * * @return the new {@link Builder} */ - static Builder builder() { + public static Builder builder() { return new Builder(); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplate.java index 895f59d8c0..cbf776a82e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplate.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplate.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.json.JsonTemplate; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.List; @@ -369,8 +368,11 @@ String getContainerUser() { return config.Volumes; } - @VisibleForTesting - DescriptorDigest getLayerDiffId(int index) { + public DescriptorDigest getLayerDiffId(int index) { return rootfs.diff_ids.get(index); } + + public int getLayerCount() { + return rootfs.diff_ids.size(); + } } From 55ab9928bdb21aecfd5da844be15b7670db8f50f Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Tue, 13 Aug 2019 10:38:44 -0400 Subject: [PATCH 03/16] Simplification --- .../google/cloud/tools/jib/blob/Blobs.java | 29 +++++++++++ .../jib/builder/steps/ExtractTarStep.java | 26 +++------- .../tools/jib/builder/steps/StepsRunner.java | 1 - .../cloud/tools/jib/cache/CacheTest.java | 49 +++---------------- 4 files changed, 45 insertions(+), 60 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index 643572f529..e21f22bc43 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -18,11 +18,14 @@ import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.json.JsonTemplate; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; /** Static methods for {@link Blob}. */ public class Blobs { @@ -77,5 +80,31 @@ public static byte[] writeToByteArray(Blob blob) throws IOException { return byteArrayOutputStream.toByteArray(); } + /** + * Gets a {@link Blob} that is {@code blob} compressed. + * + * @param blob the {@link Blob} to compress + * @return the compressed {@link Blob} + */ + public static Blob compress(Blob blob) { + return Blobs.from( + outputStream -> { + try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) { + blob.writeTo(compressorStream); + } + }); + } + + /** + * Gets a {@link Blob} that is {@code blob} decompressed. + * + * @param blob the {@link Blob} to decompress + * @return the decompressed {@link Blob} + * @throws IOException if an I/O exception occurs + */ + public static Blob decompress(Blob blob) throws IOException { + return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); + } + private Blobs() {} } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index 3c7ffab35a..ddfa6300b3 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -42,7 +42,6 @@ import java.util.List; import java.util.concurrent.Callable; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; /** Extracts a tar file base image. */ public class ExtractTarStep implements Callable { @@ -79,10 +78,11 @@ public LocalImage call() JsonTemplateMapper.readJsonFromFile( destination.resolve(loadManifest.getConfig()), ContainerConfigurationTemplate.class); - if (configuration.getLayerCount() != loadManifest.getLayerFiles().size()) { + List layerFiles = loadManifest.getLayerFiles(); + if (configuration.getLayerCount() != layerFiles.size()) { throw new LayerCountMismatchException( "Invalid base image format: manifest contains " - + loadManifest.getLayerFiles().size() + + layerFiles.size() + " layers, but container configuration contains " + configuration.getLayerCount() + " layers"); @@ -90,19 +90,16 @@ public LocalImage call() // Check the first layer to see if the layers are compressed already. 'docker save' output is // uncompressed, but a jib-built tar has compressed layers. - // TODO: Skip this check for OCI? Apparently layers don't have any compression requirement. - // - // https://containers.gitbook.io/build-containers-the-hard-way/#registry-format-oci-image-manifest boolean layersAreCompressed = false; - if (loadManifest.getLayerFiles().size() > 0) { - layersAreCompressed = isGzipped(destination.resolve(loadManifest.getLayerFiles().get(0))); + if (layerFiles.size() > 0) { + layersAreCompressed = isGzipped(destination.resolve(layerFiles.get(0))); } // Convert v1.2 manifest to v2.2 manifest List layers = new ArrayList<>(); V22ManifestTemplate newManifest = new V22ManifestTemplate(); - for (int index = 0; index < loadManifest.getLayerFiles().size(); index++) { - Path file = destination.resolve(loadManifest.getLayerFiles().get(index)); + for (int index = 0; index < layerFiles.size(); index++) { + Path file = destination.resolve(layerFiles.get(index)); Blob blob; BlobDescriptor blobDescriptor; @@ -112,14 +109,7 @@ public LocalImage call() blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); } else { // Compress uncompressed layers - // TODO: Consolidate with 'compress' method in CacheTest - blob = - Blobs.from( - outputStream -> { - try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) { - Blobs.from(file).writeTo(compressorStream); - } - }); + blob = Blobs.compress(Blobs.from(file)); blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 5c3dc2353e..1a73e959c2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.api.Credential; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java index 10408c3113..07b6eafb8f 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java @@ -24,16 +24,12 @@ import com.google.cloud.tools.jib.blob.Blobs; import com.google.common.collect.ImmutableList; import com.google.common.io.ByteStreams; -import com.google.common.io.CountingOutputStream; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileTime; import java.time.Instant; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -43,32 +39,6 @@ /** Tests for {@link Cache}. */ public class CacheTest { - /** - * Gets a {@link Blob} that is {@code blob} compressed. - * - * @param blob the {@link Blob} to compress - * @return the compressed {@link Blob} - */ - private static Blob compress(Blob blob) { - return Blobs.from( - outputStream -> { - try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) { - blob.writeTo(compressorStream); - } - }); - } - - /** - * Gets a {@link Blob} that is {@code blob} decompressed. - * - * @param blob the {@link Blob} to decompress - * @return the decompressed {@link Blob} - * @throws IOException if an I/O exception occurs - */ - private static Blob decompress(Blob blob) throws IOException { - return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); - } - /** * Gets the digest of {@code blob}. * @@ -88,10 +58,7 @@ private static DescriptorDigest digestOf(Blob blob) throws IOException { * @throws IOException if an I/O exception occurs */ private static long sizeOf(Blob blob) throws IOException { - CountingOutputStream countingOutputStream = - new CountingOutputStream(ByteStreams.nullOutputStream()); - blob.writeTo(countingOutputStream); - return countingOutputStream.getCount(); + return blob.writeTo(ByteStreams.nullOutputStream()).getSize(); } private static LayerEntry defaultLayerEntry(Path source, AbsoluteUnixPath destination) { @@ -125,9 +92,9 @@ public void setUp() throws IOException { Files.createFile(directory.resolve("another/source/file")); layerBlob1 = Blobs.from("layerBlob1"); - layerDigest1 = digestOf(compress(layerBlob1)); + layerDigest1 = digestOf(Blobs.compress(layerBlob1)); layerDiffId1 = digestOf(layerBlob1); - layerSize1 = sizeOf(compress(layerBlob1)); + layerSize1 = sizeOf(Blobs.compress(layerBlob1)); layerEntries1 = ImmutableList.of( defaultLayerEntry( @@ -137,9 +104,9 @@ public void setUp() throws IOException { AbsoluteUnixPath.get("/another/extraction/path"))); layerBlob2 = Blobs.from("layerBlob2"); - layerDigest2 = digestOf(compress(layerBlob2)); + layerDigest2 = digestOf(Blobs.compress(layerBlob2)); layerDiffId2 = digestOf(layerBlob2); - layerSize2 = sizeOf(compress(layerBlob2)); + layerSize2 = sizeOf(Blobs.compress(layerBlob2)); layerEntries2 = ImmutableList.of(); } @@ -161,7 +128,7 @@ public void testWriteCompressed_retrieveByLayerDigest() throws IOException, CacheCorruptedException { Cache cache = Cache.withDirectory(temporaryFolder.newFolder().toPath()); - verifyIsLayer1(cache.writeCompressedLayer(compress(layerBlob1))); + verifyIsLayer1(cache.writeCompressedLayer(Blobs.compress(layerBlob1))); verifyIsLayer1(cache.retrieve(layerDigest1).orElseThrow(AssertionError::new)); Assert.assertFalse(cache.retrieve(layerDigest2).isPresent()); } @@ -210,7 +177,7 @@ public void testRetrieveWithTwoEntriesInCache() throws IOException, CacheCorrupt * @throws IOException if an I/O exception occurs */ private void verifyIsLayer1(CachedLayer cachedLayer) throws IOException { - Assert.assertEquals("layerBlob1", Blobs.writeToString(decompress(cachedLayer.getBlob()))); + Assert.assertEquals("layerBlob1", Blobs.writeToString(Blobs.decompress(cachedLayer.getBlob()))); Assert.assertEquals(layerDigest1, cachedLayer.getDigest()); Assert.assertEquals(layerDiffId1, cachedLayer.getDiffId()); Assert.assertEquals(layerSize1, cachedLayer.getSize()); @@ -223,7 +190,7 @@ private void verifyIsLayer1(CachedLayer cachedLayer) throws IOException { * @throws IOException if an I/O exception occurs */ private void verifyIsLayer2(CachedLayer cachedLayer) throws IOException { - Assert.assertEquals("layerBlob2", Blobs.writeToString(decompress(cachedLayer.getBlob()))); + Assert.assertEquals("layerBlob2", Blobs.writeToString(Blobs.decompress(cachedLayer.getBlob()))); Assert.assertEquals(layerDigest2, cachedLayer.getDigest()); Assert.assertEquals(layerDiffId2, cachedLayer.getDiffId()); Assert.assertEquals(layerSize2, cachedLayer.getSize()); From 5e5b8298cf2ae1cdb3a58b20bb364eaaeb5930d2 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Tue, 13 Aug 2019 13:53:12 -0400 Subject: [PATCH 04/16] More simplification --- .../jib/builder/steps/ExtractTarStep.java | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index ddfa6300b3..5fec47a62b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; @@ -101,30 +100,21 @@ public LocalImage call() for (int index = 0; index < layerFiles.size(); index++) { Path file = destination.resolve(layerFiles.get(index)); - Blob blob; - BlobDescriptor blobDescriptor; - if (layersAreCompressed) { - // If layers are compressed already, calculate digest/size as is - blob = Blobs.from(file); - blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); - } else { - // Compress uncompressed layers - blob = Blobs.compress(Blobs.from(file)); - blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); - } + Blob blob = layersAreCompressed ? Blobs.from(file) : Blobs.compress(Blobs.from(file)); + BlobDescriptor blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); // 'manifest' contains the layer files in the same order as the diff ids in 'configuration' // https://containers.gitbook.io/build-containers-the-hard-way/#docker-load-format - DescriptorDigest diffId = configuration.getLayerDiffId(index); - CachedLayer layer = CachedLayer.builder() .setLayerBlob(blob) - .setLayerDiffId(diffId) .setLayerDigest(blobDescriptor.getDigest()) .setLayerSize(blobDescriptor.getSize()) + .setLayerDiffId(configuration.getLayerDiffId(index)) .build(); - // TODO: Check blob existence on target registry + + // TODO: Check blob existence on target registry (online mode only) + layers.add(new PreparedLayer.Builder(layer).build()); newManifest.addLayer(blobDescriptor.getSize(), blobDescriptor.getDigest()); } From 05d48cabefd38be5e8e07bc56bf45c7447dc8d0d Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Tue, 13 Aug 2019 14:04:15 -0400 Subject: [PATCH 05/16] Use parameter for destination for test friendliness --- .../cloud/tools/jib/builder/steps/ExtractTarStep.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index 5fec47a62b..4ca0057c4d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -56,10 +56,12 @@ static class LocalImage { } private final Path tarPath; + private final Path destination; private final BuildConfiguration buildConfiguration; - ExtractTarStep(Path tarPath, BuildConfiguration buildConfiguration) { + ExtractTarStep(Path tarPath, Path destination, BuildConfiguration buildConfiguration) { this.tarPath = tarPath; + this.destination = destination; this.buildConfiguration = buildConfiguration; } @@ -67,8 +69,6 @@ static class LocalImage { @Override public LocalImage call() throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException { - // TODO: Use a non-overlapping temp directory for better concurrency - Path destination = buildConfiguration.getBaseImageLayersCache().getTemporaryDirectory(); TarExtractor.extract(tarPath, destination); DockerManifestEntryTemplate loadManifest = JsonTemplateMapper.readJsonFromFile( From 2d4a127d2f9d2050a9edee2f321a69f669fef402 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Wed, 14 Aug 2019 13:37:45 -0400 Subject: [PATCH 06/16] Add test for isGzipped and fix isGzipped --- .../jib/builder/steps/ExtractTarStep.java | 31 +++++------- .../jib/builder/steps/ExtractTarStepTest.java | 47 ++++++++++++++++++ .../core/extraction/compressed.tar.gz | Bin 0 -> 200 bytes .../core/extraction/not-compressed.tar | Bin 0 -> 2560 bytes 4 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java create mode 100644 jib-core/src/test/resources/core/extraction/compressed.tar.gz create mode 100644 jib-core/src/test/resources/core/extraction/not-compressed.tar diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index 4ca0057c4d..80450ee4dc 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -21,7 +21,6 @@ import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; import com.google.cloud.tools.jib.cache.CachedLayer; -import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.LayerCountMismatchException; @@ -33,9 +32,10 @@ import com.google.cloud.tools.jib.tar.TarExtractor; import com.google.common.annotations.VisibleForTesting; import com.google.common.io.ByteStreams; +import java.io.BufferedInputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; +import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -55,17 +55,23 @@ static class LocalImage { } } + @VisibleForTesting + static boolean isGzipped(Path path) throws IOException { + try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(path))) { + inputStream.mark(2); + int magic = (inputStream.read() & 0xff) | ((inputStream.read() << 8) & 0xff00); + return magic == GZIPInputStream.GZIP_MAGIC; + } + } + private final Path tarPath; private final Path destination; - private final BuildConfiguration buildConfiguration; - ExtractTarStep(Path tarPath, Path destination, BuildConfiguration buildConfiguration) { + ExtractTarStep(Path tarPath, Path destination) { this.tarPath = tarPath; this.destination = destination; - this.buildConfiguration = buildConfiguration; } - // TODO: Future<> stuff @Override public LocalImage call() throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException { @@ -125,15 +131,4 @@ public LocalImage call() Image image = JsonToImageTranslator.toImage(newManifest, configuration); return new LocalImage(image, layers); } - - @VisibleForTesting - boolean isGzipped(Path path) throws IOException { - ByteBuffer buffer = ByteBuffer.allocate(2); - try (FileChannel channel = FileChannel.open(path)) { - for (int bytesRead = 0; bytesRead != -1 && buffer.hasRemaining(); ) { - bytesRead = channel.read(buffer); - } - } - return buffer.getInt() == GZIPInputStream.GZIP_MAGIC; - } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java new file mode 100644 index 0000000000..8a16a11265 --- /dev/null +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.junit.Assert; +import org.junit.Test; + +public class ExtractTarStepTest { + + private static Path getResource(String resource) throws URISyntaxException { + return Paths.get(Resources.getResource(resource).toURI()); + } + + @Test + public void testCall_valid() {} + + @Test + public void testCall_noLayers() {} + + @Test + public void testCall_layerCountMismatch() {} + + @Test + public void testIsGzipped() throws URISyntaxException, IOException { + Assert.assertTrue(ExtractTarStep.isGzipped(getResource("core/extraction/compressed.tar.gz"))); + Assert.assertFalse(ExtractTarStep.isGzipped(getResource("core/extraction/not-compressed.tar"))); + } +} diff --git a/jib-core/src/test/resources/core/extraction/compressed.tar.gz b/jib-core/src/test/resources/core/extraction/compressed.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..bb55593d55d53daa92e66a9e38cc5030e005fc5f GIT binary patch literal 200 zcmV;(05|_1iwFQnKvZ1-1MSnj4uUWcz;R}t;+{a;yIzX}yQ8l_q)`$Gq(XvkZ-uDw zYix|6zokvv;X3?x#kjRaY2Es0>$*)s5JG6BATbk2kPRcj^?b$>Br&p3evcreCZ+_& zAk(}UPJQA0DE+>5Wo^H{>s-|o?tF0V(0#g}e**XQPe=trRpv~*(K-seS3IG6t Cnqp)C literal 0 HcmV?d00001 diff --git a/jib-core/src/test/resources/core/extraction/not-compressed.tar b/jib-core/src/test/resources/core/extraction/not-compressed.tar new file mode 100644 index 0000000000000000000000000000000000000000..9faac26673c74ba6e993629080755695466d1a0a GIT binary patch literal 2560 zcmeH`&kBM-5XSfHQ_K_8?99%t9lUq!6(pAu6k}7cw@->x>=rBtIj8vp!}opgJLUb+ z=Y Date: Wed, 14 Aug 2019 16:40:52 -0400 Subject: [PATCH 07/16] Starting on tests --- .../jib/builder/steps/ExtractTarStep.java | 14 ++++++-- .../tools/jib/json/JsonTemplateMapper.java | 28 +++++---------- .../jib/builder/steps/ExtractTarStepTest.java | 32 +++++++++++++++++- .../resources/core/extraction/docker-save.tar | Bin 0 -> 28160 bytes .../resources/core/extraction/jib-image.tar | Bin 0 -> 8704 bytes 5 files changed, 51 insertions(+), 23 deletions(-) create mode 100644 jib-core/src/test/resources/core/extraction/docker-save.tar create mode 100644 jib-core/src/test/resources/core/extraction/jib-image.tar diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index 80450ee4dc..d16783b3d7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -16,6 +16,8 @@ package com.google.cloud.tools.jib.builder.steps; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; @@ -77,8 +79,11 @@ public LocalImage call() throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException { TarExtractor.extract(tarPath, destination); DockerManifestEntryTemplate loadManifest = - JsonTemplateMapper.readJsonFromFile( - destination.resolve("manifest.json"), DockerManifestEntryTemplate.class); + new ObjectMapper() + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true) + .readValue( + Files.newInputStream(destination.resolve("manifest.json")), + DockerManifestEntryTemplate[].class)[0]; ContainerConfigurationTemplate configuration = JsonTemplateMapper.readJsonFromFile( destination.resolve(loadManifest.getConfig()), ContainerConfigurationTemplate.class); @@ -101,15 +106,18 @@ public LocalImage call() } // Convert v1.2 manifest to v2.2 manifest + // TODO: Optimize; calculating layer digests is slow for large layers List layers = new ArrayList<>(); V22ManifestTemplate newManifest = new V22ManifestTemplate(); for (int index = 0; index < layerFiles.size(); index++) { Path file = destination.resolve(layerFiles.get(index)); + // Compress if necessary and calculate the digest/size Blob blob = layersAreCompressed ? Blobs.from(file) : Blobs.compress(Blobs.from(file)); BlobDescriptor blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); - // 'manifest' contains the layer files in the same order as the diff ids in 'configuration' + // 'manifest' contains the layer files in the same order as the diff ids in 'configuration', + // so we don't need to recalculate those. // https://containers.gitbook.io/build-containers-the-hard-way/#docker-load-format CachedLayer layer = CachedLayer.builder() diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java index 45747d3f29..916a3d8540 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java @@ -16,8 +16,6 @@ package com.google.cloud.tools.jib.json; -import com.fasterxml.jackson.core.JsonGenerationException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.CollectionType; import java.io.ByteArrayOutputStream; @@ -134,50 +132,42 @@ public static List readListOfJson( return objectMapper.readValue(jsonString, listType); } - public static String toUtf8String(JsonTemplate template) - throws JsonGenerationException, JsonMappingException, IOException { + public static String toUtf8String(JsonTemplate template) throws IOException { return toUtf8String((Object) template); } - public static String toUtf8String(List templates) - throws JsonGenerationException, JsonMappingException, IOException { + public static String toUtf8String(List templates) throws IOException { return toUtf8String((Object) templates); } - public static byte[] toByteArray(JsonTemplate template) - throws JsonGenerationException, JsonMappingException, IOException { + public static byte[] toByteArray(JsonTemplate template) throws IOException { return toByteArray((Object) template); } - public static byte[] toByteArray(List templates) - throws JsonGenerationException, JsonMappingException, IOException { + public static byte[] toByteArray(List templates) throws IOException { return toByteArray((Object) templates); } - public static void writeTo(JsonTemplate template, OutputStream out) - throws JsonGenerationException, JsonMappingException, IOException { + public static void writeTo(JsonTemplate template, OutputStream out) throws IOException { writeTo((Object) template, out); } public static void writeTo(List templates, OutputStream out) - throws JsonGenerationException, JsonMappingException, IOException { + throws IOException { writeTo((Object) templates, out); } - private static String toUtf8String(Object template) - throws JsonGenerationException, JsonMappingException, IOException { + private static String toUtf8String(Object template) throws IOException { return new String(toByteArray(template), StandardCharsets.UTF_8); } - private static byte[] toByteArray(Object template) - throws JsonGenerationException, JsonMappingException, IOException { + private static byte[] toByteArray(Object template) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); writeTo(template, out); return out.toByteArray(); } - private static void writeTo(Object template, OutputStream out) - throws JsonGenerationException, JsonMappingException, IOException { + private static void writeTo(Object template, OutputStream out) throws IOException { objectMapper.writeValue(out, template); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java index 8a16a11265..1c476c9aff 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java @@ -16,22 +16,52 @@ package com.google.cloud.tools.jib.builder.steps; +import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; +import com.google.cloud.tools.jib.image.LayerCountMismatchException; +import com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException; import com.google.common.io.Resources; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; public class ExtractTarStepTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private static Path getResource(String resource) throws URISyntaxException { return Paths.get(Resources.getResource(resource).toURI()); } @Test - public void testCall_valid() {} + public void testCall_validDocker() + throws URISyntaxException, LayerCountMismatchException, + BadContainerConfigurationFormatException, IOException { + Path dockerBuild = getResource("core/extraction/docker-save.tar"); + LocalImage result = new ExtractTarStep(dockerBuild, temporaryFolder.getRoot().toPath()).call(); + + Assert.assertEquals(2, result.layers.size()); + // TODO: Assert layers are correct + + Assert.assertEquals("value1", result.baseImage.getLabels().get("label1")); + } + + @Test + public void testCall_validTar() + throws URISyntaxException, LayerCountMismatchException, + BadContainerConfigurationFormatException, IOException { + Path tarBuild = getResource("core/extraction/jib-image.tar"); + LocalImage result = new ExtractTarStep(tarBuild, temporaryFolder.getRoot().toPath()).call(); + + Assert.assertEquals(2, result.layers.size()); + // TODO: Assert layers are correct + + Assert.assertEquals("value1", result.baseImage.getLabels().get("label1")); + } @Test public void testCall_noLayers() {} diff --git a/jib-core/src/test/resources/core/extraction/docker-save.tar b/jib-core/src/test/resources/core/extraction/docker-save.tar new file mode 100644 index 0000000000000000000000000000000000000000..08ba90b4c97be0d68dac49b16872ac9c8795a489 GIT binary patch literal 28160 zcmeG_3s_Xu)@wu@7*G>MMNM%+1?6$hnKNf*@R2VNke7gnsNtFOfWct~X9k7ed}L~9 zSZd}?ubTEU?dr{ZVVeF{K1wb9$+Rpr|35=3Ew9#1ulcVt4+KIKc@#AVzB&7`_S$=| zwf1A}$2tVVXf>2bYFI%_>O?Kguneze^_TVS=Z7RtxnM5OdjW-3FKtcA-l*aXg2U=a|C&EpvwgUwd- z7!9U^LiBKElW53RMov>knWiWs$4v-Dhs{!yZ#I~0=vU^lQ&@xxM#)8D zX{NBo0)fP2_7KVyLMahE-E1t#6YOcxmt(Nn%$6c7+5Hfpz?NgSAhtMz9+5i#9Z6|1n{A?jj=8Xr4!`PD~=naw>vU>p8th zuslm^)C8}gv<$@vdXf+cQLR;zI-Nl28Aii%yhbO`I*|}`6hmsXf^x$2=}puq{zBQ% zT9&0)R>Nx4EG1|difW5Z;-!u|6gzZr$?ov#wI1!#)o^1|HyMyHU5({ zMY+fS+GyHPab$SCXFd&86gQsVfdchz{|zwzSF0+<250|I;*jrd|4ktd)075j>R(?m zz-eASYjMnS8B$Ad0#4Z^y;6lb1*2oMdO@$k`5~ht7?IZUYKo+JMvJj3p402}=;lc+ zso_MuN}4j?OD92n!^Rp+f+g#Y<`HAI+DvSoG^0fn!_9dJaK}H|Do9g!j2U9Hv0P57 zjW?K-ks@of3YEgl1vpW!9w=BasWULmhSPqhJ1Ji|Q4WYC)6fEgu>zG#50WBLvuPIeko3cQ zP(xDjYNJK{Rm!{2@xJs)EZn`-;^v^P)VJYB*pSD9h|0*hXPpbSe0G#}{ z1cKmn(sW&=!ZksG#RXdtmu$71s1x)$M$J)_j?+|H8LTt^DTYvK z-0i<91Ry*jnJtXLC0^VC2+xRch(lJT;ULS=9b`SYIDm8hS1~k713o@3KS9yz3b%dy zSE0n<`d?vF5TJqQf3AuIHbvPRAP?N*4a{e4IRhK9BdG5llA;0X=|)jA1)75G_h|pP z)`xTciU-}4h9sNm`k$*Jfi;u+4;R|wX)kA)z8`Zswf&gRx5IiGGt&1L?(@mr{Yk4i zL4y~iZylW+v;ul{FuxP_0U6qR*Tu{8hhF*l>34Q)e>m&Lhg(*)8`i(z#uq0OdrUeT zd8=(?S=U=1MZXX<{Eh$p`qggYPv7ajxjEzPFEjRDSs9wibbWuIX>GFhlM|m^^zqLu z%KGWd1)ptmBO^cm>f7BXyS5L@wjO;c>`X{vk!317>F1XXugp|#{EXRhW$eRW?D;is z!meG*uI=ryG2`}(Wr=^6W!uJZ$BNo~-{vXvD>F;Cyzz4U?fjVXnFm*V5qI;&Z*RZ% z=-KhFhL=S6514f(v)lVU1Ewp+mtQ~kX8Fy5#l^oC&wjbMZ!x>3&6AxsyvKFkG_2br z=YK3M8P)cgqWtkaOWJo|+2O*v0fj^NJy#fapiHs)i3|H)C~vP=GLcH0wkYLT@#Lv& z|4nB*l)tcOa`Zl?}Xdbk?}7;cFKp#AT#r96vgd|KS)vTi9vpvbDV5)#A=8$EJrL9{bj*S2DMx zY+W_lfA+`>@oV2r*%{vzwk^wfw*P_>YQ@@DGv);h(2h6u`r`cfkt^pL#`X~cR#Cnw zi-Q)kA0F~MK74)4UZxBGX!&iKXGGjryPt->Q_c?w_;|ycgQrajK07ukF{#gT^_8;o zWi1Bzchh+%Pwx0l@ahGz@gEc>?*CWe6Yp*)Y!~rC;n@BC3ZMUJ&3X0iP3K$h4q3Zr z0IZ*E%>Q=|f9}mI6ZwB9UpaK{2k)$)JqP>jdv42e|Cme57VP+8`<%B6S8NXbrDY45 z|CNbx?_LeeX3PT<4BNJEn|i2l%>JU?eb`m)4+buOKXcKguaB&G`N@)>M?XHRt@oyn zj|9ZN2y2!OdiNkTs9nhPIKR=yUbv<@TE_pfJm$vi1t0%7cg|7GmNe5A8xADAvGjDv-&HNF%XN>9-x9U>^}^h8 z&8&b}?~T7IzG07lay>I}L|e@_C&qoBWsK=)Ngp0pmVPqr-Pg8k-=bf=dG5(UVME3) zK5#wcz~TQnJJ#=7?}J>gU!qHl&y=nI$lGVbOSo_C+H0JQosJ4d3 zS1y^{?)5pB6%(J|R9dbtxg2%q{4;_7*M*54er)8fy-QB7FJG9Lv183kLxLuKe&dzF zUG?eFJMuCgDZRC$^_bx^R(kgy7ni3_ytwdd?U2zWrC(;vOJ1BkcS<*Id%~+}mZi}L zE^a**F!}WPpL#C7HDX2dX8-=>V^X*FjXB@u)pajM54m(Pch0U~6t9f@$kKk`q?I3B z&zV=sJnZq^P@n7Da-!Cp3%ym6d&A?`8(l`9eYfDeV#7O^13sa%ZYj1M-Sm^8^c`wp z_POOl^18>_PKI6zIx_V6hbArGPkDDbJ~-lVug9PD_n1=_x2(*%w`0#w+s~dVbU))6 zlzDB_iha3Px}2PvvG`!-r7l-9rnd8ln(^)2to8fn4Pe68#J|_6A9Z%PPnhb|?l<>z z8F%$mtd6%v;N9-*b(e>iy7em`>J&#*&54yVS^OSM9BR3Ac*(pnR>PP7CyNwHXAOF=H z!4m?drfHoI|4i{MygbL70GI8 z5l;uwc-~B;)mp8H=loPWqoVK#nkW#Io@F$mUZ=tRVM4D{6Ev$+=>*cP|23W*G-Cf> z!S;LTe*{UZTxt_)fe!r-j|4W${XbVl0vnHI4T|}W{?{)yEc|%F;Ulpp)HEfoX)TS~ zgo+$R%mP~%uaZTl<|$gun+4#-1hOPeTHi=f4s-{vquq% z=zYPRJw0TQvA??wT-s%W7TS%h!6;Be`?m^4JnV0_G}x|io#%g~S|zRjG{^J5&8_{{ z%6~!-1Rajp8V#YR_4unIbb5kT33z~?6&Vo~0gC3eB8S5|EufQ9Ygyd;5lB|eNn3c1 z=brt4rM&=0|IbircmLmDZM-%_Nbh>>^3{gh9rwqlK&|axWI0M|+l*GLCGz1x5N@Bb2L0iKon1y3Mp9uaGL6*P?FgY+ zAvHQADEGx(*B#($7k4Y!Q)8{~bD z{T~{4djh3J0LO#pe@L2;`0w8TX%O<)isZvz-`;}&H2h5gyUbT1g=1sMv&G1dz*CGc@xFkS%@ z;4wL5DPSV7UZ9uWoC0_dX*%7HW3z}W&FQIRhJ0)+3C0j!Gn|K+t5DI1B?}#y&BVv=H z3OOO)W-yx&bzr3wu_hZDjw#4TL0TC$dGEKd6xGM7PXOM%H|$P*GWh6*f}F{l#0 zJl2FpV8zF+|?Tc_&Pwi}!gEUq-6foyo7FW8_!2H*=*U@GS43x!Z5hyU`0 zY4A8^>kHFih8$-4!Yp_K%is&M;YnYZ19N5YQ1uLu8)K4EM#d(N2pk@p5;ZI>DP?S6 zY-(hn47>`^a!)vKu=v6}cuEF7sYXG_M_z@8hx(X4b74SD=fwqUCw z_L#Oxrj?Rcfx}|7!?dxc`~n-cYBU0CvZ0Loxf0vS7mxN}5wOi2f=VG+MKpZtq-)12 z30(A+Dw)Gt6Vb@3y?4q)0WsM6&;2jxUw0U%}F--*5=@JTy zXO0yZYLr+7mN-GMTCu0C(v;G2uw7ys@w7_44>CD8E+id#LJ*Fc($fNvna~G+gV7g( zZ|oYuv+oX&ZMH{;;BPB?U;y+*x32>L`avlAydez2F$5FKzJ!+e2k@V_pv5lm7`p>n z`gv}LR(Kss(i%9bL;PNen;r0yAH%2^=bCQx?z z)8L6|3K6mV0(O@ZH&0yt|j zh&cDXpYJ$4XgQtwf5pjs?>whs;SfA_Ya0SzullD z8sb|ZxW|(s>nTCB<(m{Qn4b9_^k=6P5PokyyL3__^E-0+k?Q&8_u95 zesEyIpZ@Bh1$kvPpo(P>tc@jm_iF!W?cLREq^?0~|BPh+(cS(Viz9dNPE%HEp^gu# f*c)jBn1!A`{ZHS5?$FZha#O%f0XGGj69xVs*P1p+ literal 0 HcmV?d00001 diff --git a/jib-core/src/test/resources/core/extraction/jib-image.tar b/jib-core/src/test/resources/core/extraction/jib-image.tar new file mode 100644 index 0000000000000000000000000000000000000000..1281542d7edf0aac71d9c1a6f65a6ade52db71a3 GIT binary patch literal 8704 zcmeHM2{hGRyQk3*%21}z=@4Z&^I)FIaA+{|a)$rma5$WE6on%pi3~|4v&ayYIgun3 z3CSEvnJZ-~!ud{p@3-!Izq`Kgy=&dxyVhN||N5_U_VeF+|Ms(=z4z~V*0UiHC=?FE z;E6~Q8Gw^W00Kfp6OjM}fIx^?3<-*X!|`xDfCpd%G699c5Xo48fJGsRFc_AICaW^> zbXE6LT>tKYKp_wm0wD+a`}5ELFccIj2ZbYHNC+B&M4;s$Fc=h#;*x{>JJ6q1=g;8G zfD7_J9_ybc_p|i-i*aNx@ccf)2fLwTB6i|zm3>ru+opg9e$U#M+rq)R@wVkQ*5DK_ zc~M%O%2TMa!u{!w=k@2lT&Sz6-0Qm9T9&rUK*N9aRUb~)YgBz*NWEKX{kic~uwl{L z@my*0@_X#$w~*15kovh)Wk;0MV=bR7Ys|}@foXm*$3WNRkqQ2a2zB+rH?y^UQX(qu zjP`7m5hYw8{UqM&OAh4*3sgLSDw}iI`|81lx6A$e$&2-}#rEuwWZZRbccvY&Gf?n@ z;5ph2R&`lXjs@G-bc)rI^2+R6#w@#0XVf`YHAYQLGjzmJ`td%AU_R%mrPp_+zG+28 zZA69TL>-L6X9}JbFKi@=ml#OzANyP#b42K3ps(}37!jFN(TSUy0s0M>0?eAb`O?o! zG+dn$;Y)Ob;ZDaN?Tqp}iPtC@6`i^o?`PbAifJ=#dt#bDP$Z?!G7M`QMI6kk>9c_G z9?Gt1*haqwCC7RYDJh?g+ryJ(dX!yaZd=(d_zfq^nWU|L%sU^GCnemwYGrhwyvz|& zEK%T=>Qs|3v-=ZL#2vou93(x*;R7;w)=vV7{$e3W>qgRN?t zLuvmF$Fig4X*ObE=Jw?A9VPDe?*oLoJ>oQ?V_+#+x%QC~niyxQ{Hrl%^VIVchXa5_ z8f?eW1aJbr^@(toVZj!8pNU&rX1lklnZ2pKz;*D%n4UyO;T_%6UW%g*RyeBz*N}7F zW8Isy#iX&k)_%Ju71N^+TRaWGH7x|3c~ls%OYLcZL(`#v%gdQ#$l8)Ifm)@k2bx?3 zepKJD9;DZI=G;hMt>>P+{>1AFe$aBD;ZoT(G1Hmk=&DbZ;WYs%ca>MRY~mK1b2EE1 zFXxV;X<1S#Dl1Mt39xGltUZ8F6KRpV_Si9gX8c)Z&e@nRHbJ36yd@paBo1eAWhQAq zYJq9*QVKQ`w&}dO2ygEutz0u*4U6vh91-4*wsQ#^dQ%Mk+;EpS`L)rJ$S58h6B2tc zWQaA3TbdRhuTB>3^ud*Cx|?omXx44pcl=RaS!Ed^{cc2`wu+uZLi3VRbK8qi2jNA97NY!$aSZih zcfoUB{zAXG+A;&z<5gG*RkgG%(WR}w^(AATEvO_)_r2#vJxv>$28%_#*@>e$pEy)76^pa4%FNaSdkwDM(3K+C8drHc?yp|260kE2N##{=GV?~_renu3dNwiD z!>*Cm3GNXmq=}W5xyR{A#?8~^of3XSW6S#z)=g53?}}+m*&QoCXgVgCdo#mWZ)Vyv z{Qe5x4fE%85iPIOr%N7@)u_Fj-|O=)Rd^U>zE)n3@m$@!u_|dZ`pAEbudwc;#7l(h zI$uS5$ugz74i@YF`kJ1%j2W{}c?SGU|ME_+YfUiTJzctLZSp~JVw=Oe&62wr^}F}= ziiDj6WJb1v9T!Vd8a(GD`%c;?v^dU4&fA~dwb>|SHo~=_DN+-qnrYFv=MZevkY5Gf zUwh|)q|uG7Bg+Ih1M#4j=d2tCU z;tEiU6fF~2>H!%wk+Q(jtMZjB(23g#iuH2gI$a+>$tryOwy&ES74Obl9BW(n#JJYK zQ%XtKb}S=aVJ&KS3KT4p-K8BcafdrG=G+l^VqDBeY*v-{eLkYZX?C-^#5a=W{B}tx zmPvh#iPZGn3}qqA)-DEHdp>#asFSC8vHrI`uGs$1T%vz5%l~V5A`}9UVF)~u2#}Ew zEQ~-P5|DT#3WY+$phPkmO~9at02YlQ!XR)w7RiYS$V3bZ08m6ImW)OG1pkjfK+*rj z|3l!v{QtjRwVzk>2mhaP8e=mp({gf;Wo4^Uk~zgIKdG(ec7>71UB+CrWs0|p%Zsh| zsa6894f&PXN(UmdhYTBE?!+m@wdU&i9Z!l*J=PWBgP49YpfwRl#&ug}Ph!h-ShvR$Dv{WYf&@i;LdTjU3#}6T43~-U@wk_?MqR+Xn z?#za6dh4<6p;BlQ*4T7Em?ulR$y&6nlj{*=&1yYyBTiz+z~~!}4h?qmI)Y(xR-H7f z2SO}Aw$+H{d{tL)Tx64^%%E1@wljn;!VY&Gb>BqnIugkk8)gje7FSq^LD;21wD?mj zv_=LiyYkkjCqz~PFM(a^-`%*Frr~AT*Q4I|v`tE-rboqdsix!n3CTg3)xP$1k$cZi zKN{+2l*)+MQ*rZZ0REKwE!#XbIaTM{SA}`wVhYo>A?6=E$|IUCOP%DOT|`&}8x$^I z{SAFWfL~Dliq7zCy&A!+7f`-A%y`jrNzGPSBSiD_;jVYeepUh5vtJB?a%GtQ{a3#c zAGlPgWy?)={x+H3v#hcUcxtydWA4EaWA%dZ+&Y=gP-zqBt5ZmacCJLv4ZrAOR*&Da zp}xP+|D^6Noba?9I_lw`o>x&eiCe2(!Ji4rT1~#wGqQ~?urAu}q+{Pojjq3hP41mP z85~>`;C^8n>$=dp8r6^^{#0v4?4kQd$^iGE_*oe_}zs?H7hT4}Um7Ar;w=Xuk%%srJ zD(DveCoi6UI_z*t`OWjM+w&vU)&}B+(8}#d^U-%>ioYqB6wKpgj>8k)>)6grR6Yw5 zElL}k_OX4l!dTb7ooaNiz{fEGNBB6APj&sQb?sE~Xs^Tdch>aP5Wkp<@^AKfFe?Uz zlz^Q2NrdeIG06(W2FRJU+#u&z?`Gee%T*!LAyu|buRa^UA&bSHK$4^9kbl~pM4+3eX8Tdx(|r?7pp4Az&}QhM78Xl+i(xEz@LEa}6Pg_kjZ z*>#gWk~cHT-vL{;mTdIIQPf|3Y+S3!tgD?qQ~3B3X`p;;j-FWleh%!@%8K9PZ;>4I z_}###EQ9jFU}-BY;je_ETh4276MF?zt^_89i{bVkL$+=?>|Q%I_v)6#(6`vNf}vG` znZkj|L;lhy7G6|#XxnUSm+@M={1#I+lV;d&2v1oPG}KEG>@QRMe% zKu=-v$#t(s6}(iL0lX6hZ>!W)>(qG5Wo6&{%-YZXR`u@XnbcZo&%T7Int{CDbl%*( z&ADQ1>Z0@)lBb0Cq6Jv!)zRyII2#psejUWUgmn0Ub;~nNq)VB>cw3EO_Wa%H%JcqT zBHi4NNjVBs5NI)U1WJ2~PO%R{H|lvm{1PQ1IG#(DR%}&A4G!aBF4e-R(Jp zU$r$#`5hGlue$^;>~5&{hi)J8y!qB>ND}5a=G?SGIekmuitF?s+9Zy?XZ-f3)!IxA zR{1&afasZ*xfc&v_w`m^53jgh`#J?o)(fL84xVSGw{6Hb3>oDX-|JD=6KGG{#yh-_ zXk=1Ot_V z<-MVvMZOpq(Y0Fa*qH6PSH#O8x>?HkMbr*p>2{I$DXphCRY8xwVt%m$NrA&%!C=9? z8qZYMe9FN&;AUaCIYV0uukg~Nh|JfwM@GBg{^F6k2pf-6@0xIz4RG!=jkQMiaTiZ4 zk2sZ&oo^K#ORu;rru{KQnyz(F^V7rLEyq3-WHL&7G!=oA2ltF|{T&*y^Ps$2>q_`j z334+RQYyD!x>>1(&KHP@{bJJ|&FE0xBUVz-3r~MLQmYX%!y??Suvjr;S+!FKiW?0o zpVqs07}73R@pdm%#4S+L{Izm-t9MAWnPg1Q7(F|b*wO8!RXSvtW?jHLeQjOwb8B`k zH_idoalkZuNhmNSLz!J4xNbrT<+kTB+PUw!K03 z)NMQ#M=GGoXLhXp(`fejb#6kNKiQG{jXlhP4rXiC{-jF+7^pXDVhVZ_pd*R;Bq~#>EZ1El? z7X-$#4f=JHlS@_uoOFXn*?MoTjIxrJMjtNNu5%{?5z%5&E!=!rqITjtk8*7)lh#d? zZdTg8i3iVf(N=kur$}61QIC(3y7h2U$i0ZZHjM>##w?4+W`eDkRqq<8v$S@3F|KMg zV?Sjow+b32$7OaTktm&&&#Sko0U{ddA#S`&Sl=|7TOulxXUh!_@dJv-qwu1+rf=#N zhuMl7bhh0z`^?RHn~j_4^@$swCE4b7w>GNXA8Nl|6Ux_Q=Ng_DmfzgQ;%3|D8*r)q zys+y33=)y%L#DW^dNOD}f0@pIlK+Fkkf{G0|3f+HKj^Rc|1VpRKLDQw5$OP)36MbQ zASf0MQGq}??{Nr3{rd}b1Sx~?bfO1^2@sk7bb!+e?@dA>IGpi*g|?`Q$OGzLJjrqP+4aZd*;gDmg_fcm`*qJA$yIoh1SQ~d!b zr`wM&uz!4kfjG@=X;go2;74DM Date: Thu, 15 Aug 2019 11:36:41 -0400 Subject: [PATCH 08/16] Tests --- .../jib/builder/steps/ExtractTarStep.java | 2 +- .../jib/builder/steps/SaveDockerStep.java | 4 +-- .../jib/builder/steps/ExtractTarStepTest.java | 34 +++++++++++++------ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index d16783b3d7..56b9f41e59 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -106,7 +106,7 @@ public LocalImage call() } // Convert v1.2 manifest to v2.2 manifest - // TODO: Optimize; calculating layer digests is slow for large layers + // TODO: Parallelize/other optimizations; calculating layer digests is slow for large layers List layers = new ArrayList<>(); V22ManifestTemplate newManifest = new V22ManifestTemplate(); for (int index = 0; index < layerFiles.size(); index++) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java index c43d6f09b2..3fb28d4f79 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java @@ -20,6 +20,7 @@ import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.Callable; @@ -36,8 +37,7 @@ public class SaveDockerStep implements Callable { @Override public Path call() throws IOException, InterruptedException { - Path outputPath = - buildConfiguration.getBaseImageLayersCache().getTemporaryDirectory().resolve("out.tar"); + Path outputPath = Files.createTempDirectory("jib-docker-save").resolve("out.tar"); ImageReference imageReference = buildConfiguration.getBaseImageConfiguration().getImage(); dockerClient.save(imageReference, outputPath); return outputPath; diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java index 1c476c9aff..3afbe6a866 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java @@ -45,8 +45,18 @@ public void testCall_validDocker() LocalImage result = new ExtractTarStep(dockerBuild, temporaryFolder.getRoot().toPath()).call(); Assert.assertEquals(2, result.layers.size()); - // TODO: Assert layers are correct - + Assert.assertEquals( + "5e701122d3347fae0758cd5b7f0692c686fcd07b0e7fd9c4a125fbdbbedc04dd", + result.layers.get(0).getDiffId().getHash()); + Assert.assertEquals( + "0011328ac5dfe3dde40c7c5e0e00c98d1833a3aeae2bfb668cf9eb965c229c7f", + result.layers.get(0).getBlobDescriptor().getDigest().getHash()); + Assert.assertEquals( + "f1ac3015bcbf0ada4750d728626eb10f0f585199e2b667dcd79e49f0e926178e", + result.layers.get(1).getDiffId().getHash()); + Assert.assertEquals( + "c10ef24a5cef5092bbcb5a5666721cff7b86ce978c203a958d1fc86ee6c19f94", + result.layers.get(1).getBlobDescriptor().getDigest().getHash()); Assert.assertEquals("value1", result.baseImage.getLabels().get("label1")); } @@ -58,17 +68,21 @@ public void testCall_validTar() LocalImage result = new ExtractTarStep(tarBuild, temporaryFolder.getRoot().toPath()).call(); Assert.assertEquals(2, result.layers.size()); - // TODO: Assert layers are correct - + Assert.assertEquals( + "5e701122d3347fae0758cd5b7f0692c686fcd07b0e7fd9c4a125fbdbbedc04dd", + result.layers.get(0).getDiffId().getHash()); + Assert.assertEquals( + "0011328ac5dfe3dde40c7c5e0e00c98d1833a3aeae2bfb668cf9eb965c229c7f", + result.layers.get(0).getBlobDescriptor().getDigest().getHash()); + Assert.assertEquals( + "f1ac3015bcbf0ada4750d728626eb10f0f585199e2b667dcd79e49f0e926178e", + result.layers.get(1).getDiffId().getHash()); + Assert.assertEquals( + "c10ef24a5cef5092bbcb5a5666721cff7b86ce978c203a958d1fc86ee6c19f94", + result.layers.get(1).getBlobDescriptor().getDigest().getHash()); Assert.assertEquals("value1", result.baseImage.getLabels().get("label1")); } - @Test - public void testCall_noLayers() {} - - @Test - public void testCall_layerCountMismatch() {} - @Test public void testIsGzipped() throws URISyntaxException, IOException { Assert.assertTrue(ExtractTarStep.isGzipped(getResource("core/extraction/compressed.tar.gz"))); From e7f7f9491f3a6da1e1d6412b6446c0881ae0dbe2 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Thu, 15 Aug 2019 11:47:51 -0400 Subject: [PATCH 09/16] Cleanup --- .../cloud/tools/jib/builder/steps/ExtractTarStep.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index 56b9f41e59..af16353c19 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -100,19 +100,17 @@ public LocalImage call() // Check the first layer to see if the layers are compressed already. 'docker save' output is // uncompressed, but a jib-built tar has compressed layers. - boolean layersAreCompressed = false; - if (layerFiles.size() > 0) { - layersAreCompressed = isGzipped(destination.resolve(layerFiles.get(0))); - } + boolean layersAreCompressed = + layerFiles.size() > 0 && isGzipped(destination.resolve(layerFiles.get(0))); - // Convert v1.2 manifest to v2.2 manifest + // Process layer blobs // TODO: Parallelize/other optimizations; calculating layer digests is slow for large layers List layers = new ArrayList<>(); V22ManifestTemplate newManifest = new V22ManifestTemplate(); for (int index = 0; index < layerFiles.size(); index++) { Path file = destination.resolve(layerFiles.get(index)); - // Compress if necessary and calculate the digest/size + // Compress layers if necessary and calculate the digest/size Blob blob = layersAreCompressed ? Blobs.from(file) : Blobs.compress(Blobs.from(file)); BlobDescriptor blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); From 6bb7fc6c9e5e164d8776d7045837178a8a263ad0 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Thu, 15 Aug 2019 12:04:36 -0400 Subject: [PATCH 10/16] Remove unused cache stuff --- .../java/com/google/cloud/tools/jib/cache/Cache.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java index 25dd873478..6a5c946eed 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java @@ -53,12 +53,10 @@ public static Cache withDirectory(Path cacheDirectory) throws IOException { private final CacheStorageWriter cacheStorageWriter; private final CacheStorageReader cacheStorageReader; - private final Path temporaryDirectory; private Cache(CacheStorageFiles cacheStorageFiles) { this.cacheStorageWriter = new CacheStorageWriter(cacheStorageFiles); this.cacheStorageReader = new CacheStorageReader(cacheStorageFiles); - this.temporaryDirectory = cacheStorageFiles.getTemporaryDirectory(); } /** @@ -162,13 +160,4 @@ public Optional retrieve(DescriptorDigest layerDigest) throws IOException, CacheCorruptedException { return cacheStorageReader.retrieve(layerDigest); } - - /** - * Returns the cache's temporary directory. - * - * @return the cache's temporary directory - */ - public Path getTemporaryDirectory() { - return temporaryDirectory; - } } From 6f4c0c075942cdf5d8ccd5fd1d9236e73e7fbad7 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Thu, 15 Aug 2019 17:37:30 -0400 Subject: [PATCH 11/16] Clarify comment --- .../google/cloud/tools/jib/builder/steps/ExtractTarStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index af16353c19..31ec5d704d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -104,7 +104,7 @@ public LocalImage call() layerFiles.size() > 0 && isGzipped(destination.resolve(layerFiles.get(0))); // Process layer blobs - // TODO: Parallelize/other optimizations; calculating layer digests is slow for large layers + // TODO: Optimize; compressing/calculating layer digests is slow List layers = new ArrayList<>(); V22ManifestTemplate newManifest = new V22ManifestTemplate(); for (int index = 0; index < layerFiles.size(); index++) { From 7276084ba24cdb9fa4609e7b1bf2ba4afa38ee18 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Fri, 16 Aug 2019 12:10:44 -0400 Subject: [PATCH 12/16] Feedback --- .../google/cloud/tools/jib/blob/Blobs.java | 13 -------- .../jib/builder/steps/ExtractTarStep.java | 33 +++++++++++-------- .../cloud/tools/jib/cache/CacheTest.java | 17 ++++++++-- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java index e21f22bc43..062f02dc0e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java @@ -18,13 +18,11 @@ import com.google.cloud.tools.jib.hash.WritableContents; import com.google.cloud.tools.jib.json.JsonTemplate; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; /** Static methods for {@link Blob}. */ @@ -95,16 +93,5 @@ public static Blob compress(Blob blob) { }); } - /** - * Gets a {@link Blob} that is {@code blob} decompressed. - * - * @param blob the {@link Blob} to decompress - * @return the decompressed {@link Blob} - * @throws IOException if an I/O exception occurs - */ - public static Blob decompress(Blob blob) throws IOException { - return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); - } - private Blobs() {} } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index 31ec5d704d..f6b9f1238d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -34,7 +34,6 @@ import com.google.cloud.tools.jib.tar.TarExtractor; import com.google.common.annotations.VisibleForTesting; import com.google.common.io.ByteStreams; -import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; @@ -47,9 +46,10 @@ /** Extracts a tar file base image. */ public class ExtractTarStep implements Callable { + /** Contains an {@link Image} and its layers. * */ static class LocalImage { - Image baseImage; - List layers; + final Image baseImage; + final List layers; LocalImage(Image baseImage, List layers) { this.baseImage = baseImage; @@ -57,9 +57,17 @@ static class LocalImage { } } + /** + * Checks the first two bytes of a file to see if it has been gzipped. + * + * @param path the file to check + * @return {@code true} if the file is gzipped, {@code false} if not + * @throws IOException if reading the file fails + * @see GZIP file format + */ @VisibleForTesting static boolean isGzipped(Path path) throws IOException { - try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(path))) { + try (InputStream inputStream = Files.newInputStream(path)) { inputStream.mark(2); int magic = (inputStream.read() & 0xff) | ((inputStream.read() << 8) & 0xff00); return magic == GZIPInputStream.GZIP_MAGIC; @@ -78,12 +86,13 @@ static boolean isGzipped(Path path) throws IOException { public LocalImage call() throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException { TarExtractor.extract(tarPath, destination); + + InputStream manifestStream = Files.newInputStream(destination.resolve("manifest.json")); DockerManifestEntryTemplate loadManifest = new ObjectMapper() .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true) - .readValue( - Files.newInputStream(destination.resolve("manifest.json")), - DockerManifestEntryTemplate[].class)[0]; + .readValue(manifestStream, DockerManifestEntryTemplate[].class)[0]; + manifestStream.close(); ContainerConfigurationTemplate configuration = JsonTemplateMapper.readJsonFromFile( destination.resolve(loadManifest.getConfig()), ContainerConfigurationTemplate.class); @@ -106,7 +115,7 @@ public LocalImage call() // Process layer blobs // TODO: Optimize; compressing/calculating layer digests is slow List layers = new ArrayList<>(); - V22ManifestTemplate newManifest = new V22ManifestTemplate(); + V22ManifestTemplate v22Manifest = new V22ManifestTemplate(); for (int index = 0; index < layerFiles.size(); index++) { Path file = destination.resolve(layerFiles.get(index)); @@ -125,16 +134,14 @@ public LocalImage call() .setLayerDiffId(configuration.getLayerDiffId(index)) .build(); - // TODO: Check blob existence on target registry (online mode only) - layers.add(new PreparedLayer.Builder(layer).build()); - newManifest.addLayer(blobDescriptor.getSize(), blobDescriptor.getDigest()); + v22Manifest.addLayer(blobDescriptor.getSize(), blobDescriptor.getDigest()); } BlobDescriptor configDescriptor = Blobs.from(configuration).writeTo(ByteStreams.nullOutputStream()); - newManifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest()); - Image image = JsonToImageTranslator.toImage(newManifest, configuration); + v22Manifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest()); + Image image = JsonToImageTranslator.toImage(v22Manifest, configuration); return new LocalImage(image, layers); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java index 07b6eafb8f..61fe8f75f1 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java @@ -24,12 +24,14 @@ import com.google.cloud.tools.jib.blob.Blobs; import com.google.common.collect.ImmutableList; import com.google.common.io.ByteStreams; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileTime; import java.time.Instant; +import java.util.zip.GZIPInputStream; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -39,6 +41,17 @@ /** Tests for {@link Cache}. */ public class CacheTest { + /** + * Gets a {@link Blob} that is {@code blob} decompressed. + * + * @param blob the {@link Blob} to decompress + * @return the decompressed {@link Blob} + * @throws IOException if an I/O exception occurs + */ + public static Blob decompress(Blob blob) throws IOException { + return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); + } + /** * Gets the digest of {@code blob}. * @@ -177,7 +190,7 @@ public void testRetrieveWithTwoEntriesInCache() throws IOException, CacheCorrupt * @throws IOException if an I/O exception occurs */ private void verifyIsLayer1(CachedLayer cachedLayer) throws IOException { - Assert.assertEquals("layerBlob1", Blobs.writeToString(Blobs.decompress(cachedLayer.getBlob()))); + Assert.assertEquals("layerBlob1", Blobs.writeToString(decompress(cachedLayer.getBlob()))); Assert.assertEquals(layerDigest1, cachedLayer.getDigest()); Assert.assertEquals(layerDiffId1, cachedLayer.getDiffId()); Assert.assertEquals(layerSize1, cachedLayer.getSize()); @@ -190,7 +203,7 @@ private void verifyIsLayer1(CachedLayer cachedLayer) throws IOException { * @throws IOException if an I/O exception occurs */ private void verifyIsLayer2(CachedLayer cachedLayer) throws IOException { - Assert.assertEquals("layerBlob2", Blobs.writeToString(Blobs.decompress(cachedLayer.getBlob()))); + Assert.assertEquals("layerBlob2", Blobs.writeToString(decompress(cachedLayer.getBlob()))); Assert.assertEquals(layerDigest2, cachedLayer.getDigest()); Assert.assertEquals(layerDiffId2, cachedLayer.getDiffId()); Assert.assertEquals(layerSize2, cachedLayer.getSize()); From 64e6d8d64c0860b4b6bdba44458115467f221df3 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Fri, 16 Aug 2019 12:56:55 -0400 Subject: [PATCH 13/16] Feedback --- .../tools/jib/builder/steps/ExtractTarStep.java | 12 ++++++------ .../tools/jib/cache/CacheStorageWriterTest.java | 17 ++++------------- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index f6b9f1238d..bb018fd051 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -93,17 +93,17 @@ public LocalImage call() .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true) .readValue(manifestStream, DockerManifestEntryTemplate[].class)[0]; manifestStream.close(); - ContainerConfigurationTemplate configuration = + ContainerConfigurationTemplate configurationTemplate = JsonTemplateMapper.readJsonFromFile( destination.resolve(loadManifest.getConfig()), ContainerConfigurationTemplate.class); List layerFiles = loadManifest.getLayerFiles(); - if (configuration.getLayerCount() != layerFiles.size()) { + if (configurationTemplate.getLayerCount() != layerFiles.size()) { throw new LayerCountMismatchException( "Invalid base image format: manifest contains " + layerFiles.size() + " layers, but container configuration contains " - + configuration.getLayerCount() + + configurationTemplate.getLayerCount() + " layers"); } @@ -131,7 +131,7 @@ public LocalImage call() .setLayerBlob(blob) .setLayerDigest(blobDescriptor.getDigest()) .setLayerSize(blobDescriptor.getSize()) - .setLayerDiffId(configuration.getLayerDiffId(index)) + .setLayerDiffId(configurationTemplate.getLayerDiffId(index)) .build(); layers.add(new PreparedLayer.Builder(layer).build()); @@ -139,9 +139,9 @@ public LocalImage call() } BlobDescriptor configDescriptor = - Blobs.from(configuration).writeTo(ByteStreams.nullOutputStream()); + Blobs.from(configurationTemplate).writeTo(ByteStreams.nullOutputStream()); v22Manifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest()); - Image image = JsonToImageTranslator.toImage(v22Manifest, configuration); + Image image = JsonToImageTranslator.toImage(v22Manifest, configurationTemplate); return new LocalImage(image, layers); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java index ecf48e2f40..e5bcdb6466 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java @@ -35,7 +35,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -49,15 +48,6 @@ private static BlobDescriptor getDigest(Blob blob) throws IOException { return blob.writeTo(ByteStreams.nullOutputStream()); } - private static Blob compress(Blob blob) { - return Blobs.from( - outputStream -> { - try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) { - blob.writeTo(compressorStream); - } - }); - } - private static Blob decompress(Blob blob) throws IOException { return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); } @@ -78,7 +68,8 @@ public void testWrite_compressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); CachedLayer cachedLayer = - new CacheStorageWriter(cacheStorageFiles).writeCompressed(compress(uncompressedLayerBlob)); + new CacheStorageWriter(cacheStorageFiles) + .writeCompressed(Blobs.compress(uncompressedLayerBlob)); verifyCachedLayer(cachedLayer, uncompressedLayerBlob); } @@ -86,7 +77,7 @@ public void testWrite_compressed() throws IOException { @Test public void testWrite_uncompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); - DescriptorDigest layerDigest = getDigest(compress(uncompressedLayerBlob)).getDigest(); + DescriptorDigest layerDigest = getDigest(Blobs.compress(uncompressedLayerBlob)).getDigest(); DescriptorDigest selector = getDigest(Blobs.from("selector")).getDigest(); CachedLayer cachedLayer = @@ -159,7 +150,7 @@ public void testWriteMetadata_v22() private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob) throws IOException { - BlobDescriptor layerBlobDescriptor = getDigest(compress(uncompressedLayerBlob)); + BlobDescriptor layerBlobDescriptor = getDigest(Blobs.compress(uncompressedLayerBlob)); DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob).getDigest(); // Verifies cachedLayer is correct. From d4669ba594c66a35039cf90c79ddafb0ee97660d Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Fri, 16 Aug 2019 14:33:48 -0400 Subject: [PATCH 14/16] Delete docker saved tar --- .../tools/jib/builder/steps/SaveDockerStep.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java index 3fb28d4f79..bd26e2c4e6 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java @@ -19,6 +19,8 @@ import com.google.cloud.tools.jib.api.ImageReference; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; +import com.google.common.io.MoreFiles; +import com.google.common.io.RecursiveDeleteOption; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -37,7 +39,19 @@ public class SaveDockerStep implements Callable { @Override public Path call() throws IOException, InterruptedException { - Path outputPath = Files.createTempDirectory("jib-docker-save").resolve("out.tar"); + Path outputDir = Files.createTempDirectory("jib-docker-save"); + Path outputPath = outputDir.resolve("out.tar"); + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + if (Files.exists(outputDir)) { + try { + MoreFiles.deleteRecursively(outputDir, RecursiveDeleteOption.ALLOW_INSECURE); + } catch (IOException ignored) { + } + } + })); ImageReference imageReference = buildConfiguration.getBaseImageConfiguration().getImage(); dockerClient.save(imageReference, outputPath); return outputPath; From 1f3362f7a75abd3ece45bdc33c268c7d54d02517 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Fri, 16 Aug 2019 16:02:54 -0400 Subject: [PATCH 15/16] Delete extracted layers too --- .../jib/builder/steps/ExtractTarStep.java | 3 +++ .../jib/builder/steps/SaveDockerStep.java | 15 ++----------- .../tools/jib/filesystem/FileOperations.java | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index bb018fd051..e58baca265 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -24,6 +24,7 @@ import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate; +import com.google.cloud.tools.jib.filesystem.FileOperations; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.LayerCountMismatchException; import com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException; @@ -85,6 +86,8 @@ static boolean isGzipped(Path path) throws IOException { @Override public LocalImage call() throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException { + Files.createDirectories(destination); + FileOperations.deleteDirectoryRecursiveOnExit(destination); TarExtractor.extract(tarPath, destination); InputStream manifestStream = Files.newInputStream(destination.resolve("manifest.json")); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java index bd26e2c4e6..8c22afd793 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java @@ -19,8 +19,7 @@ import com.google.cloud.tools.jib.api.ImageReference; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; -import com.google.common.io.MoreFiles; -import com.google.common.io.RecursiveDeleteOption; +import com.google.cloud.tools.jib.filesystem.FileOperations; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -40,18 +39,8 @@ public class SaveDockerStep implements Callable { @Override public Path call() throws IOException, InterruptedException { Path outputDir = Files.createTempDirectory("jib-docker-save"); + FileOperations.deleteDirectoryRecursiveOnExit(outputDir); Path outputPath = outputDir.resolve("out.tar"); - Runtime.getRuntime() - .addShutdownHook( - new Thread( - () -> { - if (Files.exists(outputDir)) { - try { - MoreFiles.deleteRecursively(outputDir, RecursiveDeleteOption.ALLOW_INSECURE); - } catch (IOException ignored) { - } - } - })); ImageReference imageReference = buildConfiguration.getBaseImageConfiguration().getImage(); dockerClient.save(imageReference, outputPath); return outputPath; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java b/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java index d4f6e09839..9590cad4a7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java @@ -17,6 +17,8 @@ package com.google.cloud.tools.jib.filesystem; import com.google.common.collect.ImmutableList; +import com.google.common.io.MoreFiles; +import com.google.common.io.RecursiveDeleteOption; import java.io.IOException; import java.io.OutputStream; import java.nio.channels.Channels; @@ -85,5 +87,24 @@ public static OutputStream newLockingOutputStream(Path file) throws IOException return Channels.newOutputStream(channel); } + /** + * Sets up a shutdown hook that tries to delete a directory. + * + * @param path the directory + */ + public static void deleteDirectoryRecursiveOnExit(Path path) { + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + if (Files.exists(path)) { + try { + MoreFiles.deleteRecursively(path, RecursiveDeleteOption.ALLOW_INSECURE); + } catch (IOException ignored) { + } + } + })); + } + private FileOperations() {} } From bb65456fdff4a4a1646bca54095711a3b0dcdd2d Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Fri, 16 Aug 2019 16:47:56 -0400 Subject: [PATCH 16/16] Nits --- .../cloud/tools/jib/builder/steps/ExtractTarStep.java | 2 +- .../cloud/tools/jib/builder/steps/SaveDockerStep.java | 2 +- .../google/cloud/tools/jib/filesystem/FileOperations.java | 6 +++--- .../cloud/tools/jib/builder/steps/ExtractTarStepTest.java | 2 +- .../java/com/google/cloud/tools/jib/cache/CacheTest.java | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index e58baca265..62bb7cd6ea 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -87,7 +87,7 @@ static boolean isGzipped(Path path) throws IOException { public LocalImage call() throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException { Files.createDirectories(destination); - FileOperations.deleteDirectoryRecursiveOnExit(destination); + FileOperations.deleteRecursiveOnExit(destination); TarExtractor.extract(tarPath, destination); InputStream manifestStream = Files.newInputStream(destination.resolve("manifest.json")); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java index 8c22afd793..789781aa35 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java @@ -39,7 +39,7 @@ public class SaveDockerStep implements Callable { @Override public Path call() throws IOException, InterruptedException { Path outputDir = Files.createTempDirectory("jib-docker-save"); - FileOperations.deleteDirectoryRecursiveOnExit(outputDir); + FileOperations.deleteRecursiveOnExit(outputDir); Path outputPath = outputDir.resolve("out.tar"); ImageReference imageReference = buildConfiguration.getBaseImageConfiguration().getImage(); dockerClient.save(imageReference, outputPath); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java b/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java index 9590cad4a7..5a36dbec78 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java @@ -88,11 +88,11 @@ public static OutputStream newLockingOutputStream(Path file) throws IOException } /** - * Sets up a shutdown hook that tries to delete a directory. + * Sets up a shutdown hook that tries to delete a file or directory. * - * @param path the directory + * @param path the path to the file or directory */ - public static void deleteDirectoryRecursiveOnExit(Path path) { + public static void deleteRecursiveOnExit(Path path) { Runtime.getRuntime() .addShutdownHook( new Thread( diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java index 3afbe6a866..664743ab89 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java @@ -31,7 +31,7 @@ public class ExtractTarStepTest { - @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); private static Path getResource(String resource) throws URISyntaxException { return Paths.get(Resources.getResource(resource).toURI()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java index 61fe8f75f1..6d8533ba8c 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheTest.java @@ -48,7 +48,7 @@ public class CacheTest { * @return the decompressed {@link Blob} * @throws IOException if an I/O exception occurs */ - public static Blob decompress(Blob blob) throws IOException { + private static Blob decompress(Blob blob) throws IOException { return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); }